def filter(
producer,
config,
output,
mode=('', u'a',
'The mode to open the files, `a` to append and `w` to rewrite.'),
filters=('', [], 'The filters to use.'),
):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(
dustbin_template) if dustbin_template is not None else None
filters_to_include = config.filters
if filters:
filters_to_include = (f for f in config.filters if f.name in filters)
streams = tuple((
consumers.group(f.split_template, mode=mode) if f.split_template !=
'--' else consumers.show(output=output, template='{t.raw}'),
lambda c, _, f=f: c.filter(**f.predicates),
) for f in filters_to_include)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def test_filter(tweets):
pinkpop = []
dimazest = []
dustbin = []
streams = tuple((to_list(l),
lambda c, f, p=p: c.filter(**p)
) for l, p in [(pinkpop, {'follow': [],
'track': ['pinkpop'],
'locations': [],
},
),
(dimazest, {'follow': [10868922],
'track': [],
'locations': [],
},
),
]
)
target = consumers.filter(streams, to_list(dustbin))
from_iterable(consumers.to_tweet(target), tweets)
assert len(pinkpop) == 1
assert pinkpop[0].id == 190800262909276162
assert len(dimazest) == 3
assert not dustbin
def test_filter(tweets):
pinkpop = []
dimazest = []
dustbin = []
streams = tuple((to_list(l), lambda c, f, p=p: c.filter(**p)) for l, p in [
(
pinkpop,
{
'follow': [],
'track': ['pinkpop'],
'locations': [],
},
),
(
dimazest,
{
'follow': [10868922],
'track': [],
'locations': [],
},
),
])
target = consumers.filter(streams, to_list(dustbin))
from_iterable(consumers.to_tweet(target), tweets)
assert len(pinkpop) == 1
assert pinkpop[0].id == 190800262909276162
assert len(dimazest) == 3
assert not dustbin
def filter(
producer, config, output,
mode=('', u'a', 'The mode to open the files, `a` to append and `w` to rewrite.'),
filters=('', [], 'The filters to use.'),
):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None
filters_to_include = config.filters
if filters:
filters_to_include = (f for f in config.filters if f.name in filters)
streams = tuple(
(
consumers.group(f.split_template, mode=mode)
if f.split_template != '--'
else consumers.show(output=output, template='{t.raw}'),
lambda c, _, f=f: c.filter(**f.predicates),
)
for f in filters_to_include
)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def test_filter_numbers():
items = 4, 2, 3, 4, 5, 9, 0, 2, 8
result = []
streams = ((to_list(result), lambda c, f: c > f),)
from_iterable(consumers.filter(streams), items)
assert result == [5, 9, 8]
def test_filter_numbers():
items = 4, 2, 3, 4, 5, 9, 0, 2, 8
result = []
streams = ((to_list(result), lambda c, f: c > f), )
from_iterable(consumers.filter(streams), items)
assert result == [5, 9, 8]
def test_filter_dustbin(tweets):
result = []
dustbin = []
streams = ((to_list(result), lambda c, f: c.filter(follow=[-1000])), )
target = consumers.filter(streams, to_list(dustbin))
from_iterable(consumers.to_tweet(target), tweets)
assert not result
assert len(dustbin) == 3
def test_filter_dustbin(tweets):
result = []
dustbin = []
streams = ((to_list(result), lambda c, f: c.filter(follow=[-1000])), )
target = consumers.filter(streams, to_list(dustbin))
from_iterable(consumers.to_tweet(target), tweets)
assert not result
assert len(dustbin) == 3
def filter(producer, config):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(
dustbin_template) if dustbin_template is not None else None
streams = tuple((
consumers.group(f.split_template),
lambda c, _, f=f: c.filter(**f.predicates),
) for f in config.filters)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def filter(producer, config):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None
streams = tuple(
(
consumers.group(f.split_template),
lambda c, _, f=f: c.filter(**f.predicates),
)
for f in config.filters
)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))