def filter(
producer,
config,
output,
mode=('', u'a',
'The mode to open the files, `a` to append and `w` to rewrite.'),
filters=('', [], 'The filters to use.'),
):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(
dustbin_template) if dustbin_template is not None else None
filters_to_include = config.filters
if filters:
filters_to_include = (f for f in config.filters if f.name in filters)
streams = tuple((
consumers.group(f.split_template, mode=mode) if f.split_template !=
'--' else consumers.show(output=output, template='{t.raw}'),
lambda c, _, f=f: c.filter(**f.predicates),
) for f in filters_to_include)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def filter(
producer, config, output,
mode=('', u'a', 'The mode to open the files, `a` to append and `w` to rewrite.'),
filters=('', [], 'The filters to use.'),
):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None
filters_to_include = config.filters
if filters:
filters_to_include = (f for f in config.filters if f.name in filters)
streams = tuple(
(
consumers.group(f.split_template, mode=mode)
if f.split_template != '--'
else consumers.show(output=output, template='{t.raw}'),
lambda c, _, f=f: c.filter(**f.predicates),
)
for f in filters_to_include
)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def filter(producer, config):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(
dustbin_template) if dustbin_template is not None else None
streams = tuple((
consumers.group(f.split_template),
lambda c, _, f=f: c.filter(**f.predicates),
) for f in config.filters)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def filter(producer, config):
"""Filter the tweets to files by filtering predicates defined in the configuration file."""
dustbin_template = config.dustbin_template
dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None
streams = tuple(
(
consumers.group(f.split_template),
lambda c, _, f=f: c.filter(**f.predicates),
)
for f in config.filters
)
target = consumers.filter(streams, dustbin)
producer(consumers.to_tweet(target))
def group(producer,
file_name_template=('t', '%Y-%m-%d-%H.gz', ''),
):
"""Group tweets to files by date according to the template."""
producer(consumers.to_tweet(consumers.group(file_name_template)))
def group(
producer,
file_name_template=('t', '%Y-%m-%d-%H.gz', ''),
):
"""Group tweets to files by date according to the template."""
producer(consumers.to_tweet(consumers.group(file_name_template)))
