def filter( producer, config, output, mode=('', u'a', 'The mode to open the files, `a` to append and `w` to rewrite.'), filters=('', [], 'The filters to use.'), ): """Filter the tweets to files by filtering predicates defined in the configuration file.""" dustbin_template = config.dustbin_template dustbin = consumers.group( dustbin_template) if dustbin_template is not None else None filters_to_include = config.filters if filters: filters_to_include = (f for f in config.filters if f.name in filters) streams = tuple(( consumers.group(f.split_template, mode=mode) if f.split_template != '--' else consumers.show(output=output, template='{t.raw}'), lambda c, _, f=f: c.filter(**f.predicates), ) for f in filters_to_include) target = consumers.filter(streams, dustbin) producer(consumers.to_tweet(target))
def filter( producer, config, output, mode=('', u'a', 'The mode to open the files, `a` to append and `w` to rewrite.'), filters=('', [], 'The filters to use.'), ): """Filter the tweets to files by filtering predicates defined in the configuration file.""" dustbin_template = config.dustbin_template dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None filters_to_include = config.filters if filters: filters_to_include = (f for f in config.filters if f.name in filters) streams = tuple( ( consumers.group(f.split_template, mode=mode) if f.split_template != '--' else consumers.show(output=output, template='{t.raw}'), lambda c, _, f=f: c.filter(**f.predicates), ) for f in filters_to_include ) target = consumers.filter(streams, dustbin) producer(consumers.to_tweet(target))
def filter(producer, config): """Filter the tweets to files by filtering predicates defined in the configuration file.""" dustbin_template = config.dustbin_template dustbin = consumers.group( dustbin_template) if dustbin_template is not None else None streams = tuple(( consumers.group(f.split_template), lambda c, _, f=f: c.filter(**f.predicates), ) for f in config.filters) target = consumers.filter(streams, dustbin) producer(consumers.to_tweet(target))
def filter(producer, config): """Filter the tweets to files by filtering predicates defined in the configuration file.""" dustbin_template = config.dustbin_template dustbin = consumers.group(dustbin_template) if dustbin_template is not None else None streams = tuple( ( consumers.group(f.split_template), lambda c, _, f=f: c.filter(**f.predicates), ) for f in config.filters ) target = consumers.filter(streams, dustbin) producer(consumers.to_tweet(target))
def group(producer, file_name_template=('t', '%Y-%m-%d-%H.gz', ''), ): """Group tweets to files by date according to the template.""" producer(consumers.to_tweet(consumers.group(file_name_template)))
def group( producer, file_name_template=('t', '%Y-%m-%d-%H.gz', ''), ): """Group tweets to files by date according to the template.""" producer(consumers.to_tweet(consumers.group(file_name_template)))