})) engines.append(select) # Generate appropriate groups states_counties_grouper = Group(select.output(), { 'states.oid': lambda a, b: a == b, 'counties.oid': lambda a, b: a == b }) engines.append(states_counties_grouper) joiner = Join(states_counties_grouper.output(), geonames_aggregate.output()) engines.append(joiner) mux_streams.append(joiner.output()) mux = Mux(*mux_streams) engines.append(mux) states_level_select = Select( mux.output(), UniversalSelect( mux.output().schema(), { 'states.oid': { 'type': int, 'args': ['states.oid'], 'function': lambda v: v, }, 'count': { 'type': int, 'args': ['count'], 'function': lambda v: v,
'type': int, 'args': ['oid'], 'function': lambda v: v }, })) engines.append(select) counties_grouper = Group(select.output(), {'oid': lambda a, b: a == b}) engines.append(counties_grouper) joiner = Join(counties_grouper.output(), geonames_aggregate.output()) engines.append(joiner) mux_streams.append(joiner.output()) # mux_streams.append(counties_select.output()) mux = Mux(*mux_streams) engines.append(mux) result_stack = ResultFile( 'results.txt', mux.output(), ) engines.append(result_stack) #result_stack = ResultStack( # mux.output(), #) #engines.append(result_stack) info_queue = Queue()
family_genus_species_id_grouper = Group( channel, { 'family.id': lambda a, b: a == b, 'genus.id': lambda a, b: a == b, 'species.id': lambda a, b: a == b }) engines.append(family_genus_species_id_grouper) # mux_streams.append(family_genus_species_id_grouper.output()) species_plants_joiner = Join(family_genus_species_id_grouper.output(), plants_height_aggregate.output()) engines.append(species_plants_joiner) mux_streams.append(species_plants_joiner.output()) mux = Mux(*mux_streams) engines.append(mux) # First aggregation level output selection family_genus_species_select = Select( mux.output(), UniversalSelect(mux.output().schema(), [ ('family.id', { 'type': int, 'args': ['family.id'], 'function': lambda v: v }), ('genus.id', { 'type': int, 'args': ['genus.id'], 'function': lambda v: v
engines.append(geonames_aggregate) select = Select( channel, UniversalSelect(channel.schema(), {"oid": {"type": int, "args": ["oid"], "function": lambda v: v}}) ) engines.append(select) counties_grouper = Group(select.output(), {"oid": lambda a, b: a == b}) engines.append(counties_grouper) joiner = Join(counties_grouper.output(), geonames_aggregate.output()) engines.append(joiner) mux_streams.append(joiner.output()) # mux_streams.append(counties_select.output()) mux = Mux(*mux_streams) engines.append(mux) result_stack = ResultFile("results.txt", mux.output()) engines.append(result_stack) # result_stack = ResultStack( # mux.output(), # ) # engines.append(result_stack) info_queue = Queue() def manage(task): task.run()
engines.append(cover_area) ############################################################# # # 1st level aggregation # ############################################################# cover_aggregate = Aggregate( cover_area.output(), SumAggregator(cover_area.output().schema(), 'area') ) engines.append(cover_aggregate) mux_streams.append(cover_aggregate.output()) mux = Mux(*mux_streams) engines.append(mux) ############################################################# # # 2nd level aggregation # ############################################################# counties_level_select = Select( mux.output(), UniversalSelect( mux.output().schema(), [ ('states.oid', { 'type': int,
data_schema.append(Attribute('rowid', int, True)) data_source = DBTable('test.db', 'person', data_schema) # definition of the data source #data_source = CSVFile('test.csv', data_schema) data_accessors = [] selects = [] for i in range(0, 1): # create a data accessor data_accessor = DataAccessor(demux, data_source, FindRange) name_age_combiner = NameAgeCombiner(data_accessor.output().schema()) selects.append(Select(data_accessor.output(), name_age_combiner)) data_accessors.append(data_accessor) mux = Mux(*[s.output() for s in selects]) #name_age_combiner_reverse = NameAgeCombinerReverse(demux.schema()) #select2 = Select(demux, name_age_combiner_reverse) #name_age_combiner = NameAgeCombiner(data_accessor.output().schema()) #select = Select(data_accessor.output(), name_age_combiner) #name_age_combiner_reverse = NameAgeCombinerReverse(data_accessor.output().schema()) #select2 = Select(data_accessor.output(), name_age_combiner_reverse) result_stack = ResultStack( # query_streamer.output(), mux.output(), # data_accessor.output(), )
#data_source = CSVFile('test.csv', data_schema) data_accessors = [] selects = [] for i in range(0, 1): # create a data accessor data_accessor = DataAccessor( demux, data_source, FindRange ) name_age_combiner = NameAgeCombiner(data_accessor.output().schema()) selects.append(Select(data_accessor.output(), name_age_combiner)) data_accessors.append(data_accessor) mux = Mux(*[s.output() for s in selects]) #name_age_combiner_reverse = NameAgeCombinerReverse(demux.schema()) #select2 = Select(demux, name_age_combiner_reverse) #name_age_combiner = NameAgeCombiner(data_accessor.output().schema()) #select = Select(data_accessor.output(), name_age_combiner) #name_age_combiner_reverse = NameAgeCombinerReverse(data_accessor.output().schema()) #select2 = Select(data_accessor.output(), name_age_combiner_reverse) result_stack = ResultStack( # query_streamer.output(), mux.output(), # data_accessor.output(), )