engines.append(select)
counties_grouper = Group(select.output(), {'oid': lambda a, b: a == b})
engines.append(counties_grouper)
joiner = Join(counties_grouper.output(), geonames_aggregate.output())
engines.append(joiner)
mux_streams.append(joiner.output())
# mux_streams.append(counties_select.output())
mux = Mux(*mux_streams)
engines.append(mux)
result_stack = ResultFile(
'results.txt',
mux.output(),
)
engines.append(result_stack)
#result_stack = ResultStack(
# mux.output(),
#)
#engines.append(result_stack)
info_queue = Queue()
def manage(task):
task.run()
# print 'Task %s: completed' % (task.name)
info_queue.put((task, ThreadInfo()))
states_counties_grouper = Group(select.output(), {
'states.oid': lambda a, b: a == b,
'counties.oid': lambda a, b: a == b
})
engines.append(states_counties_grouper)
joiner = Join(states_counties_grouper.output(),
geonames_aggregate.output())
engines.append(joiner)
mux_streams.append(joiner.output())
mux = Mux(*mux_streams)
engines.append(mux)
states_level_select = Select(
mux.output(),
UniversalSelect(
mux.output().schema(), {
'states.oid': {
'type': int,
'args': ['states.oid'],
'function': lambda v: v,
},
'count': {
'type': int,
'args': ['count'],
'function': lambda v: v,
}
}))
engines.append(states_level_select)
'species.id': lambda a, b: a == b
})
engines.append(family_genus_species_id_grouper)
# mux_streams.append(family_genus_species_id_grouper.output())
species_plants_joiner = Join(family_genus_species_id_grouper.output(),
plants_height_aggregate.output())
engines.append(species_plants_joiner)
mux_streams.append(species_plants_joiner.output())
mux = Mux(*mux_streams)
engines.append(mux)
# First aggregation level output selection
family_genus_species_select = Select(
mux.output(),
UniversalSelect(mux.output().schema(), [
('family.id', {
'type': int,
'args': ['family.id'],
'function': lambda v: v
}),
('genus.id', {
'type': int,
'args': ['genus.id'],
'function': lambda v: v
}),
('species.id', {
'type': int,
'args': ['species.id'],
'function': lambda v: v
channel, UniversalSelect(channel.schema(), {"oid": {"type": int, "args": ["oid"], "function": lambda v: v}})
)
engines.append(select)
counties_grouper = Group(select.output(), {"oid": lambda a, b: a == b})
engines.append(counties_grouper)
joiner = Join(counties_grouper.output(), geonames_aggregate.output())
engines.append(joiner)
mux_streams.append(joiner.output())
# mux_streams.append(counties_select.output())
mux = Mux(*mux_streams)
engines.append(mux)
result_stack = ResultFile("results.txt", mux.output())
engines.append(result_stack)
# result_stack = ResultStack(
# mux.output(),
# )
# engines.append(result_stack)
info_queue = Queue()
def manage(task):
task.run()
# print 'Task %s: completed' % (task.name)
info_queue.put((task, ThreadInfo()))
# print 'Task %s: info-queued.' % (task.name)
SumAggregator(cover_area.output().schema(), 'area')
)
engines.append(cover_aggregate)
mux_streams.append(cover_aggregate.output())
mux = Mux(*mux_streams)
engines.append(mux)
#############################################################
#
# 2nd level aggregation
#
#############################################################
counties_level_select = Select(
mux.output(),
UniversalSelect(
mux.output().schema(),
[
('states.oid', {
'type': int,
'args': ['states.oid'],
'function': lambda v: v,
}),
('counties.oid', {
'type': int,
'args': ['counties.oid'],
'function': lambda v: v,
}),
('area', {
'type': float,
selects.append(Select(data_accessor.output(), name_age_combiner))
data_accessors.append(data_accessor)
mux = Mux(*[s.output() for s in selects])
#name_age_combiner_reverse = NameAgeCombinerReverse(demux.schema())
#select2 = Select(demux, name_age_combiner_reverse)
#name_age_combiner = NameAgeCombiner(data_accessor.output().schema())
#select = Select(data_accessor.output(), name_age_combiner)
#name_age_combiner_reverse = NameAgeCombinerReverse(data_accessor.output().schema())
#select2 = Select(data_accessor.output(), name_age_combiner_reverse)
result_stack = ResultStack(
# query_streamer.output(),
mux.output(),
# data_accessor.output(),
)
info_queue = Queue()
def manage(task):
print 'Running: ' + str(task)
task.run()
info_queue.put(ThreadInfo())
tasks = []
tasks += [('Select', s) for s in selects]
tasks += [('Data Accessor', da) for da in data_accessors]