Example #1
0
            }))
    engines.append(select)

    # Generate appropriate groups
    states_counties_grouper = Group(select.output(), {
        'states.oid': lambda a, b: a == b,
        'counties.oid': lambda a, b: a == b
    })
    engines.append(states_counties_grouper)

    joiner = Join(states_counties_grouper.output(),
                  geonames_aggregate.output())
    engines.append(joiner)
    mux_streams.append(joiner.output())

mux = Mux(*mux_streams)
engines.append(mux)

states_level_select = Select(
    mux.output(),
    UniversalSelect(
        mux.output().schema(), {
            'states.oid': {
                'type': int,
                'args': ['states.oid'],
                'function': lambda v: v,
            },
            'count': {
                'type': int,
                'args': ['count'],
                'function': lambda v: v,
Example #2
0
                'type': int,
                'args': ['oid'],
                'function': lambda v: v
            },
        }))
    engines.append(select)

    counties_grouper = Group(select.output(), {'oid': lambda a, b: a == b})
    engines.append(counties_grouper)

    joiner = Join(counties_grouper.output(), geonames_aggregate.output())
    engines.append(joiner)
    mux_streams.append(joiner.output())
    # mux_streams.append(counties_select.output())

mux = Mux(*mux_streams)
engines.append(mux)

result_stack = ResultFile(
    'results.txt',
    mux.output(),
)
engines.append(result_stack)

#result_stack = ResultStack(
#    mux.output(),
#)
#engines.append(result_stack)

info_queue = Queue()
Example #3
0
    family_genus_species_id_grouper = Group(
        channel, {
            'family.id': lambda a, b: a == b,
            'genus.id': lambda a, b: a == b,
            'species.id': lambda a, b: a == b
        })
    engines.append(family_genus_species_id_grouper)
    # mux_streams.append(family_genus_species_id_grouper.output())

    species_plants_joiner = Join(family_genus_species_id_grouper.output(),
                                 plants_height_aggregate.output())
    engines.append(species_plants_joiner)
    mux_streams.append(species_plants_joiner.output())

mux = Mux(*mux_streams)
engines.append(mux)

# First aggregation level output selection
family_genus_species_select = Select(
    mux.output(),
    UniversalSelect(mux.output().schema(), [
        ('family.id', {
            'type': int,
            'args': ['family.id'],
            'function': lambda v: v
        }),
        ('genus.id', {
            'type': int,
            'args': ['genus.id'],
            'function': lambda v: v
Example #4
0
    engines.append(geonames_aggregate)

    select = Select(
        channel, UniversalSelect(channel.schema(), {"oid": {"type": int, "args": ["oid"], "function": lambda v: v}})
    )
    engines.append(select)

    counties_grouper = Group(select.output(), {"oid": lambda a, b: a == b})
    engines.append(counties_grouper)

    joiner = Join(counties_grouper.output(), geonames_aggregate.output())
    engines.append(joiner)
    mux_streams.append(joiner.output())
    # mux_streams.append(counties_select.output())

mux = Mux(*mux_streams)
engines.append(mux)

result_stack = ResultFile("results.txt", mux.output())
engines.append(result_stack)

# result_stack = ResultStack(
#    mux.output(),
# )
# engines.append(result_stack)

info_queue = Queue()


def manage(task):
    task.run()
Example #5
0
    engines.append(cover_area)

#############################################################
#
# 1st level aggregation
#
#############################################################

    cover_aggregate = Aggregate(
        cover_area.output(),
        SumAggregator(cover_area.output().schema(), 'area')
    )
    engines.append(cover_aggregate)
    mux_streams.append(cover_aggregate.output())

mux = Mux(*mux_streams)
engines.append(mux)

#############################################################
#
# 2nd level aggregation
#
#############################################################

counties_level_select = Select(
    mux.output(),
    UniversalSelect(
        mux.output().schema(),
        [
            ('states.oid', {
                'type': int,
Example #6
0
data_schema.append(Attribute('rowid', int, True))
data_source = DBTable('test.db', 'person', data_schema)

# definition of the data source
#data_source = CSVFile('test.csv', data_schema)

data_accessors = []
selects = []
for i in range(0, 1):
    # create a data accessor
    data_accessor = DataAccessor(demux, data_source, FindRange)
    name_age_combiner = NameAgeCombiner(data_accessor.output().schema())
    selects.append(Select(data_accessor.output(), name_age_combiner))
    data_accessors.append(data_accessor)

mux = Mux(*[s.output() for s in selects])

#name_age_combiner_reverse = NameAgeCombinerReverse(demux.schema())
#select2 = Select(demux, name_age_combiner_reverse)

#name_age_combiner = NameAgeCombiner(data_accessor.output().schema())
#select = Select(data_accessor.output(), name_age_combiner)
#name_age_combiner_reverse = NameAgeCombinerReverse(data_accessor.output().schema())
#select2 = Select(data_accessor.output(), name_age_combiner_reverse)

result_stack = ResultStack(
    #    query_streamer.output(),
    mux.output(),
    #    data_accessor.output(),
)
Example #7
0
#data_source = CSVFile('test.csv', data_schema)

data_accessors = []
selects = []
for i in range(0, 1):
    # create a data accessor
    data_accessor = DataAccessor(
        demux, 
        data_source,
        FindRange
    )
    name_age_combiner = NameAgeCombiner(data_accessor.output().schema())
    selects.append(Select(data_accessor.output(), name_age_combiner))
    data_accessors.append(data_accessor)

mux = Mux(*[s.output() for s in selects])

#name_age_combiner_reverse = NameAgeCombinerReverse(demux.schema())
#select2 = Select(demux, name_age_combiner_reverse)

#name_age_combiner = NameAgeCombiner(data_accessor.output().schema())
#select = Select(data_accessor.output(), name_age_combiner)
#name_age_combiner_reverse = NameAgeCombinerReverse(data_accessor.output().schema())
#select2 = Select(data_accessor.output(), name_age_combiner_reverse)

result_stack = ResultStack(
#    query_streamer.output(),
    mux.output(),
#    data_accessor.output(),
)