def records_to_packages(records): records_with_key = ((r.get_key(), r) for r in records) def record_has_key(record_with_key): key, record = record_with_key return key is not None records_with_key = filter(record_has_key, records_with_key) return map(expand_args(package), grouped_by(records_with_key, first))
def get_job_sources(cursor): """Return list of HarvestJobSource instances.""" query = ("SELECT id, name, job_type, config " "FROM system.job_source " "WHERE job_type = %s") args = (JOB_TYPE, ) cursor.execute(query, args) return map(expand_args(HarvestJobSource), cursor.fetchall())
def get_all(cls, cursor): """Load and return all attributestores.""" query = ( "SELECT id, datasource_id, entitytype_id " "FROM attribute_directory.attributestore" ) cursor.execute(query) load = expand_args(partial(cls.load_attributestore, cursor)) return map(load, cursor.fetchall())
def get_job_sources(cursor): """Return list of HarvestJobSource instances.""" query = ( "SELECT id, name, job_type, config " "FROM system.job_source " "WHERE job_type = %s") args = (JOB_TYPE, ) cursor.execute(query, args) return map(expand_args(HarvestJobSource), cursor.fetchall())
def datarecords_to_packages(records): records_with_key = ((r.get_key(), r) for r in records) def record_with_key_ok(record_with_key): key, record = record_with_key return key is not None records_with_key = filter(record_with_key_ok, records_with_key) sorted_records_with_key = sorted(records_with_key, key=first) return imap(expand_args(package), groupby(sorted_records_with_key, first))
def deduce_attributes(self): """Return list of attributes matching the data in this package.""" data_types = self.deduce_data_types() return map(expand_args(Attribute), zip(self.attribute_names, data_types))
def run(conn, jobsource_id): """The actual job generation loop.""" each(expand_args(partial(enqueue, conn, jobsource_id)), job_generator())