def records_to_packages(records):
records_with_key = ((r.get_key(), r) for r in records)
def record_has_key(record_with_key):
key, record = record_with_key
return key is not None
records_with_key = filter(record_has_key, records_with_key)
return map(expand_args(package), grouped_by(records_with_key, first))
def get_job_sources(cursor):
"""Return list of HarvestJobSource instances."""
query = ("SELECT id, name, job_type, config "
"FROM system.job_source "
"WHERE job_type = %s")
args = (JOB_TYPE, )
cursor.execute(query, args)
return map(expand_args(HarvestJobSource), cursor.fetchall())
def records_to_packages(records):
records_with_key = ((r.get_key(), r) for r in records)
def record_has_key(record_with_key):
key, record = record_with_key
return key is not None
records_with_key = filter(record_has_key, records_with_key)
return map(expand_args(package), grouped_by(records_with_key, first))
def get_all(cls, cursor):
"""Load and return all attributestores."""
query = (
"SELECT id, datasource_id, entitytype_id "
"FROM attribute_directory.attributestore"
)
cursor.execute(query)
load = expand_args(partial(cls.load_attributestore, cursor))
return map(load, cursor.fetchall())
def get_job_sources(cursor):
"""Return list of HarvestJobSource instances."""
query = (
"SELECT id, name, job_type, config "
"FROM system.job_source "
"WHERE job_type = %s")
args = (JOB_TYPE, )
cursor.execute(query, args)
return map(expand_args(HarvestJobSource), cursor.fetchall())
def datarecords_to_packages(records):
records_with_key = ((r.get_key(), r) for r in records)
def record_with_key_ok(record_with_key):
key, record = record_with_key
return key is not None
records_with_key = filter(record_with_key_ok, records_with_key)
sorted_records_with_key = sorted(records_with_key, key=first)
return imap(expand_args(package), groupby(sorted_records_with_key, first))
def deduce_attributes(self):
"""Return list of attributes matching the data in this package."""
data_types = self.deduce_data_types()
return map(expand_args(Attribute),
zip(self.attribute_names, data_types))
def run(conn, jobsource_id):
"""The actual job generation loop."""
each(expand_args(partial(enqueue, conn, jobsource_id)), job_generator())
