def config_from_dict(self, config_dict, name):
"""
This worker needs two maps: the primary keys for each target table
and the table that each transformer has mapped.
Sequences may also be added to let the worker resets the values to the max id
"""
worker_section = config_dict.get(self.name, dict())
self.tables_workers = worker_section.get('tables_workers', dict())
self.tables_keys = dict()
tables_keys = worker_section.get('tables_keys', dict())
for table, keys in tables_keys.iteritems():
if not hasattr(keys, '__iter__'):
keys = [keys]
self.tables_keys[table] = keys
semaphore_name = worker_section.get('semaphore', None)
if semaphore_name:
semaphore_section = config_dict.get(semaphore_name, dict())
self.semaphore = init_object_from_dict(semaphore_section)
self.semaphore.config_from_dict(config_dict, semaphore_name)
self.can_unlock = str_to_bool(worker_section.get('can_unlock', False))
self.tables_sequences = dict()
for table, seq_info in worker_section.get('tables_sequences',
dict()).items():
self.tables_sequences.setdefault(table, seq_info)
super(DBLoader, self).config_from_dict(config_dict, self.name)
def config_from_dict(self, config_dict, name):
"""
Gets the query, the table primary keys and optional
filters that are parametric
"""
worker_section = config_dict.get(self.name, dict())
self.query = worker_section.get('query', None)
self.keys = worker_section.get('keys', None)
self.keys_are_primary = str_to_bool(worker_section.get('keys_are_primary', True))
self.filters = worker_section.get('filters', None)
super(DBExtractor, self).config_from_dict(config_dict, self.name)
def config_from_dict(self, config_dict, name):
worker_section = config_dict.get(self.name, dict())
self.separator = worker_section.get('separator', '\t')
self.has_headers = str_to_bool(worker_section.get('has_headers', True))
self.filepath = worker_section.get('filepath', None)
if not self.filepath or os.path.basename(self.filepath).split('.')[1] != 'csv':
raise Exception('Missing or wrong file configuration')
self.headers = worker_section.get('headers', None)
if not hasattr(self.headers, '__iter__'):
self.headers = [self.headers]
if not self.headers:
raise Exception('csv headers configuration missing')
super(CSVExtractor, self).config_from_dict(config_dict, name)
def config_from_dict(self, config_dict, name):
"""
Sets the needed database specifications as class attributes
"""
my_section = config_dict.get(name, dict())
self.use_split = str_to_bool(my_section.get('use_split', True))
if 'dbinfo' in my_section:
db_section = config_dict.get(my_section['dbinfo'], None)
if db_section:
self.host = db_section['host']
self.user = db_section['user']
self.dbname = db_section['dbname']
self.password = db_section.get('password', None)
self.port = db_section.get('port', 5432)
else:
print "ERROR CONFIGURING WORKER ", name
print "UNEXISTENT dbinfo SPECIFIED [%s] FOR %s CONFIGURATION" % (my_section['dbinfo'], name)
else:
print "ERROR CONFIGURING WORKER ", name
print "MANDATORY dbinfo OPTION MISSING FROM %s CONFIGURATION" % name
super(_DatabaseHandler, self).config_from_dict(config_dict, name)