def polymorphic_csv_populator(self, entity):
"""
Reads the CSV into Knowledge.
TODO: no dynamic dialects?
"""
try:
#flush_after = asint(config.get('transaction_size', 1000))
repo = utils.get_fact_from_parents(u'repo', entity)
custom_dialect = self.dialects.get(repo, None)
if not custom_dialect:
# Nothing to see hre, carry on.
pass
elif custom_dialect not in csv.list_dialects():
self.log.error("Dialect '%s' not found!" % custom_dialect)
csv_reader = csv.reader(entity['filename'], dialect=custom_dialect)
columns = None
# TODO: See if this file has already been parsed!
for i, line in enumerate(csv_reader):
if i == 0:
columns = line
# then create a Table object with the appropriate columns
table_name = u'civx_' + unicode(uuid.uuid4()).replace('-', '')
entity[u'table_name'] = table_name
entity[u'column_names'] = columns
# The actual column names behind the scenes. CIVX will
# map them to the 'column_names'
entity[u'columns'] = [u'col_%d' % i for i in
range(len(columns))]
#DBSession.flush()
table, model = utils.get_mapped_table_model_from_entity(entity)
model.__table__ = table
civx.model.models[model] = {
'csv': [entity[u'filename']],
'columns': entity[u'columns'],
'tmp_csv': {}
}
metadata.create_all()
continue
break
self.log.info("%d entries in %r table" % (
DBSession.query(model).count(),
entity[u'table_name']))
populate_csv((
utils.get_fact_from_parents('repo', entity),
entity[u'filename'],
model,
self.engine), dialect=custom_dialect)
self.log.info("%d entries in %r table" % (
DBSession.query(model).count(),
entity[u'table_name']))
DBSession.commit()
except Exception, e:
self.log.error('Unable to parse file as CSV')
self.log.exception(e)
def ascii_text_handler(self, entity):
self.log.debug("ascii_text_handler(%s)" % entity)
link = utils.get_fact_from_parents(u'link', entity)
repo = utils.get_fact_from_parents(u'repo', entity)
#~ self.git_add_and_commit(entity.name, repo=repo)
self.polymorphic_csv_populator(entity)
