def polymorphic_csv_populator(self, entity): """ Reads the CSV into Knowledge. TODO: no dynamic dialects? """ try: #flush_after = asint(config.get('transaction_size', 1000)) repo = utils.get_fact_from_parents(u'repo', entity) custom_dialect = self.dialects.get(repo, None) if not custom_dialect: # Nothing to see hre, carry on. pass elif custom_dialect not in csv.list_dialects(): self.log.error("Dialect '%s' not found!" % custom_dialect) csv_reader = csv.reader(entity['filename'], dialect=custom_dialect) columns = None # TODO: See if this file has already been parsed! for i, line in enumerate(csv_reader): if i == 0: columns = line # then create a Table object with the appropriate columns table_name = u'civx_' + unicode(uuid.uuid4()).replace('-', '') entity[u'table_name'] = table_name entity[u'column_names'] = columns # The actual column names behind the scenes. CIVX will # map them to the 'column_names' entity[u'columns'] = [u'col_%d' % i for i in range(len(columns))] #DBSession.flush() table, model = utils.get_mapped_table_model_from_entity(entity) model.__table__ = table civx.model.models[model] = { 'csv': [entity[u'filename']], 'columns': entity[u'columns'], 'tmp_csv': {} } metadata.create_all() continue break self.log.info("%d entries in %r table" % ( DBSession.query(model).count(), entity[u'table_name'])) populate_csv(( utils.get_fact_from_parents('repo', entity), entity[u'filename'], model, self.engine), dialect=custom_dialect) self.log.info("%d entries in %r table" % ( DBSession.query(model).count(), entity[u'table_name'])) DBSession.commit() except Exception, e: self.log.error('Unable to parse file as CSV') self.log.exception(e)
def ascii_text_handler(self, entity): self.log.debug("ascii_text_handler(%s)" % entity) link = utils.get_fact_from_parents(u'link', entity) repo = utils.get_fact_from_parents(u'repo', entity) #~ self.git_add_and_commit(entity.name, repo=repo) self.polymorphic_csv_populator(entity)