def main(args): sts = datetime.datetime.now() miner = MetaModel(args.src) columnmapper = ColumnMapper() tablemapper = TableMapper() pkmapper = PrimaryKeyMapper() fkmapper = ForeignKeyMapper() columns = miner.columns() tables = miner.tables() fks = fkmapper.multiple(miner.foreignKeys()) pks = pkmapper.multiple(miner.primaryKeys()) print('## cols: ' + str(len(columns))) print('## tables: ' + str(len(tables))) print('## fks: ' + str(len(fks))) print('## pks: ' + str(len(pks))) print('') print('## processing columns...') pcolumns = [] if not args.explore: cp = MPColumnProcessor(connection_string=args.src, columns=columns, columnprocessor=NumpyColumnProcessor, mapper=columnmapper) pcolumns = cp.execute(processes=int(args.cpu), verbose=True) else: pcolumns = columnmapper.multiple([(column, None) for column in columns]) # cets = datetime.datetime.now() # Notifier.notify(title='cobr.io ds-toolkit', # subtitle='MPColumnProcessor done!', # message='processed: ' + str(len(pcolumns)) + ' columns in ' + str(math.floor((cets - sts).total_seconds())) + ' seconds') print('') print('## processing tables...') tp = MPTableProcessor(connection_string=args.src, tables=tables, mapper=tablemapper) ptables = tp.execute(processes=int(args.cpu), verbose=True) # Notifier.notify(title='cobr.io ds-toolkit', # subtitle='MPTableProcessor done!', # message='processed: ' + str(len(ptables)) + ' tables in ' + str(math.floor((datetime.datetime.now() - cets).total_seconds())) + ' seconds') if not args.dry_run and args.target: engine = create_engine(args.target) Session = sessionmaker(bind=engine) session = Session() writeToDb(session, ptables) writeToDb(session, pcolumns) writeToDb(session, pks) writeToDb(session, fks) print('') print('## time elapsed: ' + str(datetime.datetime.now() - sts))
def main(args): miner = MetaModel(args.src) tables = miner.tables() columns = miner.columns() pkexplorer = PkExplorer(connection_string=args.src) pks = pkexplorer.doMultipleTables(tables) if not args.dry_run and args.target: engine = create_engine(args.target) Session = sessionmaker(bind=engine) session = Session() writeToDb(session, pks)
def main(args): colseparator = '|' miner = MetaModel(args.src) tables = miner.tables() columns = miner.columns() dic = {} for pk in [ c for c in columns if c.primary_key ]: if (pk.info['schemaname'], pk.table.name) not in dic: dic[(pk.info['schemaname'], pk.table.name)] = [] dic[(pk.info['schemaname'], pk.table.name)].append(pk) # split primary keys into singlecolumn and multicolumn keys spks = [ dic[key] for key in dic.keys() if len(dic[key]) == 1 ] mpks = [ dic[key] for key in dic.keys() if len(dic[key]) > 1 ] print("LOT'S OF WORK IN PROGRESS...") exit() affaires = Discovery(tables=tables, columns=columns, pksingle=spks, pkmulti=mkps) fks = pruneDuplicateFks([ fk[0] for fk in affaires.discoverfks(0.9) ])
conn = engine.connect() num_rows = conn.execute(table.count()).fetchone()[0] num_columns = len(table.columns) num_explicit_outlinks = len(table.foreign_keys) return self.mapper.single(table, {'num_rows': num_rows, 'num_columns': num_columns, 'num_explicit_outlinks': num_explicit_outlinks}) # return (table.name, {'num_rows': num_rows, 'num_columns': num_columns, 'num_explicit_outlinks': num_explicit_outlinks}) # cp = self.columnprocessor(values) # return (column, cp.doOperations()) except Exception as ex: print(ex) finally: conn.close() if __name__ == "__main__": parser = argparse.ArgumentParser() parser.add_argument("-s", "--src", help="connection_string for the subject-database", metavar="string") args = parser.parse_args() mm = MetaModel(args.src) sts = datetime.datetime.now() processor = MPTableProcessor(connection_string = args.src, tables = mm.tables()) result = processor.execute(processes=32, verbose=True) duration = datetime.datetime.now() - sts print('number of processed tables: ' + str(len(result))) # Calling the notification function # Notifier.notify(title='cobr.io ds-toolkit', subtitle='MPTableProcessor done!', message='processed: ' + str(len(result)) + ' tables in ' + str(math.floor(duration.total_seconds())) + ' seconds')