def main(args):
sts = datetime.datetime.now()
miner = MetaModel(args.src)
columnmapper = ColumnMapper()
tablemapper = TableMapper()
pkmapper = PrimaryKeyMapper()
fkmapper = ForeignKeyMapper()
columns = miner.columns()
tables = miner.tables()
fks = fkmapper.multiple(miner.foreignKeys())
pks = pkmapper.multiple(miner.primaryKeys())
print('## cols: ' + str(len(columns)))
print('## tables: ' + str(len(tables)))
print('## fks: ' + str(len(fks)))
print('## pks: ' + str(len(pks)))
print('')
print('## processing columns...')
pcolumns = []
if not args.explore:
cp = MPColumnProcessor(connection_string=args.src,
columns=columns,
columnprocessor=NumpyColumnProcessor,
mapper=columnmapper)
pcolumns = cp.execute(processes=int(args.cpu), verbose=True)
else:
pcolumns = columnmapper.multiple([(column, None) for column in columns])
# cets = datetime.datetime.now()
# Notifier.notify(title='cobr.io ds-toolkit',
# subtitle='MPColumnProcessor done!',
# message='processed: ' + str(len(pcolumns)) + ' columns in ' + str(math.floor((cets - sts).total_seconds())) + ' seconds')
print('')
print('## processing tables...')
tp = MPTableProcessor(connection_string=args.src,
tables=tables,
mapper=tablemapper)
ptables = tp.execute(processes=int(args.cpu), verbose=True)
# Notifier.notify(title='cobr.io ds-toolkit',
# subtitle='MPTableProcessor done!',
# message='processed: ' + str(len(ptables)) + ' tables in ' + str(math.floor((datetime.datetime.now() - cets).total_seconds())) + ' seconds')
if not args.dry_run and args.target:
engine = create_engine(args.target)
Session = sessionmaker(bind=engine)
session = Session()
writeToDb(session, ptables)
writeToDb(session, pcolumns)
writeToDb(session, pks)
writeToDb(session, fks)
print('')
print('## time elapsed: ' + str(datetime.datetime.now() - sts))
def main(args):
miner = MetaModel(args.src)
tables = miner.tables()
columns = miner.columns()
pkexplorer = PkExplorer(connection_string=args.src)
pks = pkexplorer.doMultipleTables(tables)
if not args.dry_run and args.target:
engine = create_engine(args.target)
Session = sessionmaker(bind=engine)
session = Session()
writeToDb(session, pks)
def main(args):
colseparator = '|'
miner = MetaModel(args.src)
tables = miner.tables()
columns = miner.columns()
dic = {}
for pk in [ c for c in columns if c.primary_key ]:
if (pk.info['schemaname'], pk.table.name) not in dic:
dic[(pk.info['schemaname'], pk.table.name)] = []
dic[(pk.info['schemaname'], pk.table.name)].append(pk)
# split primary keys into singlecolumn and multicolumn keys
spks = [ dic[key] for key in dic.keys() if len(dic[key]) == 1 ]
mpks = [ dic[key] for key in dic.keys() if len(dic[key]) > 1 ]
print("LOT'S OF WORK IN PROGRESS...")
exit()
affaires = Discovery(tables=tables, columns=columns, pksingle=spks, pkmulti=mkps)
fks = pruneDuplicateFks([ fk[0] for fk in affaires.discoverfks(0.9) ])
conn = engine.connect()
num_rows = conn.execute(table.count()).fetchone()[0]
num_columns = len(table.columns)
num_explicit_outlinks = len(table.foreign_keys)
return self.mapper.single(table, {'num_rows': num_rows, 'num_columns': num_columns, 'num_explicit_outlinks': num_explicit_outlinks})
# return (table.name, {'num_rows': num_rows, 'num_columns': num_columns, 'num_explicit_outlinks': num_explicit_outlinks})
# cp = self.columnprocessor(values)
# return (column, cp.doOperations())
except Exception as ex:
print(ex)
finally:
conn.close()
if __name__ == "__main__":
parser = argparse.ArgumentParser()
parser.add_argument("-s", "--src", help="connection_string for the subject-database", metavar="string")
args = parser.parse_args()
mm = MetaModel(args.src)
sts = datetime.datetime.now()
processor = MPTableProcessor(connection_string = args.src, tables = mm.tables())
result = processor.execute(processes=32, verbose=True)
duration = datetime.datetime.now() - sts
print('number of processed tables: ' + str(len(result)))
# Calling the notification function
# Notifier.notify(title='cobr.io ds-toolkit', subtitle='MPTableProcessor done!', message='processed: ' + str(len(result)) + ' tables in ' + str(math.floor(duration.total_seconds())) + ' seconds')
