def main():
"""The usage is :- python app.py env table_list """
print("The parameters are:", sys.argv)
env = sys.argv[1]
a_tables = sys.argv[2]
logger.add("logs/data-copier.info",
rotation="1 KB",
retention="10 days",
level="INFO")
logger.add("logs/data-copier.err",
rotation="1 KB",
retention="10 days",
level="ERROR")
db_details = load_db_details(env)
print(db_details)
logger.debug(db_details)
tables = get_tables('table_list', a_tables)
print(tables)
logger.debug(tables)
for table_name in tables['table_name']:
logger.info(f'reading data for {table_name}')
data, column_names = read_table(db_details, table_name)
logger.debug(f'Data is: {data}')
logger.debug(f"Column names are:{column_names}")
logger.info(f'loading data for {table_name}')
load_table(db_details, data, column_names, table_name)
def main():
print("Docker Test")
env = sys.argv[1]
db_details = load_db_details(env)
tables = get_tables('tables_list')
for table_name in tables['table_name']:
data, column_names = read_table(db_details, table_name)
load_table(db_details, data, column_names, table_name)
def main():
"""Program takes at least one argument"""
env = sys.argv[1]
db_details = load_db_details(env)
tables = get_tables('table_list')
for table_name in tables['table_name']:
print(f'reading data for {table_name}')
data, column_names = read_table(db_details, table_name)
print(f'loading data for {table_name}')
load_table(db_details, data, column_names, table_name)
def main():
env = sys.argv[1]
a_database = sys.argv[2]
a_table = sys.argv[3]
init_logger()
db_details = load_db_details(env)[a_database]
logger.info(f'reading data for {a_table}')
data, column_names = read_table(db_details, a_table)
df = pd.DataFrame(data, columns=column_names)
write_df_to_file('/tmp', table_name=a_table, df=df)
def main():
env = sys.argv[1]
a_tables = sys.argv[2]
db_details = load_db_details(env)
tables = get_tables('table_list', a_tables)
for table in tables['table_name']:
data, column_names = read_table(db_details, table)
#test = build_insert_query(table, column_names)
load_table(db_details, data, column_names, table)
print(f'Successfully completed process for {table}')
def main():
env = sys.argv[1]
db_details = get_credentials(env)
tables = get_tables("tables_to_be_loaded.txt")
for table in tables:
print("Discovering data...")
data, cols = read_table(db_details, table)
print(f"Table: {table} | {len(data)} rows")
print("Writing data...")
write_table(db_details, table, data, cols)
print("Process completed")
def main():
env = sys.argv[1]
db_details = load_db_details(env)
# print(db_details)
tables = get_tables('tables_list.txt')
# for idx,table in tables.iterrows():
# print (table)
for table_name in tables['table_name']:
print(f'reading data for {table_name}')
data, column_names = read_table(db_details, table_name)
print(f'loading data for {table_name}')
def main():
env = sys.argv[1]
detail = load_db_details(env)
# print(detail)
table = get_table('tableload')
for t in table['table_name']:
print(t)
data, column = read_table(detail, t, 0)
field_names = [i[0] for i in column]
for i in data:
print(i)
print(field_names)
def main():
env = sys.argv[1]
db_details = load_db_details(env)
print(db_details)
tables = get_tables('table_list')
db_details = DB_DETAILS[env]
for table in tables['table_name']:
print(f'Reading data from {table}')
data, columns = read_table(db_details, table)
print(data)
print(f'Loading data into {table}')
load_table(db_details, table, columns, data)
def main():
if len(sys.argv) < 2:
print("Usage: python <filename> <env>")
env = sys.argv[1]
db_details = DB_DETAILS[env]
tables = get_tables('table_list.txt')
for table in tables['table_name']:
print(f'Reading data for table {table}')
data, column_names = read_table(db_details, table)
print(data)
print(column_names)
print(f'Loading data for table {table}')
def main():
"""Program takes at least one argument"""
env = sys.argv[1]
db_details = load_db_details(env)
tables = get_tables('table_list')
print("These are the tables to be loaded", tables)
for table_name in tables['table_name']:
print('Reading data for {}'.format(table_name))
data, column_names = read_table(db_details, table_name)
# print(column_names)
print('Loading data for {}'.format(table_name))
load_table(db_details, data, column_names, table_name)
def main():
"""Program takes atleast one argument"""
env = sys.argv[1]
db_details = DB_DETAILS[env]
tables = get_tables('table_list')
for table_name in tables['table_name']:
print(table_name)
data,column_names = read_table(db_details,table_name,10)
query = build_insert_query('film_actor_target', column_names)
insert_data(db_details, query, data, batchsize=100)
def main():
"""Program takes at least one arguments"""
db_details = load_db_details(env)
tables = get_tables(path='src/main/python/table_list.csv',
table_list='all')
for table in tables['table_name']:
table_n = table
# reading the data from mysql
data, columns = read_table(db_details,
DB_USER=DB_USER_MYSQL,
DB_PASS=DB_PASS_MYSQL,
table_name=table_n)
# loading data to postgres
load_table(db_details=db_details,
DB_USER=DB_USER_POSTGRES,
DB_PASS=DB_PASS_POSTGRES,
data=data,
column_names=columns,
table_name=table_n)
