def read_file(f): try: root_dir=file_system_tasks.get_project_dir(-3) path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir']+'/' except Exception as e: print(e) with open(path+f, 'r') as file: lines = file.readlines() return lines
def get_local_files(): paths = file_system_tasks.get_parameters( 'parameters.json')['param']['s3_upload_dirs'].split(',') project_dir = file_system_tasks.get_project_dir(-3)[:-1] paths = [project_dir + path for path in paths] file_list = [] for path in paths: files = [(path + "/" + f) for f in listdir(path) if isfile(join(path, f))] file_list += files return file_list
def get_bucket(): credentials = file_system_tasks.get_parameters('s3_credentials.txt') key_id = credentials['Access key ID'] secret_key = credentials['Secret access key'] s3 = boto3.resource( 's3', aws_access_key_id=key_id, aws_secret_access_key=secret_key, ) global pcr_storage pcr_storage = s3.Bucket('pcr-storage')
def get_sorted_file_names(): try: root_dir=file_system_tasks.get_project_dir(-3) path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir'] except Exception as e: print(e) try: file_names = [f for f in listdir(path) if isfile(join(path, f))] file_names.sort(reverse=False) except Exception as e: print(e) return -1 return file_names
def connect_cloud(): conn_cloud=-1 try: param_dict=file_system_tasks.get_parameters('RDS_credentials.txt') endpoint=param_dict['endpoint'] port=int(param_dict['port']) database_name=param_dict['database_name'] username=param_dict['username'] password=param_dict['password'] except Exception as e: print(e) return -1 try: conn_cloud = pymysql.connect(host=endpoint, user=username,port=port,passwd=password, db=database_name) return conn_cloud if(not (type(conn_cloud) == pymysql.connections.Connection)): raise Exception("counld not obtain proper connection to RDS...") except Exception as e: print(e) return -1 return conn_cloud
#!/usr/bin/env python3 # -*- coding: utf-8 -*- """ Created on Wed Nov 6 10:52:22 2019 @author: sleek_eagle """ import file_system_tasks from os import listdir from os.path import isfile, join import dep_data dep_id=dep_data.get_dep_id(file_system_tasks.get_project_dir(-3)) col_names=file_system_tasks.get_parameters('parameters.json')['param']['m2g_fields'] #read the log file def read_file(f): try: root_dir=file_system_tasks.get_project_dir(-3) path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir']+'/' except Exception as e: print(e) with open(path+f, 'r') as file: lines = file.readlines() return lines def get_sorted_file_names(): try: root_dir=file_system_tasks.get_project_dir(-3)