def read_file(f):
try:
root_dir=file_system_tasks.get_project_dir(-3)
path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir']+'/'
except Exception as e:
print(e)
with open(path+f, 'r') as file:
lines = file.readlines()
return lines
def get_local_files():
paths = file_system_tasks.get_parameters(
'parameters.json')['param']['s3_upload_dirs'].split(',')
project_dir = file_system_tasks.get_project_dir(-3)[:-1]
paths = [project_dir + path for path in paths]
file_list = []
for path in paths:
files = [(path + "/" + f) for f in listdir(path)
if isfile(join(path, f))]
file_list += files
return file_list
def get_bucket():
credentials = file_system_tasks.get_parameters('s3_credentials.txt')
key_id = credentials['Access key ID']
secret_key = credentials['Secret access key']
s3 = boto3.resource(
's3',
aws_access_key_id=key_id,
aws_secret_access_key=secret_key,
)
global pcr_storage
pcr_storage = s3.Bucket('pcr-storage')
def get_sorted_file_names():
try:
root_dir=file_system_tasks.get_project_dir(-3)
path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir']
except Exception as e:
print(e)
try:
file_names = [f for f in listdir(path) if isfile(join(path, f))]
file_names.sort(reverse=False)
except Exception as e:
print(e)
return -1
return file_names
def connect_cloud():
conn_cloud=-1
try:
param_dict=file_system_tasks.get_parameters('RDS_credentials.txt')
endpoint=param_dict['endpoint']
port=int(param_dict['port'])
database_name=param_dict['database_name']
username=param_dict['username']
password=param_dict['password']
except Exception as e:
print(e)
return -1
try:
conn_cloud = pymysql.connect(host=endpoint, user=username,port=port,passwd=password, db=database_name)
return conn_cloud
if(not (type(conn_cloud) == pymysql.connections.Connection)):
raise Exception("counld not obtain proper connection to RDS...")
except Exception as e:
print(e)
return -1
return conn_cloud
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
"""
Created on Wed Nov 6 10:52:22 2019
@author: sleek_eagle
"""
import file_system_tasks
from os import listdir
from os.path import isfile, join
import dep_data
dep_id=dep_data.get_dep_id(file_system_tasks.get_project_dir(-3))
col_names=file_system_tasks.get_parameters('parameters.json')['param']['m2g_fields']
#read the log file
def read_file(f):
try:
root_dir=file_system_tasks.get_project_dir(-3)
path=root_dir[0:-1]+file_system_tasks.get_parameters('parameters.json')['param']['m2g_log_dir']+'/'
except Exception as e:
print(e)
with open(path+f, 'r') as file:
lines = file.readlines()
return lines
def get_sorted_file_names():
try:
root_dir=file_system_tasks.get_project_dir(-3)