def get_log_path():
return '{}/{}/Log Files/Log_{}_{}.txt'.format(
get_year_from_sys_args(),
get_month_from_sys_args(),
get_src_filename(),
time.strftime("%H:%M:%S")
)
def download_tab2():
tab2_filename = get_tab2_filename()
s3_tab2_key = '{}/{}/Successful Files/{}'.format(get_year_from_sys_args(),
get_month_from_sys_args(),
tab2_filename)
s3_client = get_s3_client()
s3_client.download_file(get_s3_bucket(), s3_tab2_key, s3_tab2_key)
def insert_csv_data_tab3():
connection = get_redshift_connection()
q3="COPY us_gtmsales.stg_sellercampaigns_v1(selleruid, sellercompanyname, gtmcampaignsource, campaignname, " \
"crmsystemcampaignid, campaigncreatedate, investment, boxmonth, boxyear, insertiondate) " \
"FROM 's3://{}/{}/{}/Successful Files/{}' " \
"CREDENTIALS 'aws_access_key_id={};aws_secret_access_key={}' " \
"CSV timeformat 'auto' dateformat 'auto';".format(get_s3_bucket(),
get_year_from_sys_args(),
get_month_from_sys_args(),
get_tab3_filename(),
get_s3_config()['access_key_id'],
get_s3_config()['secret_access_key'])
connection.execution_options(autocommit=True).execute(q3)
connection.close()
def insert_csv_data_tab2():
connection = get_redshift_connection()
q3 = "COPY us_gtmsales.stg_selleropportunities_v1(" \
"selleruid, sellercompanyname, gtmcampaignsource, campaignname, crmsystemcampaignid, " \
"campaigncreatedate, opportunityid, convertdate, opportunitycountry, " \
"opportunitystatus, awsmarketopportunity, pipelinerevenue, accountname, accountid, windate, billedrevenue, " \
"boxmonth, boxyear, insertiondate )" \
"FROM 's3://{}/{}/{}/Successful Files/{}' " \
"CREDENTIALS 'aws_access_key_id={};aws_secret_access_key={}' " \
"CSV timeformat 'auto' dateformat 'auto';".format(get_s3_bucket(),
get_year_from_sys_args(),
get_month_from_sys_args(),
get_tab2_filename(),
get_s3_config()['access_key_id'],
get_s3_config()['secret_access_key'])
connection.execution_options(autocommit=True).execute(q3)
connection.close()
def get_src_file_dir():
return '{}/{}/Raw Files/'.format(get_year_from_sys_args(), get_month_from_sys_args())
def get_s3_log_prefix():
return '{}/{}/Log Files/{}'.format(get_year_from_sys_args(),
get_month_from_sys_args(),
get_log_file_prefix())