def main():
try:
body = 'init'
line = '\n'
count = 0
# get result from rds
with aws.rdsconn(config) as rdsconn:
colnames = " maker_id,data_type, to_char(data_date,'YYYY-MM-DD'), status, msg "
sql = "select {0} from data_ctrl_mst where status != 'success' order by 1,2,3,4;".format(colnames)
res = aws.fetch_sql(rdsconn, sql)
count = len(res)
rescount = '未完了の件数 : {0}\n'.format(count)
log.print_log(logger,rescount)
if count > 10:
colnames = " status, count(*) "
sql = "select {0} from data_ctrl_mst group by status order by 1;".format(colnames)
res = aws.fetch_sql(rdsconn, sql)
body = rescount + colnames + line + line.join(map(str, res))
except Exception as e:
log.except_log(logger, e)
body = str(type(e)) + line + e.message
finally:
if count == 0:
sys.exit(0)
ml = mail.kaden_mail()
ml.send_message(body, '[kaden] データ更新状況', config.get('mail', 'mailfrom'), config.get('mail', 'mailto'))
def main(param):
try:
c = ctrl.data_ctrl_mst(param.maker_id, param.data_type, param.data_date.replace("-", ""), param.status)
except Exception as e:
log.except_log(logger, e)
c.status = ERROR_STATUS.format(c.status)
finally:
# log to RDS
with aws.rdsconn(config) as rdsconn:
aws.commit_ctrl_update(rdsconn, c)
def process_data(s3bucket, rdsconn, row):
try:
# get file info
datatype = row[1]
datadate = str(row[2]).replace('-', '')
filename = row[4].split('/')[-1]
log.print_log(logger, 'file:{0} type:{1} date:{2}'.format(filename, datatype, datadate))
# download file to local
aws.get_contents(s3bucket, filename, TMP_DIR)
except Exception as e:
log.except_log(logger, e)
def main(param):
try:
# get result from rds
with aws.rdsconn(config) as rdsconn:
sql = "select to_char(data_date, 'YYYY-MM-DD'), msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and status= '%s' order by 1,2;" % (param.maker_id, param.data_type, param.status)
res = aws.fetch_sql(rdsconn, sql)
for i in res:
datadate = i[0]
filename = i[1].split('/')[-1]
log.print_log(logger, '{0}, {1}'.format(datadate, filename))
except Exception as e:
log.except_log(logger, e)
def main(param):
try:
# s3 url access
s3conn = aws.s3conn()
s3bucket = aws.get_bucket(s3conn, config.get('s3', 'p_refined_data_bucket'))
rdsconn = aws.rdsconn(config)
getsql = "select maker_id, data_type, data_date, status, msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and data_date = '%s' and status = 'refine' and msg is not null; " % (param.maker_id, param.data_type, param.data_date)
res = aws.fetch_sql(rdsconn, getsql)
if len(res) == 1:
process_data(s3bucket, rdsconn, res[0])
else:
raise ValueError('抽出ファイルエラー sq:{0}'.format(getsql))
except Exception as e:
log.except_log(logger, e)
