def main(): try: body = 'init' line = '\n' count = 0 # get result from rds with aws.rdsconn(config) as rdsconn: colnames = " maker_id,data_type, to_char(data_date,'YYYY-MM-DD'), status, msg " sql = "select {0} from data_ctrl_mst where status != 'success' order by 1,2,3,4;".format(colnames) res = aws.fetch_sql(rdsconn, sql) count = len(res) rescount = '未完了の件数 : {0}\n'.format(count) log.print_log(logger,rescount) if count > 10: colnames = " status, count(*) " sql = "select {0} from data_ctrl_mst group by status order by 1;".format(colnames) res = aws.fetch_sql(rdsconn, sql) body = rescount + colnames + line + line.join(map(str, res)) except Exception as e: log.except_log(logger, e) body = str(type(e)) + line + e.message finally: if count == 0: sys.exit(0) ml = mail.kaden_mail() ml.send_message(body, '[kaden] データ更新状況', config.get('mail', 'mailfrom'), config.get('mail', 'mailto'))
def main(param): try: c = ctrl.data_ctrl_mst(param.maker_id, param.data_type, param.data_date.replace("-", ""), param.status) except Exception as e: log.except_log(logger, e) c.status = ERROR_STATUS.format(c.status) finally: # log to RDS with aws.rdsconn(config) as rdsconn: aws.commit_ctrl_update(rdsconn, c)
def main(param): try: # get result from rds with aws.rdsconn(config) as rdsconn: sql = "select to_char(data_date, 'YYYY-MM-DD'), msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and status= '%s' order by 1,2;" % (param.maker_id, param.data_type, param.status) res = aws.fetch_sql(rdsconn, sql) for i in res: datadate = i[0] filename = i[1].split('/')[-1] log.print_log(logger, '{0}, {1}'.format(datadate, filename)) except Exception as e: log.except_log(logger, e)
def main(param): try: # s3 url access s3conn = aws.s3conn() s3bucket = aws.get_bucket(s3conn, config.get('s3', 'p_refined_data_bucket')) rdsconn = aws.rdsconn(config) getsql = "select maker_id, data_type, data_date, status, msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and data_date = '%s' and status = 'refine' and msg is not null; " % (param.maker_id, param.data_type, param.data_date) res = aws.fetch_sql(rdsconn, getsql) if len(res) == 1: process_data(s3bucket, rdsconn, res[0]) else: raise ValueError('抽出ファイルエラー sq:{0}'.format(getsql)) except Exception as e: log.except_log(logger, e)
def test_commit_sql(self): conn = aws.rdsconn(self.config) actual = aws.commit_sql(conn,'create table foo (bar int);') assert actual.status == psycopg2.extensions.STATUS_READY actual = aws.commit_sql(conn,'drop table foo;') assert actual.status == psycopg2.extensions.STATUS_READY
def test_fetch_sql2(self): conn = aws.rdsconn(self.config) actual = aws.fetch_sql(conn,'select * from data_ctrl_mst;') print(actual) assert len(actual) != 0
def test_fetch_sql(self): conn = aws.rdsconn(self.config) actual = aws.fetch_sql(conn,'select 1;') assert len(actual) == 1
def test_close_rdsconn(self): conn = aws.rdsconn(self.config) actual = aws.close_rdsconn(conn) assert actual.closed != 0
def test_rdsconn(self): actual = aws.rdsconn(self.config) assert actual.status == psycopg2.extensions.STATUS_READY