def test_getrefine_params_noexit(self): argv = [0,1,'epg','2015-08-15'] actual = params.getrefine_params(argv) assert actual.data_type == 'epg'
def test_getrefine_params_exit2(self): argv = [0,1,2,3] params.getrefine_params(argv)
def test_getrefine_params_exit3(self): argv = [0,1,'epg','20150815'] params.getrefine_params(argv)
s3conn = aws.s3conn()
s3bucket = aws.get_bucket(s3conn, config.get('s3', 'p_refined_data_bucket'))
rdsconn = aws.rdsconn(config)
getsql = "select maker_id, data_type, data_date, status, msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and data_date = '%s' and status = 'refine' and msg is not null; " % (param.maker_id, param.data_type, param.data_date)
res = aws.fetch_sql(rdsconn, getsql)
if len(res) == 1:
process_data(s3bucket, rdsconn, res[0])
else:
raise ValueError('抽出ファイルエラー sq:{0}'.format(getsql))
except Exception as e:
log.except_log(logger, e)
def process_data(s3bucket, rdsconn, row):
try:
# get file info
datatype = row[1]
datadate = str(row[2]).replace('-', '')
filename = row[4].split('/')[-1]
log.print_log(logger, 'file:{0} type:{1} date:{2}'.format(filename, datatype, datadate))
# download file to local
aws.get_contents(s3bucket, filename, TMP_DIR)
except Exception as e:
log.except_log(logger, e)
if __name__ == '__main__':
logger = log.init_logger(True, os.path.basename(__file__))
confpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conf', 'app.config')
config = util.get_config(confpath)
param = params.getrefine_params(sys.argv)
main(param)