예제 #1
0
 def setup_class(clazz):
   setstatus.logger = log.init_logger(False, os.path.basename(__file__))
   setstatus.config = ConfigParser.SafeConfigParser()
   confpath = os.path.join(os.path.dirname(os.path.dirname(os.path.realpath(__file__))),'conf','app.config.pro')
   setstatus.config.read(confpath)
예제 #2
0
    body = 'init'
    line = '\n'
    count = 0
    # get result from rds
    with aws.rdsconn(config) as rdsconn:
      colnames = " maker_id,data_type, to_char(data_date,'YYYY-MM-DD'), status, msg "
      sql = "select {0} from data_ctrl_mst where status != 'success' order by 1,2,3,4;".format(colnames)
      res = aws.fetch_sql(rdsconn, sql)
      count = len(res)
      rescount = '未完了の件数 : {0}\n'.format(count)
      log.print_log(logger,rescount)
      if count > 10:
        colnames = " status, count(*) "
        sql = "select {0} from data_ctrl_mst group by status order by 1;".format(colnames)
        res = aws.fetch_sql(rdsconn, sql)
      body = rescount + colnames + line + line.join(map(str, res))
  except Exception as e:
    log.except_log(logger, e)
    body = str(type(e)) + line + e.message
  finally:
    if count == 0:
      sys.exit(0)
    ml = mail.kaden_mail()
    ml.send_message(body, '[kaden] データ更新状況', config.get('mail', 'mailfrom'), config.get('mail', 'mailto'))

if __name__ == '__main__':
  logger = log.init_logger(False, os.path.basename(__file__))
  confpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conf', 'app.config')
  config = util.get_config(confpath)
  main()
예제 #3
0
    s3conn = aws.s3conn()
    s3bucket = aws.get_bucket(s3conn, config.get('s3', 'p_refined_data_bucket'))
    rdsconn = aws.rdsconn(config)
    getsql = "select maker_id, data_type, data_date, status, msg from data_ctrl_mst where maker_id = %s and data_type = '%s' and data_date = '%s' and status = 'refine' and msg is not null; " % (param.maker_id, param.data_type, param.data_date)
    res = aws.fetch_sql(rdsconn, getsql)
    if len(res) == 1:
      process_data(s3bucket, rdsconn, res[0])
    else:
      raise ValueError('抽出ファイルエラー sq:{0}'.format(getsql))
  except Exception as e:
    log.except_log(logger, e)

def process_data(s3bucket, rdsconn, row):
  try:
    # get file info
    datatype = row[1]
    datadate = str(row[2]).replace('-', '')
    filename = row[4].split('/')[-1]
    log.print_log(logger, 'file:{0} type:{1} date:{2}'.format(filename, datatype, datadate))
    # download file to local
    aws.get_contents(s3bucket, filename, TMP_DIR)
  except Exception as e:
    log.except_log(logger, e)

if __name__ == '__main__':
  logger = log.init_logger(True, os.path.basename(__file__))
  confpath = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'conf', 'app.config')
  config = util.get_config(confpath)
  param = params.getrefine_params(sys.argv)
  main(param)