def main():
global logger, globopts
parser = argparse.ArgumentParser(description='Fetch metric profile for every job of the customer')
parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False)
parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
for cust in confcust.get_customers():
custname = confcust.get_custname(cust)
for job in confcust.get_jobs(cust):
logger.customer = confcust.get_custname(cust)
logger.job = job
profiles = confcust.get_profiles(job)
webapi_custopts = confcust.get_webapiopts(cust)
webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi')
webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi')
if not webapi_complete:
logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'webapi', ' '.join(missopt)))
continue
webapi = WebAPI(custname, job, profiles, confcust.get_namespace(job),
webapi_opts['webapihost'],
webapi_opts['webapitoken'])
fetched_profiles = webapi.get_profiles()
jobdir = confcust.get_fulldir(cust, job)
jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job)
ams_custopts = confcust.get_amsopts(cust)
ams_opts = cglob.merge_opts(ams_custopts, 'ams')
ams_complete, missopt = cglob.is_complete(ams_opts, 'ams')
if not ams_complete:
logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt)))
continue
if fixed_date:
output.write_state(sys.argv[0], jobstatedir,
webapi.state,
globopts['InputStateDays'.lower()],
fixed_date.replace('-', '_'))
else:
output.write_state(sys.argv[0], jobstatedir,
webapi.state,
globopts['InputStateDays'.lower()])
if not webapi.state:
continue
if eval(globopts['GeneralPublishAms'.lower()]):
if fixed_date:
partdate = fixed_date
else:
partdate = datestamp(1).replace('_', '-')
ams = output.AmsPublish(ams_opts['amshost'],
ams_opts['amsproject'],
ams_opts['amstoken'],
ams_opts['amstopic'],
confcust.get_jobdir(job),
ams_opts['amsbulk'],
ams_opts['amspacksinglemsg'],
logger,
int(globopts['ConnectionRetry'.lower()]),
int(globopts['ConnectionTimeout'.lower()]))
ams.send(globopts['AvroSchemasMetricProfile'.lower()], 'metric_profile',
partdate, fetched_profiles)
if eval(globopts['GeneralWriteAvro'.lower()]):
if fixed_date:
filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir, fixed_date.replace('-', '_'))
else:
filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir)
avro = output.AvroWriter(globopts['AvroSchemasMetricProfile'.lower()], filename)
ret, excep = avro.write(fetched_profiles)
if not ret:
logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep)))
raise SystemExit(1)
logger.info('Customer:' + custname + ' Job:' + job + ' Profiles:%s Tuples:%d' % (', '.join(profiles), len(fetched_profiles)))
def main():
global logger, globopts
parser = argparse.ArgumentParser(description='Fetch POEM profile for every job of the customer and write POEM expanded profiles needed for prefilter for EGI customer')
parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False)
parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
for cust in confcust.get_customers():
custname = confcust.get_custname(cust)
for job in confcust.get_jobs(cust):
logger.customer = confcust.get_custname(cust)
logger.job = job
poemserver = dict()
profiles = confcust.get_profiles(job)
namespace = confcust.get_namespace(job)
poemserver[confcust.get_poemserver_host(job)] = confcust.get_poemserver_vo(job)
poem = PoemReader(custname, job)
psa = poem.getProfiles(profiles, namespace, poemserver)
jobdir = confcust.get_fulldir(cust, job)
jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job)
ams_custopts = confcust.get_amsopts(cust)
ams_opts = cglob.merge_opts(ams_custopts, 'ams')
ams_complete, missopt = cglob.is_complete(ams_opts, 'ams')
if not ams_complete:
logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt)))
continue
if fixed_date:
output.write_state(sys.argv[0], jobstatedir,
poem.state,
globopts['InputStateDays'.lower()],
fixed_date.replace('-', '_'))
else:
output.write_state(sys.argv[0], jobstatedir,
poem.state,
globopts['InputStateDays'.lower()])
if not poem.state:
continue
lfprofiles = gen_outprofiles(psa, profiles)
if eval(globopts['GeneralPublishAms'.lower()]):
if fixed_date:
partdate = fixed_date
else:
partdate = datestamp(1).replace('_', '-')
ams = output.AmsPublish(ams_opts['amshost'],
ams_opts['amsproject'],
ams_opts['amstoken'],
ams_opts['amstopic'],
confcust.get_jobdir(job),
ams_opts['amsbulk'],
ams_opts['amspacksinglemsg'],
logger,
int(globopts['ConnectionRetry'.lower()]),
int(globopts['ConnectionTimeout'.lower()]))
ams.send(globopts['AvroSchemasPoem'.lower()], 'metric_profile',
partdate, lfprofiles)
if eval(globopts['GeneralWriteAvro'.lower()]):
if fixed_date:
filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir, fixed_date.replace('-', '_'))
else:
filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir)
avro = output.AvroWriter(globopts['AvroSchemasPoem'.lower()], filename)
ret, excep = avro.write(lfprofiles)
if not ret:
logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep)))
raise SystemExit(1)
logger.info('Customer:'+custname+' Job:'+job+' Profiles:%s Tuples:%d' % (','.join(profiles), len(lfprofiles)))
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description='Fetch metric profile for every job of the customer')
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
for cust in confcust.get_customers():
custname = confcust.get_custname(cust)
for job in confcust.get_jobs(cust):
logger.customer = confcust.get_custname(cust)
logger.job = job
profiles = confcust.get_profiles(job)
webapi_custopts = confcust.get_webapiopts(cust)
webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi')
webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi')
if not webapi_complete:
logger.error(
'Customer:%s Job:%s %s options incomplete, missing %s' %
(custname, logger.job, 'webapi', ' '.join(missopt)))
continue
try:
res = loop.run_until_complete(
fetch_data(webapi_opts['webapihost'],
webapi_opts['webapitoken']))
fetched_profiles = parse_source(res, profiles,
confcust.get_namespace(job))
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, True))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(cust, job, confcust, fixed_date,
fetched_profiles)
logger.info('Customer:' + custname + ' Job:' + job +
' Profiles:%s Tuples:%d' %
(', '.join(profiles), len(fetched_profiles)))
except (ConnectorHttpError, KeyboardInterrupt,
ConnectorParseError) as exc:
logger.error(repr(exc))
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, False))