def main():
global logger, globopts
parser = argparse.ArgumentParser(
description="""Fetch weights information from Gstat provider
for every job listed in customer.conf"""
)
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI)
j = 0
for feed, jobcust in feeds.items():
weights = Vapor(feed)
datawr = None
customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust))
customers = customers.pop() if len(customers) == 1 else '({0})'.format(
','.join(customers))
sjobs = set(map(lambda jc: jc[0], jobcust))
jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(
','.join(sjobs))
logger.job = jobs
logger.customer = customers
for job, cust in jobcust:
logger.customer = confcust.get_custname(cust)
logger.job = job
write_empty = confcust.send_empty(sys.argv[0], cust)
if not write_empty:
w = weights.getWeights()
else:
w = []
weights.state = True
jobdir = confcust.get_fulldir(cust, job)
jobstatedir = confcust.get_fullstatedir(
globopts['InputStateSaveDir'.lower()], cust, job)
custname = confcust.get_custname(cust)
ams_custopts = confcust.get_amsopts(cust)
ams_opts = cglob.merge_opts(ams_custopts, 'ams')
ams_complete, missopt = cglob.is_complete(ams_opts, 'ams')
if not ams_complete:
logger.error('Customer:%s %s options incomplete, missing %s' %
(custname, 'ams', ' '.join(missopt)))
continue
if fixed_date:
output.write_state(sys.argv[0], jobstatedir, weights.state,
globopts['InputStateDays'.lower()],
fixed_date.replace('-', '_'))
else:
output.write_state(sys.argv[0], jobstatedir, weights.state,
globopts['InputStateDays'.lower()])
if not weights.state:
continue
datawr = data_out(w)
if eval(globopts['GeneralPublishAms'.lower()]):
if fixed_date:
partdate = fixed_date
else:
partdate = datestamp(1).replace('_', '-')
ams = output.AmsPublish(
ams_opts['amshost'], ams_opts['amsproject'],
ams_opts['amstoken'], ams_opts['amstopic'],
confcust.get_jobdir(job), ams_opts['amsbulk'],
ams_opts['amspacksinglemsg'], logger,
int(globopts['ConnectionRetry'.lower()]),
int(globopts['ConnectionTimeout'.lower()]))
ams.send(globopts['AvroSchemasWeights'.lower()], 'weights',
partdate, datawr)
if eval(globopts['GeneralWriteAvro'.lower()]):
if fixed_date:
filename = filename_date(logger,
globopts['OutputWeights'.lower()],
jobdir,
fixed_date.replace('-', '_'))
else:
filename = filename_date(logger,
globopts['OutputWeights'.lower()],
jobdir)
avro = output.AvroWriter(
globopts['AvroSchemasWeights'.lower()], filename)
ret, excep = avro.write(datawr)
if not ret:
logger.error('Customer:%s Job:%s %s' %
(logger.customer, logger.job, repr(excep)))
raise SystemExit(1)
j += 1
if datawr or write_empty:
custs = set([cust for job, cust in jobcust])
for cust in custs:
jobs = [job for job, lcust in jobcust if cust == lcust]
logger.info(
'Customer:%s Jobs:%s Sites:%d' %
(confcust.get_custname(cust), jobs[0] if len(jobs) == 1
else '({0})'.format(','.join(jobs)), len(datawr)))
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description='Fetch downtimes from GOCDB for given date')
parser.add_argument('-d',
dest='date',
nargs=1,
metavar='YEAR-MONTH-DAY',
required=True)
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
feed = confcust.get_topofeed()
logger.customer = confcust.get_custname()
if len(args.date) == 0:
print(parser.print_help())
raise SystemExit(1)
# calculate start and end times
try:
start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
timestamp = start.strftime('%Y_%m_%d')
start = start.replace(hour=0, minute=0, second=0)
end = end.replace(hour=23, minute=59, second=59)
except ValueError as exc:
logger.error(exc)
raise SystemExit(1)
uidservtype = confcust.get_uidserviceendpoints()
auth_custopts = confcust.get_authopts()
auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
if not auth_complete:
missing_err = ''.join(missing)
logger.error(
'Customer:{} authentication options incomplete, missing {}'.format(
logger.customer, missing_err))
raise SystemExit(1)
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
try:
# we don't have multiple tenant definitions in one
# customer file so we can safely assume one tenant/customer
write_empty = confcust.send_empty(sys.argv[0])
if not write_empty:
res = loop.run_until_complete(
fetch_data(feed, auth_opts, start, end))
dts = parse_source(res, start, end, uidservtype)
else:
dts = []
loop.run_until_complete(write_state(confcust, timestamp, True))
webapi_opts = get_webapi_opts(cglob, confcust)
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(send_webapi(webapi_opts, args.date[0],
dts))
if dts or write_empty:
cust = list(confcust.get_customers())[0]
logger.info('Customer:%s Fetched Date:%s Endpoints:%d' %
(confcust.get_custname(cust), args.date[0], len(dts)))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(confcust, dts, timestamp)
except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
logger.error(repr(exc))
loop.run_until_complete(write_state(confcust, timestamp, False))
loop.close()
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description="""Fetch weights information from Gstat provider
for every job listed in customer.conf"""
)
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI)
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
for feed, jobcust in feeds.items():
customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust))
customers = customers.pop() if len(customers) == 1 else '({0})'.format(
','.join(customers))
sjobs = set(map(lambda jc: jc[0], jobcust))
jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(
','.join(sjobs))
logger.job = jobs
logger.customer = customers
try:
res = loop.run_until_complete(fetch_data(feed))
weights = parse_source(res)
for job, cust in jobcust:
logger.customer = confcust.get_custname(cust)
logger.job = job
write_empty = confcust.send_empty(sys.argv[0], cust)
if write_empty:
weights = []
webapi_opts = get_webapi_opts(cust, job, cglob, confcust)
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(
send_webapi(job, confcust, webapi_opts, fixed_date,
weights))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(cust, job, confcust, fixed_date, weights)
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, True))
if weights or write_empty:
custs = set([cust for job, cust in jobcust])
for cust in custs:
jobs = [job for job, lcust in jobcust if cust == lcust]
logger.info(
'Customer:%s Jobs:%s Sites:%d' %
(confcust.get_custname(cust), jobs[0] if len(jobs)
== 1 else '({0})'.format(','.join(jobs)),
len(weights)))
except (ConnectorHttpError, ConnectorParseError,
KeyboardInterrupt) as exc:
logger.error(repr(exc))
for job, cust in jobcust:
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, False))
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description='Fetch downtimes from GOCDB for given date')
parser.add_argument('-d',
dest='date',
nargs=1,
metavar='YEAR-MONTH-DAY',
required=True)
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
feeds = confcust.get_mapfeedjobs(sys.argv[0],
deffeed='https://goc.egi.eu/gocdbpi/')
if len(args.date) == 0:
print(parser.print_help())
raise SystemExit(1)
# calculate start and end times
try:
start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
timestamp = start.strftime('%Y_%m_%d')
start = start.replace(hour=0, minute=0, second=0)
end = end.replace(hour=23, minute=59, second=59)
except ValueError as e:
logger.error(e)
raise SystemExit(1)
j = 0
for feed, jobcust in feeds.items():
customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust))
customers = customers.pop() if len(customers) == 1 else '({0})'.format(
','.join(customers))
sjobs = set(map(lambda jc: jc[0], jobcust))
jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(
','.join(sjobs))
logger.job = jobs
logger.customer = customers
uidjob = confcust.pass_uidserviceendpoints(sjobs)
auth_custopts = confcust.get_authopts(feed, jobcust)
auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
if not auth_complete:
logger.error(
'Customer:%s Jobs:%s %s options incomplete, missing %s' %
(logger.customer, logger.job, 'authentication',
''.join(missing)))
continue
# we don't have multiple tenant definitions in one
# customer file so we can safely assume one tenant/customer
cust = jobcust[0][1]
write_empty = confcust.send_empty(sys.argv[0], cust)
# do fetch only once
if all_same(uidjob):
gocdb = GOCDBReader(feed, auth_opts, uidjob[j])
if not write_empty:
dts = gocdb.getDowntimes(start, end)
else:
dts = []
gocdb.state = True
for job, cust in jobcust:
# fetch for every job because of different TopoUIDServiceEndpoints
# setting for each job
if not all_same(uidjob):
gocdb = GOCDBReader(feed, auth_opts, uidjob[j])
if not write_empty:
dts = gocdb.getDowntimes(start, end)
else:
dts = []
gocdb.state = True
jobdir = confcust.get_fulldir(cust, job)
jobstatedir = confcust.get_fullstatedir(
globopts['InputStateSaveDir'.lower()], cust, job)
logger.customer = confcust.get_custname(cust)
logger.job = job
ams_custopts = confcust.get_amsopts(cust)
ams_opts = cglob.merge_opts(ams_custopts, 'ams')
ams_complete, missopt = cglob.is_complete(ams_opts, 'ams')
if not ams_complete:
logger.error(
'Customer:%s Job:%s %s options incomplete, missing %s' %
(logger.customer, job, 'ams', ' '.join(missopt)))
continue
output.write_state(sys.argv[0], jobstatedir, gocdb.state,
globopts['InputStateDays'.lower()], timestamp)
if not gocdb.state:
continue
if eval(globopts['GeneralPublishAms'.lower()]):
ams = output.AmsPublish(
ams_opts['amshost'], ams_opts['amsproject'],
ams_opts['amstoken'], ams_opts['amstopic'],
confcust.get_jobdir(job), ams_opts['amsbulk'],
ams_opts['amspacksinglemsg'], logger,
int(globopts['ConnectionRetry'.lower()]),
int(globopts['ConnectionTimeout'.lower()]))
ams.send(globopts['AvroSchemasDowntimes'.lower()], 'downtimes',
timestamp.replace('_', '-'), dts)
if eval(globopts['GeneralWriteAvro'.lower()]):
filename = filename_date(logger,
globopts['OutputDowntimes'.lower()],
jobdir,
stamp=timestamp)
avro = output.AvroWriter(
globopts['AvroSchemasDowntimes'.lower()], filename)
ret, excep = avro.write(dts)
if not ret:
logger.error('Customer:%s Job:%s %s' %
(logger.customer, logger.job, repr(excep)))
raise SystemExit(1)
j += 1
if gocdb.state:
custs = set([cust for job, cust in jobcust])
for cust in custs:
jobs = [job for job, lcust in jobcust if cust == lcust]
logger.info(
'Customer:%s Jobs:%s Fetched Date:%s Endpoints:%d' %
(confcust.get_custname(cust), jobs[0] if len(jobs)
== 1 else '({0})'.format(','.join(jobs)), args.date[0],
len(dts)))