def main():
global logger, globopts, confcust
parser = argparse.ArgumentParser(description="""Fetch and construct entities from EOSC-PORTAL feed""")
parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False)
parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
args = parser.parse_args()
group_endpoints, group_groups = list(), list()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
global custname
custname = confcust.get_custname()
# safely assume here one customer defined in customer file
cust = list(confcust.get_customers())[0]
jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust)
fetchtype = confcust.get_topofetchtype()[0]
state = None
logger.customer = custname
uidservtype = confcust.get_uidserviceendpoints()
topofeed = confcust.get_topofeed()
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
try:
if is_feed(topofeed):
res = loop.run_until_complete(fetch_data(topofeed))
group_groups, group_endpoints = parse_source_topo(res, uidservtype, fetchtype)
contacts = ParseContacts(logger, res, uidservtype, is_csv=False).get_contacts()
attach_contacts_topodata(logger, contacts, group_endpoints)
else:
try:
with open(topofeed) as fp:
js = json.load(fp)
group_groups, group_endpoints = parse_source_topo(js, uidservtype, fetchtype)
except IOError as exc:
logger.error('Customer:%s : Problem opening %s - %s' % (logger.customer, topofeed, repr(exc)))
loop.run_until_complete(
write_state(confcust, fixed_date, True)
)
webapi_opts = get_webapi_opts(cglob, confcust)
numge = len(group_endpoints)
numgg = len(group_groups)
# send concurrently to WEB-API in coroutines
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(
asyncio.gather(
send_webapi(webapi_opts, group_groups, 'groups', fixed_date),
send_webapi(webapi_opts, group_endpoints,'endpoints', fixed_date)
)
)
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(confcust, group_groups, group_endpoints, fixed_date)
logger.info('Customer:' + custname + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg))
except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
logger.error(repr(exc))
loop.run_until_complete(
write_state(confcust, fixed_date, False )
)
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description="""Fetch weights information from Gstat provider
for every job listed in customer.conf"""
)
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI)
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
for feed, jobcust in feeds.items():
customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust))
customers = customers.pop() if len(customers) == 1 else '({0})'.format(
','.join(customers))
sjobs = set(map(lambda jc: jc[0], jobcust))
jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format(
','.join(sjobs))
logger.job = jobs
logger.customer = customers
try:
res = loop.run_until_complete(fetch_data(feed))
weights = parse_source(res)
for job, cust in jobcust:
logger.customer = confcust.get_custname(cust)
logger.job = job
write_empty = confcust.send_empty(sys.argv[0], cust)
if write_empty:
weights = []
webapi_opts = get_webapi_opts(cust, job, cglob, confcust)
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(
send_webapi(job, confcust, webapi_opts, fixed_date,
weights))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(cust, job, confcust, fixed_date, weights)
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, True))
if weights or write_empty:
custs = set([cust for job, cust in jobcust])
for cust in custs:
jobs = [job for job, lcust in jobcust if cust == lcust]
logger.info(
'Customer:%s Jobs:%s Sites:%d' %
(confcust.get_custname(cust), jobs[0] if len(jobs)
== 1 else '({0})'.format(','.join(jobs)),
len(weights)))
except (ConnectorHttpError, ConnectorParseError,
KeyboardInterrupt) as exc:
logger.error(repr(exc))
for job, cust in jobcust:
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, False))
def main():
global logger, globopts, confcust
parser = argparse.ArgumentParser(
description="""Fetch entities (ServiceGroups, Sites, Endpoints)
from GOCDB for every customer and job listed in customer.conf and write them
in an appropriate place""")
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
group_endpoints, group_groups = [], []
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
pass_extensions = eval(globopts['GeneralPassExtensions'.lower()])
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
topofeed = confcust.get_topofeed()
topofeedpaging = confcust.get_topofeedpaging()
uidservtype = confcust.get_uidserviceendpoints()
topofetchtype = confcust.get_topofetchtype()
custname = confcust.get_custname()
logger.customer = custname
auth_custopts = confcust.get_authopts()
auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
if not auth_complete:
logger.error('%s options incomplete, missing %s' %
('authentication', ' '.join(missing)))
raise SystemExit(1)
bdii_opts = get_bdii_opts(confcust)
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
group_endpoints, group_groups = list(), list()
parsed_site_contacts, parsed_servicegroups_contacts, parsed_serviceendpoint_contacts = None, None, None
try:
contact_coros = [
fetch_data(topofeed + SITE_CONTACTS, auth_opts, False),
fetch_data(topofeed + SERVICEGROUP_CONTACTS, auth_opts, False)
]
contacts = loop.run_until_complete(
asyncio.gather(*contact_coros, return_exceptions=True))
exc_raised, exc = contains_exception(contacts)
if exc_raised:
raise ConnectorHttpError(repr(exc))
parsed_site_contacts = parse_source_sitescontacts(
contacts[0], custname)
parsed_servicegroups_contacts = parse_source_servicegroupsroles(
contacts[1], custname)
except (ConnectorHttpError, ConnectorParseError) as exc:
logger.warn(
'SITE_CONTACTS and SERVICERGOUP_CONTACT methods not implemented')
try:
toposcope = confcust.get_toposcope()
topofeedendpoints = confcust.get_topofeedendpoints()
topofeedservicegroups = confcust.get_topofeedservicegroups()
topofeedsites = confcust.get_topofeedsites()
global SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI
if toposcope:
SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI + toposcope
SERVICE_GROUPS_PI = SERVICE_GROUPS_PI + toposcope
SITES_PI = SITES_PI + toposcope
if topofeedendpoints:
SERVICE_ENDPOINTS_PI = topofeedendpoints
else:
SERVICE_ENDPOINTS_PI = topofeed + SERVICE_ENDPOINTS_PI
if topofeedservicegroups:
SERVICE_GROUPS_PI = topofeedservicegroups
else:
SERVICE_GROUPS_PI = topofeed + SERVICE_GROUPS_PI
if topofeedsites:
SITES_PI = topofeedsites
else:
SITES_PI = topofeed + SITES_PI
fetched_sites, fetched_servicegroups, fetched_endpoints = None, None, None
fetched_bdii = None
coros = [fetch_data(SERVICE_ENDPOINTS_PI, auth_opts, topofeedpaging)]
if 'servicegroups' in topofetchtype:
coros.append(
fetch_data(SERVICE_GROUPS_PI, auth_opts, topofeedpaging))
if 'sites' in topofetchtype:
coros.append(fetch_data(SITES_PI, auth_opts, topofeedpaging))
if bdii_opts and eval(bdii_opts['bdii']):
host = bdii_opts['bdiihost']
port = bdii_opts['bdiiport']
base = bdii_opts['bdiiquerybase']
coros.append(
fetch_ldap_data(
host, port, base, bdii_opts['bdiiqueryfiltersrm'],
bdii_opts['bdiiqueryattributessrm'].split(' ')))
coros.append(
fetch_ldap_data(
host, port, base, bdii_opts['bdiiqueryfiltersepath'],
bdii_opts['bdiiqueryattributessepath'].split(' ')))
# fetch topology data concurrently in coroutines
fetched_topology = loop.run_until_complete(
asyncio.gather(*coros, return_exceptions=True))
fetched_endpoints = fetched_topology[0]
if bdii_opts and eval(bdii_opts['bdii']):
fetched_bdii = list()
fetched_bdii.append(fetched_topology[-2])
fetched_bdii.append(fetched_topology[-1])
if 'sites' in topofetchtype and 'servicegroups' in topofetchtype:
fetched_servicegroups, fetched_sites = (fetched_topology[1],
fetched_topology[2])
elif 'sites' in topofetchtype:
fetched_sites = fetched_topology[1]
elif 'servicegroups' in topofetchtype:
fetched_servicegroups = fetched_topology[1]
exc_raised, exc = contains_exception(fetched_topology)
if exc_raised:
raise ConnectorHttpError(repr(exc))
# proces data in parallel using multiprocessing
executor = ProcessPoolExecutor(max_workers=3)
parse_workers = list()
exe_parse_source_endpoints = partial(parse_source_endpoints,
fetched_endpoints, custname,
uidservtype, pass_extensions)
exe_parse_source_servicegroups = partial(parse_source_servicegroups,
fetched_servicegroups,
custname, uidservtype,
pass_extensions)
exe_parse_source_sites = partial(parse_source_sites, fetched_sites,
custname, uidservtype,
pass_extensions)
# parse topology depend on configured components fetch. we can fetch
# only sites, only servicegroups or both.
if fetched_servicegroups and fetched_sites:
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_endpoints))
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_servicegroups))
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_sites))
elif fetched_servicegroups and not fetched_sites:
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_servicegroups))
elif fetched_sites and not fetched_servicegroups:
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_endpoints))
parse_workers.append(
loop.run_in_executor(executor, exe_parse_source_sites))
parsed_topology = loop.run_until_complete(
asyncio.gather(*parse_workers))
if fetched_servicegroups and fetched_sites:
group_endpoints = parsed_topology[0]
group_groups, group_endpoints_sg = parsed_topology[1]
group_endpoints += group_endpoints_sg
group_groups += parsed_topology[2]
elif fetched_servicegroups and not fetched_sites:
group_groups, group_endpoints = parsed_topology[0]
elif fetched_sites and not fetched_servicegroups:
group_endpoints = parsed_topology[0]
group_groups = parsed_topology[1]
# check if we fetched SRM port info and attach it appropriate endpoint
# data
if bdii_opts and eval(bdii_opts['bdii']):
attach_srmport_topodata(
logger, bdii_opts['bdiiqueryattributessrm'].split(' ')[0],
fetched_bdii[0], group_endpoints)
attach_sepath_topodata(
logger, bdii_opts['bdiiqueryattributessepath'].split(' ')[0],
fetched_bdii[1], group_endpoints)
# parse contacts from fetched service endpoints topology, if there are
# any
parsed_serviceendpoint_contacts = parse_source_serviceendpoints_contacts(
fetched_endpoints, custname)
if not parsed_site_contacts and fetched_sites:
# GOCDB has not SITE_CONTACTS, try to grab contacts from fetched
# sites topology entities
parsed_site_contacts = parse_source_siteswithcontacts(
fetched_sites, custname)
attach_contacts_workers = [
loop.run_in_executor(
executor,
partial(attach_contacts_topodata, logger, parsed_site_contacts,
group_groups)),
loop.run_in_executor(
executor,
partial(attach_contacts_topodata, logger,
parsed_serviceendpoint_contacts, group_endpoints))
]
executor = ProcessPoolExecutor(max_workers=2)
group_groups, group_endpoints = loop.run_until_complete(
asyncio.gather(*attach_contacts_workers))
if parsed_servicegroups_contacts:
attach_contacts_topodata(logger, parsed_servicegroups_contacts,
group_groups)
elif fetched_servicegroups:
# GOCDB has not SERVICEGROUP_CONTACTS, try to grab contacts from fetched
# servicegroups topology entities
parsed_servicegroups_contacts = parse_source_servicegroupscontacts(
fetched_servicegroups, custname)
attach_contacts_topodata(logger, parsed_servicegroups_contacts,
group_groups)
loop.run_until_complete(write_state(confcust, fixed_date, True))
webapi_opts = get_webapi_opts(cglob, confcust)
numge = len(group_endpoints)
numgg = len(group_groups)
# send concurrently to WEB-API in coroutines
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(
asyncio.gather(
send_webapi(webapi_opts, group_groups, 'groups',
fixed_date),
send_webapi(webapi_opts, group_endpoints, 'endpoints',
fixed_date)))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(confcust, group_groups, group_endpoints, fixed_date)
logger.info('Customer:' + custname + ' Type:%s ' %
(','.join(topofetchtype)) + 'Fetched Endpoints:%d' %
(numge) + ' Groups:%d' % (numgg))
except (ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc:
logger.error(repr(exc))
loop.run_until_complete(write_state(confcust, fixed_date, False))
finally:
loop.close()
def main():
global logger, globopts, confcust
parser = argparse.ArgumentParser(
description="""Fetch entities (ServiceGroups, Sites, Endpoints)
from CSV topology feed for every customer and job listed in customer.conf and write them
in an appropriate place""")
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
group_endpoints, group_groups = [], []
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
topofeed = confcust.get_topofeed()
uidservtype = confcust.get_uidserviceendpoints()
topofetchtype = confcust.get_topofetchtype()
custname = confcust.get_custname()
logger.customer = custname
auth_custopts = confcust.get_authopts()
auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
if not auth_complete:
logger.error('%s options incomplete, missing %s' %
('authentication', ' '.join(missing)))
raise SystemExit(1)
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
try:
group_endpoints, group_groups = list(), list()
# fetch topology data concurrently in coroutines
fetched_topology = loop.run_until_complete(
fetch_data(topofeed, auth_opts))
group_groups, group_endpoints = parse_source_topo(
fetched_topology, custname, uidservtype)
contacts = ParseContacts(logger,
fetched_topology,
uidservtype,
is_csv=True).get_contacts()
attach_contacts_topodata(logger, contacts, group_endpoints)
loop.run_until_complete(write_state(confcust, fixed_date, True))
webapi_opts = get_webapi_opts(cglob, confcust)
numgg = len(group_groups)
numge = len(group_endpoints)
# send concurrently to WEB-API in coroutines
if eval(globopts['GeneralPublishWebAPI'.lower()]):
loop.run_until_complete(
asyncio.gather(
send_webapi(webapi_opts, group_groups, 'groups',
fixed_date),
send_webapi(webapi_opts, group_endpoints, 'endpoints',
fixed_date)))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(confcust, group_groups, group_endpoints, fixed_date)
logger.info('Customer:' + custname + ' Type:%s ' %
(','.join(topofetchtype)) + 'Fetched Endpoints:%d' %
(numge) + ' Groups:%d' % (numgg))
except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
logger.error(repr(exc))
loop.run_until_complete(write_state(confcust, fixed_date, False))
finally:
loop.close()
def main():
global logger, globopts
parser = argparse.ArgumentParser(
description='Fetch metric profile for every job of the customer')
parser.add_argument('-c',
dest='custconf',
nargs=1,
metavar='customer.conf',
help='path to customer configuration file',
type=str,
required=False)
parser.add_argument('-g',
dest='gloconf',
nargs=1,
metavar='global.conf',
help='path to global configuration file',
type=str,
required=False)
parser.add_argument('-d',
dest='date',
metavar='YEAR-MONTH-DAY',
help='write data for this date',
type=str,
required=False)
args = parser.parse_args()
logger = Logger(os.path.basename(sys.argv[0]))
fixed_date = None
if args.date and date_check(args.date):
fixed_date = args.date
confpath = args.gloconf[0] if args.gloconf else None
cglob = Global(sys.argv[0], confpath)
globopts = cglob.parse()
confpath = args.custconf[0] if args.custconf else None
confcust = CustomerConf(sys.argv[0], confpath)
confcust.parse()
confcust.make_dirstruct()
confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
loop = uvloop.new_event_loop()
asyncio.set_event_loop(loop)
for cust in confcust.get_customers():
custname = confcust.get_custname(cust)
for job in confcust.get_jobs(cust):
logger.customer = confcust.get_custname(cust)
logger.job = job
profiles = confcust.get_profiles(job)
webapi_custopts = confcust.get_webapiopts(cust)
webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi')
webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi')
if not webapi_complete:
logger.error(
'Customer:%s Job:%s %s options incomplete, missing %s' %
(custname, logger.job, 'webapi', ' '.join(missopt)))
continue
try:
res = loop.run_until_complete(
fetch_data(webapi_opts['webapihost'],
webapi_opts['webapitoken']))
fetched_profiles = parse_source(res, profiles,
confcust.get_namespace(job))
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, True))
if eval(globopts['GeneralWriteAvro'.lower()]):
write_avro(cust, job, confcust, fixed_date,
fetched_profiles)
logger.info('Customer:' + custname + ' Job:' + job +
' Profiles:%s Tuples:%d' %
(', '.join(profiles), len(fetched_profiles)))
except (ConnectorHttpError, KeyboardInterrupt,
ConnectorParseError) as exc:
logger.error(repr(exc))
loop.run_until_complete(
write_state(cust, job, confcust, fixed_date, False))