def poemProfileFilenameCheck(year, month, day): count = 0 dt = datetime.datetime(year=int(year), month=int(month), day=int(day)) while True: year = dt.strftime("%Y") month = dt.strftime("%m") day = dt.strftime("%d") fileName = filename_date( logger, globopts['PrefilterPoemExpandedProfiles'.lower()], '', stamp=year + '_' + month + '_' + day) if os.path.isfile(fileName): break if count >= int( globopts['PrefilterLookbackPoemExpandedProfiles'.lower()]): fileName = None break count = count + 1 dt = dt - datetime.timedelta(days=1) return fileName
def main(): global logger, globopts parser = argparse.ArgumentParser(description="""Fetch weights information from Gstat provider for every job listed in customer.conf""") parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) for feed, jobcust in feeds.items(): weights = Vapor(feed) datawr = None customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) customers = customers.pop() if len(customers) == 1 else '({0})'.format(','.join(customers)) jobs = set(map(lambda jc: jc[0], jobcust)) jobs = jobs.pop() if len(jobs) == 1 else '({0})'.format(','.join(jobs)) logger.job = jobs logger.customer = customers w = weights.getWeights() for job, cust in jobcust: logger.customer = confcust.get_custname(cust) logger.job = job jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) custname = confcust.get_custname(cust) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) continue if fixed_date: output.write_state(sys.argv[0], jobstatedir, weights.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, weights.state, globopts['InputStateDays'.lower()]) if not weights.state: continue datawr = data_out(w) if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish(ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasWeights'.lower()], 'weights', partdate, datawr) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasWeights'.lower()], filename) ret, excep = avro.write(datawr) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) if datawr: custs = set([cust for job, cust in jobcust]) for cust in custs: jobs = [job for job, lcust in jobcust if cust == lcust] logger.info('Customer:%s Jobs:%s Sites:%d' % (confcust.get_custname(cust), jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), len(datawr)))
def main(): global logger, globopts parser = argparse.ArgumentParser(description='Fetch downtimes from GOCDB for given date') parser.add_argument('-d', dest='date', nargs=1, metavar='YEAR-MONTH-DAY', required=True) parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed='https://goc.egi.eu/gocdbpi/') if len(args.date) == 0: print parser.print_help() raise SystemExit(1) # calculate start and end times try: start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d') timestamp = start.strftime('%Y_%m_%d') start = start.replace(hour=0, minute=0, second=0) end = end.replace(hour=23, minute=59, second=59) except ValueError as e: logger.error(e) raise SystemExit(1) for feed, jobcust in feeds.items(): customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) customers = customers.pop() if len(customers) == 1 else '({0})'.format(','.join(customers)) jobs = set(map(lambda jc: jc[0], jobcust)) jobs = jobs.pop() if len(jobs) == 1 else '({0})'.format(','.join(jobs)) logger.job = jobs logger.customer = customers auth_custopts = confcust.get_authopts(feed, jobcust) auth_opts = cglob.merge_opts(auth_custopts, 'authentication') auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') if auth_complete: gocdb = GOCDBReader(feed, auth=auth_opts) dts = gocdb.getDowntimes(start, end) else: logger.error('Customer:%s Jobs:%s %s options incomplete, missing %s' % (logger.customer, logger.job, 'authentication', ''.join(missing))) continue for job, cust in jobcust: jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) logger.customer = confcust.get_custname(cust) logger.job = job ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (logger.customer, job, 'ams', ' '.join(missopt))) continue output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()], timestamp) if not gocdb.state: continue if eval(globopts['GeneralPublishAms'.lower()]): ams = output.AmsPublish(ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasDowntimes'.lower()], 'downtimes', timestamp.replace('_', '-'), dts) if eval(globopts['GeneralWriteAvro'.lower()]): filename = filename_date(logger, globopts['OutputDowntimes'.lower()], jobdir, stamp=timestamp) avro = output.AvroWriter(globopts['AvroSchemasDowntimes'.lower()], filename) ret, excep = avro.write(dts) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) if gocdb.state: custs = set([cust for job, cust in jobcust]) for cust in custs: jobs = [job for job, lcust in jobcust if cust == lcust] logger.info('Customer:%s Jobs:%s Fetched Date:%s Endpoints:%d' % (confcust.get_custname(cust), jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), args.date[0], len(dts)))
def main(): parser = argparse.ArgumentParser( description="""Fetch and construct entities from NEANIAS feed""") parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = [], [] logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) for cust in confcust.get_customers(): custname = confcust.get_custname(cust) for job in confcust.get_jobs(cust): jobdir = confcust.get_fulldir(cust, job) logger.customer = confcust.get_custname(cust) jobstatedir = confcust.get_fullstatedir( globopts['InputStateSaveDir'.lower()], cust, job) fetchtype = confcust.get_fetchtype(job) state = None logger.job = job logger.customer = custname uidservtype = confcust.pass_uidserviceendpoints(job) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') feeds = confcust.get_mapfeedjobs(sys.argv[0]) if is_feed(feeds.keys()[0]): remote_topo = urlparse(feeds.keys()[0]) res = input.connection(logger, 'NEANIAS', globopts, remote_topo.scheme, remote_topo.netloc, remote_topo.path) if not res: raise input.ConnectorError() doc = input.parse_json( logger, 'NEANIAS', globopts, res, remote_topo.scheme + '://' + remote_topo.netloc + remote_topo.path) eosc = EOSCReader(doc, uidservtype, fetchtype) group_groups = eosc.get_groupgroups() group_endpoints = eosc.get_groupendpoints() state = True else: try: with open(feeds.keys()[0]) as fp: js = json.load(fp) eosc = EOSCReader(js, uidservtype, fetchtype) group_groups = eosc.get_groupgroups() group_endpoints = eosc.get_groupendpoints() state = True except IOError as exc: logger.error( 'Customer:%s Job:%s : Problem opening %s - %s' % (logger.customer, logger.job, feeds.keys()[0], repr(exc))) state = False if fixed_date: output.write_state(sys.argv[0], jobstatedir, state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, state, globopts['InputStateDays'.lower()]) if not state: continue numge = len(group_endpoints) numgg = len(group_groups) if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish( ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], 'group_groups', partdate, group_groups) ams.send( globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], 'group_endpoints', partdate, group_endpoints) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date( logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date( logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir) avro = output.AvroWriter( globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) ret, excep = avro.write(group_groups) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) if fixed_date: filename = filename_date( logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date( logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir) avro = output.AvroWriter( globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) ret, excep = avro.write(group_endpoints) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:' + custname + ' Job:' + job + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg))
def main(): global logger, globopts parser = argparse.ArgumentParser( description="""Fetch weights information from Gstat provider for every job listed in customer.conf""" ) parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], deffeed=VAPORPI) j = 0 for feed, jobcust in feeds.items(): weights = Vapor(feed) datawr = None customers = set(map(lambda jc: confcust.get_custname(jc[1]), jobcust)) customers = customers.pop() if len(customers) == 1 else '({0})'.format( ','.join(customers)) sjobs = set(map(lambda jc: jc[0], jobcust)) jobs = list(sjobs)[0] if len(sjobs) == 1 else '({0})'.format( ','.join(sjobs)) logger.job = jobs logger.customer = customers for job, cust in jobcust: logger.customer = confcust.get_custname(cust) logger.job = job write_empty = confcust.send_empty(sys.argv[0], cust) if not write_empty: w = weights.getWeights() else: w = [] weights.state = True jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir( globopts['InputStateSaveDir'.lower()], cust, job) custname = confcust.get_custname(cust) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) continue if fixed_date: output.write_state(sys.argv[0], jobstatedir, weights.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, weights.state, globopts['InputStateDays'.lower()]) if not weights.state: continue datawr = data_out(w) if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish( ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasWeights'.lower()], 'weights', partdate, datawr) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputWeights'.lower()], jobdir) avro = output.AvroWriter( globopts['AvroSchemasWeights'.lower()], filename) ret, excep = avro.write(datawr) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) j += 1 if datawr or write_empty: custs = set([cust for job, cust in jobcust]) for cust in custs: jobs = [job for job, lcust in jobcust if cust == lcust] logger.info( 'Customer:%s Jobs:%s Sites:%d' % (confcust.get_custname(cust), jobs[0] if len(jobs) == 1 else '({0})'.format(','.join(jobs)), len(datawr)))
def main(): global logger, globopts, confcust parser = argparse.ArgumentParser( description="""Fetch entities (ServiceGroups, Sites, Endpoints) from GOCDB for every customer and job listed in customer.conf and write them in an appropriate place""") parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = [], [] logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], 'GOCDB', deffeed='https://goc.egi.eu/gocdbpi/') for feed, jobcust in feeds.items(): scopes = confcust.get_feedscopes(feed, jobcust) paging = confcust.is_paginated(feed, jobcust) auth_custopts = confcust.get_authopts(feed, jobcust) auth_opts = cglob.merge_opts(auth_custopts, 'authentication') auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') if auth_complete: gocdb = GOCDBReader(feed, scopes, paging, auth=auth_opts) else: logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) continue for job, cust in jobcust: jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir( globopts['InputStateSaveDir'.lower()], cust, job) global fetchtype, custname fetchtype = confcust.get_fetchtype(job) uidservtype = confcust.pass_uidserviceendpoints(job) custname = confcust.get_custname(cust) logger.customer = custname logger.job = job ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error( 'Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'ams', ' '.join(missopt))) continue if fetchtype == 'ServiceGroups': group_endpoints = gocdb.getGroupOfServices(uidservtype) else: group_endpoints = gocdb.getGroupOfEndpoints(uidservtype) group_groups = gocdb.getGroupOfGroups() if fixed_date: output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()]) if not gocdb.state: continue numge = len(group_endpoints) numgg = len(group_groups) ggtags = confcust.get_gocdb_ggtags(job) getags = confcust.get_gocdb_getags(job) tf = TopoFilter(group_groups, group_endpoints, ggtags, getags) group_groups = tf.gg group_endpoints = tf.ge if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish( ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], 'group_groups', partdate, group_groups) ams.send( globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], 'group_endpoints', partdate, group_endpoints) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date( logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date( logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir) avro = output.AvroWriter( globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) ret, excep = avro.write(group_groups) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) if fixed_date: filename = filename_date( logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date( logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir) avro = output.AvroWriter( globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) ret, excep = avro.write(group_endpoints) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:' + custname + ' Job:' + job + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg)) if getags or ggtags: selstr = 'Customer:%s Job:%s Selected ' % (custname, job) selge, selgg = '', '' if getags: for key, value in getags.items(): if isinstance(value, list): value = '[' + ','.join(value) + ']' selge += '%s:%s,' % (key, value) selstr += 'Endpoints(%s):' % selge[:len(selge) - 1] selstr += '%d ' % (len(group_endpoints)) if ggtags: for key, value in ggtags.items(): if isinstance(value, list): value = '[' + ','.join(value) + ']' selgg += '%s:%s,' % (key, value) selstr += 'Groups(%s):' % selgg[:len(selgg) - 1] selstr += '%d' % (len(group_groups)) logger.info(selstr)
def main(): parser = optparse.OptionParser( description="""Filters consumer messages based on various criteria (allowed NGIs, service flavours, metrics...)""" ) parser.add_option('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str) group = optparse.OptionGroup(parser, 'Compute Engine usage') group.add_option('-d', dest='date', nargs=1, metavar='YEAR-MONTH-DAY') parser.add_option_group(group) group = optparse.OptionGroup(parser, 'Debugging usage') group.add_option('-c', dest='cfile', default=None, metavar='consumer_log_YEAR-MONTH-DAY.avro') group.add_option('-p', dest='pfile', default=None, metavar='poem_sync_YEAR-MONTH-DAY.out') group.add_option('-o', dest='ofile', default=None, metavar='output_YEAR-MONTH-DAY.avro') parser.add_option_group(group) (options, args) = parser.parse_args() global logger logger = Logger(os.path.basename(sys.argv[0])) confpath = options.gloconf if options.gloconf else None cglob = Global(sys.argv[0], confpath) global globopts globopts = cglob.parse() if options.cfile and options.date: parser.print_help() raise SystemExit(1) elif options.cfile: fname = options.cfile date = options.cfile.split('_')[-1] date = date.split('.')[0] date = date.split('-') elif options.date: date = options.date.split('-') else: parser.print_help() raise SystemExit(1) if len(date) == 0 or len(date) != 3: logger.error( 'Consumer file does not end with correctly formatted date') parser.print_help() raise SystemExit(1) year, month, day = date # avro files if options.cfile: inputFile = options.cfile else: inputFile = filename_date( logger, globopts['PrefilterConsumerFilePath'.lower()], '', stamp=year + '-' + month + '-' + day) if options.ofile: fname = options.ofile + '_DATE.avro' outputFile = filename_date(logger, fname, '', stamp=year + '_' + month + '_' + day) else: outputFile = filename_date(logger, globopts['OutputPrefilter'.lower()], '', stamp=year + '_' + month + '_' + day) try: schema = avro.schema.parse( open(globopts['AvroSchemasPrefilter'.lower()]).read()) writer = DataFileWriter(open(outputFile, "w"), DatumWriter(), schema) reader = DataFileReader(open(inputFile, "r"), DatumReader()) except IOError as e: logger.error(str(e)) raise SystemExit(1) # load poem data if options.pfile: ngis = loadNGIs(poemfile=options.pfile) profiles = loadFilteredProfiles(poemfile=options.pfile) else: ngis = loadNGIs(year, month, day) profiles = loadFilteredProfiles(year, month, day) nameMapping = loadNameMapping(year, month, day) s = time.time() msgs, msgswrit, msgsfilt, falsemonhost, falseroc, falseprofile = prefilterit( reader, writer, ngis, profiles, nameMapping) e = time.time() logger.info( 'ExecTime:%.2fs ConsumerDate:%s Read:%d Written:%d Filtered:%d(Monitoring_Host:%d,ROC:%d,ServiceTypes_Metrics:%d)' % (round(e - s, 2), year + '-' + month + '-' + day, msgs, msgswrit, msgsfilt, falsemonhost, falseroc, falseprofile)) reader.close() writer.close()
def main(): global logger, globopts parser = argparse.ArgumentParser(description='Fetch POEM profile for every job of the customer and write POEM expanded profiles needed for prefilter for EGI customer') parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) for cust in confcust.get_customers(): custname = confcust.get_custname(cust) for job in confcust.get_jobs(cust): logger.customer = confcust.get_custname(cust) logger.job = job poemserver = dict() profiles = confcust.get_profiles(job) namespace = confcust.get_namespace(job) poemserver[confcust.get_poemserver_host(job)] = confcust.get_poemserver_vo(job) poem = PoemReader(custname, job) psa = poem.getProfiles(profiles, namespace, poemserver) jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) continue if fixed_date: output.write_state(sys.argv[0], jobstatedir, poem.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, poem.state, globopts['InputStateDays'.lower()]) if not poem.state: continue lfprofiles = gen_outprofiles(psa, profiles) if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish(ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasPoem'.lower()], 'metric_profile', partdate, lfprofiles) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasPoem'.lower()], filename) ret, excep = avro.write(lfprofiles) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:'+custname+' Job:'+job+' Profiles:%s Tuples:%d' % (','.join(profiles), len(lfprofiles)))
def main(): global logger, globopts, confcust parser = argparse.ArgumentParser(description="""Fetch entities (ServiceGroups, Sites, Endpoints) from GOCDB for every customer and job listed in customer.conf and write them in an appropriate place""") parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() group_endpoints, group_groups = [], [] logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) feeds = confcust.get_mapfeedjobs(sys.argv[0], 'GOCDB', deffeed='https://goc.egi.eu/gocdbpi/') for feed, jobcust in feeds.items(): scopes = confcust.get_feedscopes(feed, jobcust) paging = confcust.is_paginated(feed, jobcust) auth_custopts = confcust.get_authopts(feed, jobcust) auth_opts = cglob.merge_opts(auth_custopts, 'authentication') auth_complete, missing = cglob.is_complete(auth_opts, 'authentication') if auth_complete: gocdb = GOCDBReader(feed, scopes, paging, auth=auth_opts) else: logger.error('%s options incomplete, missing %s' % ('authentication', ' '.join(missing))) continue for job, cust in jobcust: jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) global fetchtype, custname fetchtype = confcust.get_gocdb_fetchtype(job) custname = confcust.get_custname(cust) logger.customer = custname logger.job = job ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'ams', ' '.join(missopt))) continue if fetchtype == 'ServiceGroups': group_endpoints = gocdb.getGroupOfServices() else: group_endpoints = gocdb.getGroupOfEndpoints() group_groups = gocdb.getGroupOfGroups() if fixed_date: output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, gocdb.state, globopts['InputStateDays'.lower()]) if not gocdb.state: continue numge = len(group_endpoints) numgg = len(group_groups) ggtags = confcust.get_gocdb_ggtags(job) getags = confcust.get_gocdb_getags(job) tf = TopoFilter(group_groups, group_endpoints, ggtags, getags) group_groups = tf.gg group_endpoints = tf.ge if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish(ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], 'group_groups', partdate, group_groups) ams.send(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], 'group_endpoints', partdate, group_endpoints) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputTopologyGroupOfGroups'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfGroups'.lower()], filename) ret, excep = avro.write(group_groups) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) if fixed_date: filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputTopologyGroupOfEndpoints'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasTopologyGroupOfEndpoints'.lower()], filename) ret, excep = avro.write(group_endpoints) if not ret: logger.error('Customer:%s Job:%s : %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:'+custname+' Job:'+job+' Fetched Endpoints:%d' % (numge) +' Groups(%s):%d' % (fetchtype, numgg)) if getags or ggtags: selstr = 'Customer:%s Job:%s Selected ' % (custname, job) selge, selgg = '', '' if getags: for key, value in getags.items(): if isinstance(value, list): value = '['+','.join(value)+']' selge += '%s:%s,' % (key, value) selstr += 'Endpoints(%s):' % selge[:len(selge) - 1] selstr += '%d ' % (len(group_endpoints)) if ggtags: for key, value in ggtags.items(): if isinstance(value, list): value = '['+','.join(value)+']' selgg += '%s:%s,' % (key, value) selstr += 'Groups(%s):' % selgg[:len(selgg) - 1] selstr += '%d' % (len(group_groups)) logger.info(selstr)
def main(): global logger, globopts parser = argparse.ArgumentParser(description='Fetch metric profile for every job of the customer') parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) for cust in confcust.get_customers(): custname = confcust.get_custname(cust) for job in confcust.get_jobs(cust): logger.customer = confcust.get_custname(cust) logger.job = job profiles = confcust.get_profiles(job) webapi_custopts = confcust.get_webapiopts(cust) webapi_opts = cglob.merge_opts(webapi_custopts, 'webapi') webapi_complete, missopt = cglob.is_complete(webapi_opts, 'webapi') if not webapi_complete: logger.error('Customer:%s Job:%s %s options incomplete, missing %s' % (custname, logger.job, 'webapi', ' '.join(missopt))) continue webapi = WebAPI(custname, job, profiles, confcust.get_namespace(job), webapi_opts['webapihost'], webapi_opts['webapitoken']) fetched_profiles = webapi.get_profiles() jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust, job) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) continue if fixed_date: output.write_state(sys.argv[0], jobstatedir, webapi.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, webapi.state, globopts['InputStateDays'.lower()]) if not webapi.state: continue if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp(1).replace('_', '-') ams = output.AmsPublish(ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasMetricProfile'.lower()], 'metric_profile', partdate, fetched_profiles) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputMetricProfile'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasMetricProfile'.lower()], filename) ret, excep = avro.write(fetched_profiles) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:' + custname + ' Job:' + job + ' Profiles:%s Tuples:%d' % (', '.join(profiles), len(fetched_profiles)))
def main(): global logger, globopts parser = argparse.ArgumentParser( description= 'Fetch POEM profile for every job of the customer and write POEM expanded profiles needed for prefilter for EGI customer' ) parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False) parser.add_argument( '-np', dest='noprefilter', help='do not write POEM expanded profiles for prefilter', required=False, action='store_true') parser.add_argument('-p', dest='poemconf', nargs=1, metavar='poem-connector.conf', help='path to poem-connector configuration file', type=str, required=False) parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False) parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False) args = parser.parse_args() logger = Logger(os.path.basename(sys.argv[0])) fixed_date = None if args.date and date_check(args.date): fixed_date = args.date confpath = args.gloconf[0] if args.gloconf else None cglob = Global(sys.argv[0], confpath) globopts = cglob.parse() servers = {'PoemServer': ['Host', 'VO']} filterprofiles = {'FetchProfiles': ['List']} prefilterdata = { 'PrefilterData': ['AllowedNGI', 'AllowedNGIProfiles', 'AllNGI', 'AllNGIProfiles'] } global cpoem, poemopts confpath = args.poemconf[0] if args.poemconf else None cpoem = PoemConf(confpath, servers, filterprofiles, prefilterdata) poemopts = cpoem.parse() confpath = args.custconf[0] if args.custconf else None confcust = CustomerConf(sys.argv[0], confpath) confcust.parse() confcust.make_dirstruct() confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()]) customers = set( map(lambda c: confcust.get_custname(c), confcust.get_customers())) customers = customers.pop() if len(customers) == 1 else '({0})'.format( ','.join(customers)) logger.customer = customers customers = confcust.get_customers() jobs = list() for c in customers: jobs = jobs + confcust.get_jobs(c) jobs = jobs.pop() if len(jobs) == 1 else '({0})'.format(','.join(jobs)) logger.job = jobs readerInstance = PoemReader(args.noprefilter) ps, psa = readerInstance.getProfiles() if not args.noprefilter and ps: poempref = PrefilterPoem() preffname = filename_date( logger, globopts['PrefilterPoemExpandedProfiles'.lower()], '') poempref.writeProfiles(ps, preffname) for cust in confcust.get_customers(): # write profiles custname = confcust.get_custname(cust) for job in confcust.get_jobs(cust): logger.customer = confcust.get_custname(cust) logger.job = job jobdir = confcust.get_fulldir(cust, job) jobstatedir = confcust.get_fullstatedir( globopts['InputStateSaveDir'.lower()], cust, job) ams_custopts = confcust.get_amsopts(cust) ams_opts = cglob.merge_opts(ams_custopts, 'ams') ams_complete, missopt = cglob.is_complete(ams_opts, 'ams') if not ams_complete: logger.error('Customer:%s %s options incomplete, missing %s' % (custname, 'ams', ' '.join(missopt))) continue if fixed_date: output.write_state(sys.argv[0], jobstatedir, readerInstance.state, globopts['InputStateDays'.lower()], fixed_date.replace('-', '_')) else: output.write_state(sys.argv[0], jobstatedir, readerInstance.state, globopts['InputStateDays'.lower()]) if not readerInstance.state: continue profiles = confcust.get_profiles(job) lfprofiles = gen_outprofiles(psa, profiles) if eval(globopts['GeneralPublishAms'.lower()]): if fixed_date: partdate = fixed_date else: partdate = datestamp().replace('_', '-') ams = output.AmsPublish( ams_opts['amshost'], ams_opts['amsproject'], ams_opts['amstoken'], ams_opts['amstopic'], confcust.get_jobdir(job), ams_opts['amsbulk'], ams_opts['amspacksinglemsg'], logger, int(globopts['ConnectionRetry'.lower()]), int(globopts['ConnectionTimeout'.lower()])) ams.send(globopts['AvroSchemasPoem'.lower()], 'metric_profile', partdate, lfprofiles) if eval(globopts['GeneralWriteAvro'.lower()]): if fixed_date: filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir, fixed_date.replace('-', '_')) else: filename = filename_date(logger, globopts['OutputPoem'.lower()], jobdir) avro = output.AvroWriter(globopts['AvroSchemasPoem'.lower()], filename) ret, excep = avro.write(lfprofiles) if not ret: logger.error('Customer:%s Job:%s %s' % (logger.customer, logger.job, repr(excep))) raise SystemExit(1) logger.info('Customer:' + custname + ' Job:' + job + ' Profiles:%s Tuples:%d' % (','.join(profiles), len(lfprofiles)))