def main():
    global logger, globopts, confcust

    parser = argparse.ArgumentParser(description="""Fetch and construct entities from EOSC-PORTAL feed""")
    parser.add_argument('-c', dest='custconf', nargs=1, metavar='customer.conf', help='path to customer configuration file', type=str, required=False)
    parser.add_argument('-g', dest='gloconf', nargs=1, metavar='global.conf', help='path to global configuration file', type=str, required=False)
    parser.add_argument('-d', dest='date', metavar='YEAR-MONTH-DAY', help='write data for this date', type=str, required=False)
    args = parser.parse_args()
    group_endpoints, group_groups = list(), list()
    logger = Logger(os.path.basename(sys.argv[0]))

    fixed_date = None
    if args.date and date_check(args.date):
        fixed_date = args.date

    confpath = args.gloconf[0] if args.gloconf else None
    cglob = Global(sys.argv[0], confpath)
    globopts = cglob.parse()

    confpath = args.custconf[0] if args.custconf else None
    confcust = CustomerConf(sys.argv[0], confpath)
    confcust.parse()
    confcust.make_dirstruct()
    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
    global custname
    custname = confcust.get_custname()

    # safely assume here one customer defined in customer file
    cust = list(confcust.get_customers())[0]
    jobstatedir = confcust.get_fullstatedir(globopts['InputStateSaveDir'.lower()], cust)
    fetchtype = confcust.get_topofetchtype()[0]

    state = None
    logger.customer = custname
    uidservtype = confcust.get_uidserviceendpoints()
    topofeed = confcust.get_topofeed()

    loop = uvloop.new_event_loop()
    asyncio.set_event_loop(loop)

    try:
        if is_feed(topofeed):
            res = loop.run_until_complete(fetch_data(topofeed))
            group_groups, group_endpoints = parse_source_topo(res, uidservtype, fetchtype)
            contacts = ParseContacts(logger, res, uidservtype, is_csv=False).get_contacts()
            attach_contacts_topodata(logger, contacts, group_endpoints)
        else:
            try:
                with open(topofeed) as fp:
                    js = json.load(fp)
                    group_groups, group_endpoints = parse_source_topo(js, uidservtype, fetchtype)
            except IOError as exc:
                logger.error('Customer:%s : Problem opening %s - %s' % (logger.customer, topofeed, repr(exc)))

        loop.run_until_complete(
            write_state(confcust, fixed_date, True)
        )

        webapi_opts = get_webapi_opts(cglob, confcust)

        numge = len(group_endpoints)
        numgg = len(group_groups)

        # send concurrently to WEB-API in coroutines
        if eval(globopts['GeneralPublishWebAPI'.lower()]):
            loop.run_until_complete(
                asyncio.gather(
                    send_webapi(webapi_opts, group_groups, 'groups', fixed_date),
                    send_webapi(webapi_opts, group_endpoints,'endpoints', fixed_date)
                )
            )

        if eval(globopts['GeneralWriteAvro'.lower()]):
            write_avro(confcust, group_groups, group_endpoints, fixed_date)

        logger.info('Customer:' + custname + ' Fetched Endpoints:%d' % (numge) + ' Groups(%s):%d' % (fetchtype, numgg))

    except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
        logger.error(repr(exc))
        loop.run_until_complete(
            write_state(confcust, fixed_date, False )
        )
Пример #2
0
def main():
    global logger, globopts, confcust
    parser = argparse.ArgumentParser(
        description="""Fetch entities (ServiceGroups, Sites, Endpoints)
                                                    from CSV topology feed for every customer and job listed in customer.conf and write them
                                                    in an appropriate place""")
    parser.add_argument('-c',
                        dest='custconf',
                        nargs=1,
                        metavar='customer.conf',
                        help='path to customer configuration file',
                        type=str,
                        required=False)
    parser.add_argument('-g',
                        dest='gloconf',
                        nargs=1,
                        metavar='global.conf',
                        help='path to global configuration file',
                        type=str,
                        required=False)
    parser.add_argument('-d',
                        dest='date',
                        metavar='YEAR-MONTH-DAY',
                        help='write data for this date',
                        type=str,
                        required=False)
    args = parser.parse_args()
    group_endpoints, group_groups = [], []
    logger = Logger(os.path.basename(sys.argv[0]))

    fixed_date = None
    if args.date and date_check(args.date):
        fixed_date = args.date

    confpath = args.gloconf[0] if args.gloconf else None
    cglob = Global(sys.argv[0], confpath)
    globopts = cglob.parse()

    confpath = args.custconf[0] if args.custconf else None
    confcust = CustomerConf(sys.argv[0], confpath)
    confcust.parse()
    confcust.make_dirstruct()
    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
    topofeed = confcust.get_topofeed()
    uidservtype = confcust.get_uidserviceendpoints()
    topofetchtype = confcust.get_topofetchtype()
    custname = confcust.get_custname()
    logger.customer = custname

    auth_custopts = confcust.get_authopts()
    auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
    auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
    if not auth_complete:
        logger.error('%s options incomplete, missing %s' %
                     ('authentication', ' '.join(missing)))
        raise SystemExit(1)

    loop = uvloop.new_event_loop()
    asyncio.set_event_loop(loop)

    try:
        group_endpoints, group_groups = list(), list()

        # fetch topology data concurrently in coroutines
        fetched_topology = loop.run_until_complete(
            fetch_data(topofeed, auth_opts))

        group_groups, group_endpoints = parse_source_topo(
            fetched_topology, custname, uidservtype)
        contacts = ParseContacts(logger,
                                 fetched_topology,
                                 uidservtype,
                                 is_csv=True).get_contacts()
        attach_contacts_topodata(logger, contacts, group_endpoints)

        loop.run_until_complete(write_state(confcust, fixed_date, True))

        webapi_opts = get_webapi_opts(cglob, confcust)

        numgg = len(group_groups)
        numge = len(group_endpoints)

        # send concurrently to WEB-API in coroutines
        if eval(globopts['GeneralPublishWebAPI'.lower()]):
            loop.run_until_complete(
                asyncio.gather(
                    send_webapi(webapi_opts, group_groups, 'groups',
                                fixed_date),
                    send_webapi(webapi_opts, group_endpoints, 'endpoints',
                                fixed_date)))

        if eval(globopts['GeneralWriteAvro'.lower()]):
            write_avro(confcust, group_groups, group_endpoints, fixed_date)

        logger.info('Customer:' + custname + ' Type:%s ' %
                    (','.join(topofetchtype)) + 'Fetched Endpoints:%d' %
                    (numge) + ' Groups:%d' % (numgg))

    except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
        logger.error(repr(exc))
        loop.run_until_complete(write_state(confcust, fixed_date, False))

    finally:
        loop.close()
Пример #3
0
def main():
    global logger, globopts
    parser = argparse.ArgumentParser(
        description='Fetch downtimes from GOCDB for given date')
    parser.add_argument('-d',
                        dest='date',
                        nargs=1,
                        metavar='YEAR-MONTH-DAY',
                        required=True)
    parser.add_argument('-c',
                        dest='custconf',
                        nargs=1,
                        metavar='customer.conf',
                        help='path to customer configuration file',
                        type=str,
                        required=False)
    parser.add_argument('-g',
                        dest='gloconf',
                        nargs=1,
                        metavar='global.conf',
                        help='path to global configuration file',
                        type=str,
                        required=False)
    args = parser.parse_args()

    logger = Logger(os.path.basename(sys.argv[0]))
    confpath = args.gloconf[0] if args.gloconf else None
    cglob = Global(sys.argv[0], confpath)
    globopts = cglob.parse()

    confpath = args.custconf[0] if args.custconf else None
    confcust = CustomerConf(sys.argv[0], confpath)
    confcust.parse()
    confcust.make_dirstruct()
    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
    feed = confcust.get_topofeed()
    logger.customer = confcust.get_custname()

    if len(args.date) == 0:
        print(parser.print_help())
        raise SystemExit(1)

    # calculate start and end times
    try:
        start = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
        end = datetime.datetime.strptime(args.date[0], '%Y-%m-%d')
        timestamp = start.strftime('%Y_%m_%d')
        start = start.replace(hour=0, minute=0, second=0)
        end = end.replace(hour=23, minute=59, second=59)
    except ValueError as exc:
        logger.error(exc)
        raise SystemExit(1)

    uidservtype = confcust.get_uidserviceendpoints()

    auth_custopts = confcust.get_authopts()
    auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
    auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
    if not auth_complete:
        missing_err = ''.join(missing)
        logger.error(
            'Customer:{} authentication options incomplete, missing {}'.format(
                logger.customer, missing_err))
        raise SystemExit(1)

    loop = uvloop.new_event_loop()
    asyncio.set_event_loop(loop)

    try:
        # we don't have multiple tenant definitions in one
        # customer file so we can safely assume one tenant/customer
        write_empty = confcust.send_empty(sys.argv[0])
        if not write_empty:
            res = loop.run_until_complete(
                fetch_data(feed, auth_opts, start, end))
            dts = parse_source(res, start, end, uidservtype)
        else:
            dts = []

        loop.run_until_complete(write_state(confcust, timestamp, True))

        webapi_opts = get_webapi_opts(cglob, confcust)

        if eval(globopts['GeneralPublishWebAPI'.lower()]):
            loop.run_until_complete(send_webapi(webapi_opts, args.date[0],
                                                dts))

        if dts or write_empty:
            cust = list(confcust.get_customers())[0]
            logger.info('Customer:%s Fetched Date:%s Endpoints:%d' %
                        (confcust.get_custname(cust), args.date[0], len(dts)))

        if eval(globopts['GeneralWriteAvro'.lower()]):
            write_avro(confcust, dts, timestamp)

    except (ConnectorHttpError, ConnectorParseError, KeyboardInterrupt) as exc:
        logger.error(repr(exc))
        loop.run_until_complete(write_state(confcust, timestamp, False))

    loop.close()
def main():
    global logger, globopts, confcust
    parser = argparse.ArgumentParser(
        description="""Fetch entities (ServiceGroups, Sites, Endpoints)
                                                    from GOCDB for every customer and job listed in customer.conf and write them
                                                    in an appropriate place""")
    parser.add_argument('-c',
                        dest='custconf',
                        nargs=1,
                        metavar='customer.conf',
                        help='path to customer configuration file',
                        type=str,
                        required=False)
    parser.add_argument('-g',
                        dest='gloconf',
                        nargs=1,
                        metavar='global.conf',
                        help='path to global configuration file',
                        type=str,
                        required=False)
    parser.add_argument('-d',
                        dest='date',
                        metavar='YEAR-MONTH-DAY',
                        help='write data for this date',
                        type=str,
                        required=False)
    args = parser.parse_args()
    group_endpoints, group_groups = [], []
    logger = Logger(os.path.basename(sys.argv[0]))

    fixed_date = None
    if args.date and date_check(args.date):
        fixed_date = args.date

    confpath = args.gloconf[0] if args.gloconf else None
    cglob = Global(sys.argv[0], confpath)
    globopts = cglob.parse()
    pass_extensions = eval(globopts['GeneralPassExtensions'.lower()])

    confpath = args.custconf[0] if args.custconf else None
    confcust = CustomerConf(sys.argv[0], confpath)
    confcust.parse()
    confcust.make_dirstruct()
    confcust.make_dirstruct(globopts['InputStateSaveDir'.lower()])
    topofeed = confcust.get_topofeed()
    topofeedpaging = confcust.get_topofeedpaging()
    uidservtype = confcust.get_uidserviceendpoints()
    topofetchtype = confcust.get_topofetchtype()
    custname = confcust.get_custname()
    logger.customer = custname

    auth_custopts = confcust.get_authopts()
    auth_opts = cglob.merge_opts(auth_custopts, 'authentication')
    auth_complete, missing = cglob.is_complete(auth_opts, 'authentication')
    if not auth_complete:
        logger.error('%s options incomplete, missing %s' %
                     ('authentication', ' '.join(missing)))
        raise SystemExit(1)

    bdii_opts = get_bdii_opts(confcust)

    loop = uvloop.new_event_loop()
    asyncio.set_event_loop(loop)

    group_endpoints, group_groups = list(), list()
    parsed_site_contacts, parsed_servicegroups_contacts, parsed_serviceendpoint_contacts = None, None, None

    try:
        contact_coros = [
            fetch_data(topofeed + SITE_CONTACTS, auth_opts, False),
            fetch_data(topofeed + SERVICEGROUP_CONTACTS, auth_opts, False)
        ]
        contacts = loop.run_until_complete(
            asyncio.gather(*contact_coros, return_exceptions=True))

        exc_raised, exc = contains_exception(contacts)
        if exc_raised:
            raise ConnectorHttpError(repr(exc))

        parsed_site_contacts = parse_source_sitescontacts(
            contacts[0], custname)
        parsed_servicegroups_contacts = parse_source_servicegroupsroles(
            contacts[1], custname)

    except (ConnectorHttpError, ConnectorParseError) as exc:
        logger.warn(
            'SITE_CONTACTS and SERVICERGOUP_CONTACT methods not implemented')

    try:
        toposcope = confcust.get_toposcope()
        topofeedendpoints = confcust.get_topofeedendpoints()
        topofeedservicegroups = confcust.get_topofeedservicegroups()
        topofeedsites = confcust.get_topofeedsites()
        global SERVICE_ENDPOINTS_PI, SERVICE_GROUPS_PI, SITES_PI
        if toposcope:
            SERVICE_ENDPOINTS_PI = SERVICE_ENDPOINTS_PI + toposcope
            SERVICE_GROUPS_PI = SERVICE_GROUPS_PI + toposcope
            SITES_PI = SITES_PI + toposcope
        if topofeedendpoints:
            SERVICE_ENDPOINTS_PI = topofeedendpoints
        else:
            SERVICE_ENDPOINTS_PI = topofeed + SERVICE_ENDPOINTS_PI
        if topofeedservicegroups:
            SERVICE_GROUPS_PI = topofeedservicegroups
        else:
            SERVICE_GROUPS_PI = topofeed + SERVICE_GROUPS_PI
        if topofeedsites:
            SITES_PI = topofeedsites
        else:
            SITES_PI = topofeed + SITES_PI

        fetched_sites, fetched_servicegroups, fetched_endpoints = None, None, None
        fetched_bdii = None

        coros = [fetch_data(SERVICE_ENDPOINTS_PI, auth_opts, topofeedpaging)]
        if 'servicegroups' in topofetchtype:
            coros.append(
                fetch_data(SERVICE_GROUPS_PI, auth_opts, topofeedpaging))
        if 'sites' in topofetchtype:
            coros.append(fetch_data(SITES_PI, auth_opts, topofeedpaging))

        if bdii_opts and eval(bdii_opts['bdii']):
            host = bdii_opts['bdiihost']
            port = bdii_opts['bdiiport']
            base = bdii_opts['bdiiquerybase']

            coros.append(
                fetch_ldap_data(
                    host, port, base, bdii_opts['bdiiqueryfiltersrm'],
                    bdii_opts['bdiiqueryattributessrm'].split(' ')))

            coros.append(
                fetch_ldap_data(
                    host, port, base, bdii_opts['bdiiqueryfiltersepath'],
                    bdii_opts['bdiiqueryattributessepath'].split(' ')))

        # fetch topology data concurrently in coroutines
        fetched_topology = loop.run_until_complete(
            asyncio.gather(*coros, return_exceptions=True))

        fetched_endpoints = fetched_topology[0]
        if bdii_opts and eval(bdii_opts['bdii']):
            fetched_bdii = list()
            fetched_bdii.append(fetched_topology[-2])
            fetched_bdii.append(fetched_topology[-1])
        if 'sites' in topofetchtype and 'servicegroups' in topofetchtype:
            fetched_servicegroups, fetched_sites = (fetched_topology[1],
                                                    fetched_topology[2])
        elif 'sites' in topofetchtype:
            fetched_sites = fetched_topology[1]
        elif 'servicegroups' in topofetchtype:
            fetched_servicegroups = fetched_topology[1]

        exc_raised, exc = contains_exception(fetched_topology)
        if exc_raised:
            raise ConnectorHttpError(repr(exc))

        # proces data in parallel using multiprocessing
        executor = ProcessPoolExecutor(max_workers=3)
        parse_workers = list()
        exe_parse_source_endpoints = partial(parse_source_endpoints,
                                             fetched_endpoints, custname,
                                             uidservtype, pass_extensions)
        exe_parse_source_servicegroups = partial(parse_source_servicegroups,
                                                 fetched_servicegroups,
                                                 custname, uidservtype,
                                                 pass_extensions)
        exe_parse_source_sites = partial(parse_source_sites, fetched_sites,
                                         custname, uidservtype,
                                         pass_extensions)

        # parse topology depend on configured components fetch. we can fetch
        # only sites, only servicegroups or both.
        if fetched_servicegroups and fetched_sites:
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_endpoints))
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_servicegroups))
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_sites))
        elif fetched_servicegroups and not fetched_sites:
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_servicegroups))
        elif fetched_sites and not fetched_servicegroups:
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_endpoints))
            parse_workers.append(
                loop.run_in_executor(executor, exe_parse_source_sites))

        parsed_topology = loop.run_until_complete(
            asyncio.gather(*parse_workers))

        if fetched_servicegroups and fetched_sites:
            group_endpoints = parsed_topology[0]
            group_groups, group_endpoints_sg = parsed_topology[1]
            group_endpoints += group_endpoints_sg
            group_groups += parsed_topology[2]
        elif fetched_servicegroups and not fetched_sites:
            group_groups, group_endpoints = parsed_topology[0]
        elif fetched_sites and not fetched_servicegroups:
            group_endpoints = parsed_topology[0]
            group_groups = parsed_topology[1]

        # check if we fetched SRM port info and attach it appropriate endpoint
        # data
        if bdii_opts and eval(bdii_opts['bdii']):
            attach_srmport_topodata(
                logger, bdii_opts['bdiiqueryattributessrm'].split(' ')[0],
                fetched_bdii[0], group_endpoints)
            attach_sepath_topodata(
                logger, bdii_opts['bdiiqueryattributessepath'].split(' ')[0],
                fetched_bdii[1], group_endpoints)

        # parse contacts from fetched service endpoints topology, if there are
        # any
        parsed_serviceendpoint_contacts = parse_source_serviceendpoints_contacts(
            fetched_endpoints, custname)

        if not parsed_site_contacts and fetched_sites:
            # GOCDB has not SITE_CONTACTS, try to grab contacts from fetched
            # sites topology entities
            parsed_site_contacts = parse_source_siteswithcontacts(
                fetched_sites, custname)

        attach_contacts_workers = [
            loop.run_in_executor(
                executor,
                partial(attach_contacts_topodata, logger, parsed_site_contacts,
                        group_groups)),
            loop.run_in_executor(
                executor,
                partial(attach_contacts_topodata, logger,
                        parsed_serviceendpoint_contacts, group_endpoints))
        ]

        executor = ProcessPoolExecutor(max_workers=2)
        group_groups, group_endpoints = loop.run_until_complete(
            asyncio.gather(*attach_contacts_workers))

        if parsed_servicegroups_contacts:
            attach_contacts_topodata(logger, parsed_servicegroups_contacts,
                                     group_groups)
        elif fetched_servicegroups:
            # GOCDB has not SERVICEGROUP_CONTACTS, try to grab contacts from fetched
            # servicegroups topology entities
            parsed_servicegroups_contacts = parse_source_servicegroupscontacts(
                fetched_servicegroups, custname)
            attach_contacts_topodata(logger, parsed_servicegroups_contacts,
                                     group_groups)

        loop.run_until_complete(write_state(confcust, fixed_date, True))

        webapi_opts = get_webapi_opts(cglob, confcust)

        numge = len(group_endpoints)
        numgg = len(group_groups)

        # send concurrently to WEB-API in coroutines
        if eval(globopts['GeneralPublishWebAPI'.lower()]):
            loop.run_until_complete(
                asyncio.gather(
                    send_webapi(webapi_opts, group_groups, 'groups',
                                fixed_date),
                    send_webapi(webapi_opts, group_endpoints, 'endpoints',
                                fixed_date)))

        if eval(globopts['GeneralWriteAvro'.lower()]):
            write_avro(confcust, group_groups, group_endpoints, fixed_date)

        logger.info('Customer:' + custname + ' Type:%s ' %
                    (','.join(topofetchtype)) + 'Fetched Endpoints:%d' %
                    (numge) + ' Groups:%d' % (numgg))

    except (ConnectorParseError, ConnectorHttpError, KeyboardInterrupt) as exc:
        logger.error(repr(exc))
        loop.run_until_complete(write_state(confcust, fixed_date, False))

    finally:
        loop.close()