예제 #1
0
def list_repos_ids():
    logging.debug("Listing repos ids")
    elastic = get_elastic()
    ConfOcean.set_elastic(elastic)

    for repo_id in ConfOcean.get_repos_ids():
        print(repo_id)
예제 #2
0
def get_perceval_params(url, index):
    logging.info("Get perceval params for index: %s" % (index))
    elastic = get_elastic(url, ConfOcean.get_index())
    ConfOcean.set_elastic(elastic)

    r = requests.get(elastic.index_url+"/repos/"+index)

    params = r.json()['_source']['params']

    return params
예제 #3
0
def list_repos():
    logging.debug("Listing repos")
    elastic = get_elastic()
    ConfOcean.set_elastic(elastic)

    for repo_id in ConfOcean.get_repos_ids():
        elastic = get_elastic()
        url = elastic.index_url + "/repos/" + repo_id
        r = requests.get(url)
        repo = r.json()['_source']
        print ("%s %s %s" % (repo_id, repo['repo_update'], repo['success']))
예제 #4
0
파일: p2o.py 프로젝트: dpose/GrimoireELK
def feed_backends(url, clean, debug = False, redis = None):
    ''' Update Ocean for all existing backends '''

    logging.info("Updating all Ocean")
    elastic = get_elastic(url, ConfOcean.get_index(), clean)
    ConfOcean.set_elastic(elastic)
    fetch_cache = False

    q = Queue('update', connection=Redis(redis), async=async_)

    for repo in ConfOcean.get_repos():
        task_feed = q.enqueue(feed_backend, url, clean, fetch_cache,
                              repo['backend_name'], repo['backend_params'],
                              repo['index'], repo['index_enrich'], repo['project'])
        logging.info("Queued job")
        logging.info(task_feed)
예제 #5
0
def feed_backends(url, clean, debug = False, redis = None):
    ''' Update Ocean for all existing backends '''

    logging.info("Updating all Ocean")
    elastic = get_elastic(url, ConfOcean.get_index(), clean)
    ConfOcean.set_elastic(elastic)
    fetch_cache = False

    q = Queue('update', connection=Redis(redis), async=async_)

    for repo in ConfOcean.get_repos():
        task_feed = q.enqueue(feed_backend, url, clean, fetch_cache,
                              repo['backend_name'], repo['backend_params'],
                              repo['index'], repo['index_enrich'], repo['project'])
        logging.info("Queued job")
        logging.info(task_feed)
예제 #6
0
파일: p2o.py 프로젝트: dpose/GrimoireELK
def enrich_backends(url, clean, debug = False, redis = None,
                    db_projects_map=None, db_sortinghat=None):
    ''' Enrich all existing indexes '''

    logging.info("Enriching repositories")

    elastic = get_elastic(url, ConfOcean.get_index(), clean)
    ConfOcean.set_elastic(elastic)
    fetch_cache = False

    q = Queue('update', connection=Redis(redis), async=async_)

    for repo in ConfOcean.get_repos():
        enrich_task = q.enqueue(enrich_backend,
                                url, clean,
                                repo['backend_name'], repo['backend_params'],
                                repo['index'], repo['index_enrich'], db_projects_map, db_sortinghat)
        logging.info("Queued job")
        logging.info(enrich_task)
예제 #7
0
def enrich_backends(url, clean, debug = False, redis = None,
                    db_projects_map=None, db_sortinghat=None):
    ''' Enrich all existing indexes '''

    logging.info("Enriching repositories")

    elastic = get_elastic(url, ConfOcean.get_index(), clean)
    ConfOcean.set_elastic(elastic)
    fetch_cache = False

    q = Queue('update', connection=Redis(redis), async=async_)

    for repo in ConfOcean.get_repos():
        enrich_task = q.enqueue(enrich_backend,
                                url, clean,
                                repo['backend_name'], repo['backend_params'],
                                repo['index'], repo['index_enrich'], db_projects_map, db_sortinghat)
        logging.info("Queued job")
        logging.info(enrich_task)
예제 #8
0
def get_elastic():

    try:
        ocean_index = ConfOcean.get_index()
        elastic_ocean = ElasticSearch(args.elastic_url, ocean_index)

    except ElasticConnectException:
        logging.error("Can't connect to Elastic Search. Is it running?")
        sys.exit(1)

    except ElasticWriteException:
        logging.error("Can't write to Elastic Search.")
        sys.exit(1)


    return elastic_ocean
예제 #9
0
def feed_backend(url, clean, fetch_cache, backend_name, backend_params,
                 es_index=None, es_index_enrich=None, project=None):
    """ Feed Ocean with backend data """

    backend = None
    repo = {}    # repository data to be stored in conf
    repo['backend_name'] = backend_name
    repo['backend_params'] = backend_params

    if es_index:
        clean = False  # don't remove index, it could be shared

    if not get_connector_from_name(backend_name):
        raise RuntimeError("Unknown backend %s" % backend_name)
    connector = get_connector_from_name(backend_name)
    klass = connector[3]  # BackendCmd for the connector

    try:
        backend_cmd = klass(*backend_params)

        backend = backend_cmd.backend
        ocean_backend = connector[1](backend, fetch_cache=fetch_cache, project=project)

        logging.info("Feeding Ocean from %s (%s)", backend_name, backend.origin)

        if not es_index:
            es_index = backend_name + "_" + backend.origin
        elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)

        ocean_backend.set_elastic(elastic_ocean)

        ConfOcean.set_elastic(elastic_ocean)

        repo['repo_update_start'] = datetime.now().isoformat()

        # perceval backends fetch params
        offset = None
        from_date = None
        category = None

        signature = inspect.signature(backend.fetch)

        if 'from_date' in signature.parameters:
            from_date = backend_cmd.from_date

        if 'offset' in signature.parameters:
            offset = backend_cmd.offset

        if 'category' in signature.parameters:
            category = backend_cmd.category

        # from_date param support
        if offset and category:
            ocean_backend.feed(from_offset=offset, category=category)
        elif offset:
            ocean_backend.feed(from_offset=offset)
        elif from_date and from_date.replace(tzinfo=None) != parser.parse("1970-01-01"):
            if category:
                ocean_backend.feed(backend_cmd.from_date, category=category)
            else:
                ocean_backend.feed(backend_cmd.from_date)
        elif category:
            ocean_backend.feed(category=category)
        else:
            ocean_backend.feed()

    except Exception as ex:
        if backend:
            logging.error("Error feeding ocean from %s (%s): %s" %
                          (backend_name, backend.origin, ex))
            # this print makes blackbird fails
            traceback.print_exc()
        else:
            logging.error("Error feeding ocean %s" % ex)

        repo['success'] = False
        repo['error'] = str(ex)
    else:
        repo['success'] = True

    repo['repo_update'] = datetime.now().isoformat()
    repo['index'] = es_index
    repo['index_enrich'] = es_index_enrich
    repo['project'] = project

    if es_index:
        unique_id = es_index+"_"+backend.origin
        ConfOcean.add_repo(unique_id, repo)
    else:
        logging.debug("Repository not added to Ocean because errors.")
        logging.debug(backend_params)

    logging.info("Done %s " % (backend_name))
예제 #10
0
def feed_backend(url,
                 clean,
                 fetch_cache,
                 backend_name,
                 backend_params,
                 es_index=None,
                 es_index_enrich=None,
                 project=None):
    """ Feed Ocean with backend data """

    backend = None
    repo = {}  # repository data to be stored in conf
    repo['backend_name'] = backend_name
    repo['backend_params'] = backend_params

    if es_index:
        clean = False  # don't remove index, it could be shared

    if not get_connector_from_name(backend_name):
        raise RuntimeError("Unknown backend %s" % backend_name)
    connector = get_connector_from_name(backend_name)
    klass = connector[3]  # BackendCmd for the connector

    try:
        backend_cmd = klass(*backend_params)

        backend = backend_cmd.backend
        ocean_backend = connector[1](backend,
                                     fetch_cache=fetch_cache,
                                     project=project)

        logging.info("Feeding Ocean from %s (%s)", backend_name,
                     backend.origin)

        if not es_index:
            es_index = backend_name + "_" + backend.origin
        elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)

        ocean_backend.set_elastic(elastic_ocean)

        ConfOcean.set_elastic(elastic_ocean)

        repo['repo_update_start'] = datetime.now().isoformat()

        # offset param suppport
        offset = None

        try:
            offset = backend_cmd.offset
        except AttributeError:
            # The backend does not support offset
            pass

        # from_date param support
        try:
            if offset:
                ocean_backend.feed(offset=offset)
            else:
                if backend_cmd.from_date.replace(tzinfo=None) == \
                    parser.parse("1970-01-01").replace(tzinfo=None):
                    # Don't use the default value
                    ocean_backend.feed()
                else:
                    ocean_backend.feed(backend_cmd.from_date)
        except AttributeError:
            # The backend does not support from_date
            ocean_backend.feed()

    except Exception as ex:
        if backend:
            logging.error("Error feeding ocean from %s (%s): %s" %
                          (backend_name, backend.origin, ex))
            # this print makes blackbird fails
            traceback.print_exc()
        else:
            logging.error("Error feeding ocean %s" % ex)

        repo['success'] = False
        repo['error'] = str(ex)
    else:
        repo['success'] = True

    repo['repo_update'] = datetime.now().isoformat()
    repo['index'] = es_index
    repo['index_enrich'] = es_index_enrich
    repo['project'] = project

    if es_index:
        unique_id = es_index + "_" + backend.origin
        ConfOcean.add_repo(unique_id, repo)
    else:
        logging.debug("Repository not added to Ocean because errors.")
        logging.debug(backend_params)

    logging.info("Done %s " % (backend_name))
예제 #11
0
def feed_backend(url, clean, fetch_cache, backend_name, backend_params,
                 es_index=None, es_index_enrich=None, project=None):
    """ Feed Ocean with backend data """

    backend = None
    repo = {}    # repository data to be stored in conf
    repo['backend_name'] = backend_name
    repo['backend_params'] = backend_params
    if es_index:
        clean = False  # don't remove index, it could be shared


    if not get_connector_from_name(backend_name):
        raise RuntimeError("Unknown backend %s" % backend_name)
    connector = get_connector_from_name(backend_name)
    klass = connector[3]  # BackendCmd for the connector

    try:
        backend_cmd = klass(*backend_params)

        backend = backend_cmd.backend
        ocean_backend = connector[1](backend, fetch_cache=fetch_cache, project=project)

        logging.info("Feeding Ocean from %s (%s)" % (backend_name,
                                                     backend.origin))

        if not es_index:
            es_index = backend_name + "_" + backend.origin
        elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)

        ocean_backend.set_elastic(elastic_ocean)

        ConfOcean.set_elastic(elastic_ocean)

        repo['repo_update_start'] = datetime.now().isoformat()

        try:
            if backend_cmd.from_date.replace(tzinfo=None) == \
                parser.parse("1970-01-01").replace(tzinfo=None):
                # Don't use the default value
                ocean_backend.feed()
            else:
                ocean_backend.feed(backend_cmd.from_date)
        except AttributeError:
            # The backend does not support from_date
            ocean_backend.feed()

    except Exception as ex:
        if backend:
            logging.error("Error feeding ocean from %s (%s): %s" %
                          (backend_name, backend.origin, ex))
            # don't propagete ... it makes blackbird fails
            # TODO: manage it in p2o
            # traceback.print_exc()
        else:
            logging.error("Error feeding ocean %s" % ex)

        repo['success'] = False
        repo['error'] = str(ex)
    else:
        repo['success'] = True

    repo['repo_update'] = datetime.now().isoformat()
    repo['index'] = es_index
    repo['index_enrich'] = es_index_enrich
    repo['project'] = project

    if es_index:
        unique_id = es_index+"_"+backend.origin
        ConfOcean.add_repo(unique_id, repo)
    else:
        logging.debug("Repository not added to Ocean because errors.")
        logging.debug(backend_params)

    logging.info("Done %s " % (backend_name))