def list_repos_ids(): logging.debug("Listing repos ids") elastic = get_elastic() ConfOcean.set_elastic(elastic) for repo_id in ConfOcean.get_repos_ids(): print(repo_id)
def get_perceval_params(url, index): logging.info("Get perceval params for index: %s" % (index)) elastic = get_elastic(url, ConfOcean.get_index()) ConfOcean.set_elastic(elastic) r = requests.get(elastic.index_url+"/repos/"+index) params = r.json()['_source']['params'] return params
def get_perceval_params(url, index): logging.info("Get perceval params for index: %s" % (index)) elastic = get_elastic(url, ConfOcean.get_index()) ConfOcean.set_elastic(elastic) r = requests.get(elastic.index_url + "/repos/" + index) params = r.json()['_source']['params'] return params
def list_repos(): logging.debug("Listing repos") elastic = get_elastic() ConfOcean.set_elastic(elastic) for repo_id in ConfOcean.get_repos_ids(): elastic = get_elastic() url = elastic.index_url + "/repos/" + repo_id r = requests.get(url) repo = r.json()['_source'] print("%s %s %s" % (repo_id, repo['repo_update'], repo['success']))
def feed_backends(url, clean, debug=False, redis=None): ''' Update Ocean for all existing backends ''' logging.info("Updating all Ocean") elastic = get_elastic(url, ConfOcean.get_index(), clean) ConfOcean.set_elastic(elastic) fetch_cache = False q = Queue('update', connection=Redis(redis), async=async_) for repo in ConfOcean.get_repos(): task_feed = q.enqueue(feed_backend, url, clean, fetch_cache, repo['backend_name'], repo['backend_params'], repo['index'], repo['index_enrich'], repo['project']) logging.info("Queued job") logging.info(task_feed)
def enrich_backends(url, clean, debug=False, redis=None, db_projects_map=None, db_sortinghat=None): ''' Enrich all existing indexes ''' logging.info("Enriching repositories") elastic = get_elastic(url, ConfOcean.get_index(), clean) ConfOcean.set_elastic(elastic) fetch_cache = False q = Queue('update', connection=Redis(redis), async=async_) for repo in ConfOcean.get_repos(): enrich_task = q.enqueue(enrich_backend, url, clean, repo['backend_name'], repo['backend_params'], repo['index'], repo['index_enrich'], db_projects_map, db_sortinghat) logging.info("Queued job") logging.info(enrich_task)