def list_repos_ids():
logging.debug("Listing repos ids")
elastic = get_elastic()
ConfOcean.set_elastic(elastic)
for repo_id in ConfOcean.get_repos_ids():
print(repo_id)
def get_perceval_params(url, index):
logging.info("Get perceval params for index: %s" % (index))
elastic = get_elastic(url, ConfOcean.get_index())
ConfOcean.set_elastic(elastic)
r = requests.get(elastic.index_url+"/repos/"+index)
params = r.json()['_source']['params']
return params
def list_repos():
logging.debug("Listing repos")
elastic = get_elastic()
ConfOcean.set_elastic(elastic)
for repo_id in ConfOcean.get_repos_ids():
elastic = get_elastic()
url = elastic.index_url + "/repos/" + repo_id
r = requests.get(url)
repo = r.json()['_source']
print ("%s %s %s" % (repo_id, repo['repo_update'], repo['success']))
def feed_backends(url, clean, debug = False, redis = None):
''' Update Ocean for all existing backends '''
logging.info("Updating all Ocean")
elastic = get_elastic(url, ConfOcean.get_index(), clean)
ConfOcean.set_elastic(elastic)
fetch_cache = False
q = Queue('update', connection=Redis(redis), async=async_)
for repo in ConfOcean.get_repos():
task_feed = q.enqueue(feed_backend, url, clean, fetch_cache,
repo['backend_name'], repo['backend_params'],
repo['index'], repo['index_enrich'], repo['project'])
logging.info("Queued job")
logging.info(task_feed)
def feed_backends(url, clean, debug = False, redis = None):
''' Update Ocean for all existing backends '''
logging.info("Updating all Ocean")
elastic = get_elastic(url, ConfOcean.get_index(), clean)
ConfOcean.set_elastic(elastic)
fetch_cache = False
q = Queue('update', connection=Redis(redis), async=async_)
for repo in ConfOcean.get_repos():
task_feed = q.enqueue(feed_backend, url, clean, fetch_cache,
repo['backend_name'], repo['backend_params'],
repo['index'], repo['index_enrich'], repo['project'])
logging.info("Queued job")
logging.info(task_feed)
def enrich_backends(url, clean, debug = False, redis = None,
db_projects_map=None, db_sortinghat=None):
''' Enrich all existing indexes '''
logging.info("Enriching repositories")
elastic = get_elastic(url, ConfOcean.get_index(), clean)
ConfOcean.set_elastic(elastic)
fetch_cache = False
q = Queue('update', connection=Redis(redis), async=async_)
for repo in ConfOcean.get_repos():
enrich_task = q.enqueue(enrich_backend,
url, clean,
repo['backend_name'], repo['backend_params'],
repo['index'], repo['index_enrich'], db_projects_map, db_sortinghat)
logging.info("Queued job")
logging.info(enrich_task)
def enrich_backends(url, clean, debug = False, redis = None,
db_projects_map=None, db_sortinghat=None):
''' Enrich all existing indexes '''
logging.info("Enriching repositories")
elastic = get_elastic(url, ConfOcean.get_index(), clean)
ConfOcean.set_elastic(elastic)
fetch_cache = False
q = Queue('update', connection=Redis(redis), async=async_)
for repo in ConfOcean.get_repos():
enrich_task = q.enqueue(enrich_backend,
url, clean,
repo['backend_name'], repo['backend_params'],
repo['index'], repo['index_enrich'], db_projects_map, db_sortinghat)
logging.info("Queued job")
logging.info(enrich_task)
def get_elastic():
try:
ocean_index = ConfOcean.get_index()
elastic_ocean = ElasticSearch(args.elastic_url, ocean_index)
except ElasticConnectException:
logging.error("Can't connect to Elastic Search. Is it running?")
sys.exit(1)
except ElasticWriteException:
logging.error("Can't write to Elastic Search.")
sys.exit(1)
return elastic_ocean
def feed_backend(url, clean, fetch_cache, backend_name, backend_params,
es_index=None, es_index_enrich=None, project=None):
""" Feed Ocean with backend data """
backend = None
repo = {} # repository data to be stored in conf
repo['backend_name'] = backend_name
repo['backend_params'] = backend_params
if es_index:
clean = False # don't remove index, it could be shared
if not get_connector_from_name(backend_name):
raise RuntimeError("Unknown backend %s" % backend_name)
connector = get_connector_from_name(backend_name)
klass = connector[3] # BackendCmd for the connector
try:
backend_cmd = klass(*backend_params)
backend = backend_cmd.backend
ocean_backend = connector[1](backend, fetch_cache=fetch_cache, project=project)
logging.info("Feeding Ocean from %s (%s)", backend_name, backend.origin)
if not es_index:
es_index = backend_name + "_" + backend.origin
elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
ConfOcean.set_elastic(elastic_ocean)
repo['repo_update_start'] = datetime.now().isoformat()
# perceval backends fetch params
offset = None
from_date = None
category = None
signature = inspect.signature(backend.fetch)
if 'from_date' in signature.parameters:
from_date = backend_cmd.from_date
if 'offset' in signature.parameters:
offset = backend_cmd.offset
if 'category' in signature.parameters:
category = backend_cmd.category
# from_date param support
if offset and category:
ocean_backend.feed(from_offset=offset, category=category)
elif offset:
ocean_backend.feed(from_offset=offset)
elif from_date and from_date.replace(tzinfo=None) != parser.parse("1970-01-01"):
if category:
ocean_backend.feed(backend_cmd.from_date, category=category)
else:
ocean_backend.feed(backend_cmd.from_date)
elif category:
ocean_backend.feed(category=category)
else:
ocean_backend.feed()
except Exception as ex:
if backend:
logging.error("Error feeding ocean from %s (%s): %s" %
(backend_name, backend.origin, ex))
# this print makes blackbird fails
traceback.print_exc()
else:
logging.error("Error feeding ocean %s" % ex)
repo['success'] = False
repo['error'] = str(ex)
else:
repo['success'] = True
repo['repo_update'] = datetime.now().isoformat()
repo['index'] = es_index
repo['index_enrich'] = es_index_enrich
repo['project'] = project
if es_index:
unique_id = es_index+"_"+backend.origin
ConfOcean.add_repo(unique_id, repo)
else:
logging.debug("Repository not added to Ocean because errors.")
logging.debug(backend_params)
logging.info("Done %s " % (backend_name))
def feed_backend(url,
clean,
fetch_cache,
backend_name,
backend_params,
es_index=None,
es_index_enrich=None,
project=None):
""" Feed Ocean with backend data """
backend = None
repo = {} # repository data to be stored in conf
repo['backend_name'] = backend_name
repo['backend_params'] = backend_params
if es_index:
clean = False # don't remove index, it could be shared
if not get_connector_from_name(backend_name):
raise RuntimeError("Unknown backend %s" % backend_name)
connector = get_connector_from_name(backend_name)
klass = connector[3] # BackendCmd for the connector
try:
backend_cmd = klass(*backend_params)
backend = backend_cmd.backend
ocean_backend = connector[1](backend,
fetch_cache=fetch_cache,
project=project)
logging.info("Feeding Ocean from %s (%s)", backend_name,
backend.origin)
if not es_index:
es_index = backend_name + "_" + backend.origin
elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
ConfOcean.set_elastic(elastic_ocean)
repo['repo_update_start'] = datetime.now().isoformat()
# offset param suppport
offset = None
try:
offset = backend_cmd.offset
except AttributeError:
# The backend does not support offset
pass
# from_date param support
try:
if offset:
ocean_backend.feed(offset=offset)
else:
if backend_cmd.from_date.replace(tzinfo=None) == \
parser.parse("1970-01-01").replace(tzinfo=None):
# Don't use the default value
ocean_backend.feed()
else:
ocean_backend.feed(backend_cmd.from_date)
except AttributeError:
# The backend does not support from_date
ocean_backend.feed()
except Exception as ex:
if backend:
logging.error("Error feeding ocean from %s (%s): %s" %
(backend_name, backend.origin, ex))
# this print makes blackbird fails
traceback.print_exc()
else:
logging.error("Error feeding ocean %s" % ex)
repo['success'] = False
repo['error'] = str(ex)
else:
repo['success'] = True
repo['repo_update'] = datetime.now().isoformat()
repo['index'] = es_index
repo['index_enrich'] = es_index_enrich
repo['project'] = project
if es_index:
unique_id = es_index + "_" + backend.origin
ConfOcean.add_repo(unique_id, repo)
else:
logging.debug("Repository not added to Ocean because errors.")
logging.debug(backend_params)
logging.info("Done %s " % (backend_name))
def feed_backend(url, clean, fetch_cache, backend_name, backend_params,
es_index=None, es_index_enrich=None, project=None):
""" Feed Ocean with backend data """
backend = None
repo = {} # repository data to be stored in conf
repo['backend_name'] = backend_name
repo['backend_params'] = backend_params
if es_index:
clean = False # don't remove index, it could be shared
if not get_connector_from_name(backend_name):
raise RuntimeError("Unknown backend %s" % backend_name)
connector = get_connector_from_name(backend_name)
klass = connector[3] # BackendCmd for the connector
try:
backend_cmd = klass(*backend_params)
backend = backend_cmd.backend
ocean_backend = connector[1](backend, fetch_cache=fetch_cache, project=project)
logging.info("Feeding Ocean from %s (%s)" % (backend_name,
backend.origin))
if not es_index:
es_index = backend_name + "_" + backend.origin
elastic_ocean = get_elastic(url, es_index, clean, ocean_backend)
ocean_backend.set_elastic(elastic_ocean)
ConfOcean.set_elastic(elastic_ocean)
repo['repo_update_start'] = datetime.now().isoformat()
try:
if backend_cmd.from_date.replace(tzinfo=None) == \
parser.parse("1970-01-01").replace(tzinfo=None):
# Don't use the default value
ocean_backend.feed()
else:
ocean_backend.feed(backend_cmd.from_date)
except AttributeError:
# The backend does not support from_date
ocean_backend.feed()
except Exception as ex:
if backend:
logging.error("Error feeding ocean from %s (%s): %s" %
(backend_name, backend.origin, ex))
# don't propagete ... it makes blackbird fails
# TODO: manage it in p2o
# traceback.print_exc()
else:
logging.error("Error feeding ocean %s" % ex)
repo['success'] = False
repo['error'] = str(ex)
else:
repo['success'] = True
repo['repo_update'] = datetime.now().isoformat()
repo['index'] = es_index
repo['index_enrich'] = es_index_enrich
repo['project'] = project
if es_index:
unique_id = es_index+"_"+backend.origin
ConfOcean.add_repo(unique_id, repo)
else:
logging.debug("Repository not added to Ocean because errors.")
logging.debug(backend_params)
logging.info("Done %s " % (backend_name))