def watch_registry_harvest_task(): """ Check each WATCH_REGISTRY_HARVEST_RATE seconds if new registries need to be harvested. """ try: logger.info('START watching registries.') registries = oai_registry_api.get_all_activated_registry() # We launch the background task for each registry for registry in registries: # If we need to harvest and a task doesn't already exist for this registry. if registry.harvest and not registry.is_queued: harvest_task.apply_async((str(registry.id), )) registry.is_queued = True oai_registry_api.upsert(registry) logger.info( 'Registry {0} has been queued and will be harvested.'. format(registry.name.encode("utf-8"))) logger.info('FINISH watching registries.') except Exception as e: logger.error( 'ERROR : Error while watching new registries to harvest: {0}'. format(e.message)) finally: # Periodic call every WATCH_REGISTRY_HARVEST_RATE seconds watch_registry_harvest_task.apply_async( countdown=WATCH_REGISTRY_HARVEST_RATE)
def watch_registry_harvest_task(): """Check each WATCH_REGISTRY_HARVEST_RATE seconds if new registries need to be harvested.""" from core_oaipmh_harvester_app.components.oai_registry import ( api as oai_registry_api, ) try: logger.info("START watching registries.") registries = oai_registry_api.get_all_activated_registry() # We launch the background task for each registry for registry in registries: # If we need to harvest and a task doesn't already exist for this # registry. if registry.harvest and not registry.is_queued: harvest_task.apply_async((str(registry.id), )) registry.is_queued = True oai_registry_api.upsert(registry) logger.info( f"Registry {registry.name} has been queued and will be " f"harvested.") logger.info("FINISH watching registries.") except Exception as e: logger.error( f"ERROR : Error while watching new registries to harvest: {str(e)}" ) finally: # Periodic call every WATCH_REGISTRY_HARVEST_RATE seconds watch_registry_harvest_task.apply_async( countdown=WATCH_REGISTRY_HARVEST_RATE)
def get_data_source_list_oaipmh(request): """Ajax method to fill the list of data sources. Args: request: Returns: """ try: id_query = request.GET.get("id_query", None) if id_query is not None: # Get query from id query = api_query.get_by_id(id_query, request.user) instance_list = oai_registry_api.get_all_activated_registry( order_by_field="name") item_list = [] for instance_item in instance_list: checked = False # compare instance with existing data source in query # in order to know if they have to be checked for data_source_item in query.data_sources: if ("instance_id" in data_source_item.query_options and data_source_item.query_options["instance_id"] == str(instance_item.id)): checked = True # update the result item list for the context item_list.extend([{ "instance_id": instance_item.id, "instance_name": instance_item.name, "is_checked": checked, }]) # Here, data sources are instances context_params = dict() context_params["instances"] = item_list # return context context = {} context.update(request) context.update(context_params) return render( request, "core_explore_oaipmh_app/user/data_sources/list-content.html", context, ) else: return HttpResponseBadRequest( "Error during loading data sources from oaipmh search.") except AccessControlError: return HttpResponseForbidden() except Exception as e: return HttpResponseBadRequest( "Error during loading data sources from oaipmh search: %s" % escape(str(e)))
def __init__(self): super(RequestForm, self).__init__() self.data_providers = [] self.data_providers.append(('0', 'Pick one')) self.fields['metadata_prefix'].choices = self.data_providers self.fields['set'].choices = self.data_providers for o in oai_registry_api.get_all_activated_registry(): self.data_providers.append((str(o.id)+'|'+o.url, str(o.name))) self.fields['data_provider'].choices = self.data_providers
def build_query(self, query, templates, registries): """Build the raw query. Args: query: templates: registries: Returns: The raw query """ # build query builder query_builder = OaiPmhQueryBuilder(query, self.sub_document_root) if type(templates) is str: templates = json.loads(templates) if type(registries) is str: registries = json.loads(registries) # if registries, check if activated list_activated_registry = ( oai_registry_api.get_all_activated_registry().values_list("id")) if len(registries) > 0: activated_registries = [ str(id_) for id_ in registries if ObjectId(id_) in list_activated_registry ] else: activated_registries = list_activated_registry if len(templates) > 0: # get list of template ids list_template_ids = [template["id"] for template in templates] # get all metadata formats used by the registries list_metadata_format = ( oai_harvester_metadata_format_api.get_all_by_list_registry_ids( activated_registries)) # Filter metadata formats that use the given templates list_metadata_formats_id = [ str(x.id) for x in list_metadata_format if x.template is not None and str(x.template.id) in list_template_ids ] query_builder.add_list_metadata_formats_criteria( list_metadata_formats_id) else: # Only activated registries query_builder.add_list_registries_criteria(activated_registries) # do not include deleted records query_builder.add_not_deleted_criteria() # create a raw query return query_builder.get_raw_query()
def test_get_all_contains_only_oai_registry(self, mock_get_all): """ Args: mock_get_all: Returns: """ _generic_get_all_test(self, mock_get_all, registry_api.get_all_activated_registry())
def __init__(self): super(RequestForm, self).__init__() default_fields = [("0", "Pick one")] self.fields["metadata_prefix"].choices = default_fields self.fields["set"].choices = default_fields for registry in oai_registry_api.get_all_activated_registry(): default_fields.append(("%s|%s" % (str(registry.id), registry.url), str(registry.name))) self.fields["data_provider"].choices = default_fields
def init_harvest(): """ Init harvest process. """ # Revoke all scheduled tasks _revoke_all_scheduled_tasks() # Init all registry is_queued to False in case of a server reboot after an issue. registries = oai_registry_api.get_all_activated_registry() for registry in registries: registry.is_queued = False oai_registry_api.upsert(registry) # Watch Registries watch_registry_harvest_task.apply_async()
def get_data_source_list_oaipmh(request): """ Ajax method to fill the list of data sources. Args: request: Returns: """ try: id_query = request.GET.get('id_query', None) if id_query is not None: # Get query from id query = api_query.get_by_id(id_query) instance_list = oai_registry_api.get_all_activated_registry(order_by_field='name') item_list = [] url_instance = request.build_absolute_uri(reverse("core_explore_oaipmh_rest_execute_query")) for instance_item in instance_list: checked = False # compare instance with existing data source in query # in order to know if they have to be checked for data_source_item in query.data_sources: if data_source_item.name == instance_item.name\ and data_source_item.url_query == url_instance: checked = True break # update the result item list for the context item_list.extend([{'instance_id': instance_item.id, 'instance_name': instance_item.name, 'is_checked': checked}]) # Here, data sources are instances context_params = dict() context_params['instances'] = item_list # return context context = {} context.update(request) context.update(context_params) return render(request, 'core_explore_oaipmh_app/user/data_sources/list-content.html', context) else: return HttpResponseBadRequest("Error during loading data sources from oaipmh search.") except Exception as e: return HttpResponseBadRequest("Error during loading data sources from oaipmh search: %s" % e.message)