def extract_seeds(self, set_specification, latest_update): queryset = self.get_queryset().filter( set_specification=set_specification, since__date__gte=latest_update.date(), status=200, is_extracted=False) oaipmh_objective = { "@": EdurepDataExtraction.get_oaipmh_records, "external_id": EdurepDataExtraction.get_oaipmh_external_id, "state": EdurepDataExtraction.get_oaipmh_record_state } oaipmh_objective.update(EDUREP_EXTRACTION_OBJECTIVE) extract_config = create_config("extract_processor", {"objective": oaipmh_objective}) prc = ExtractProcessor(config=extract_config) results = [] for harvest in queryset: seed_resource = { "resource": f"{harvest._meta.app_label}.{harvest._meta.model_name}", "id": harvest.id, "success": True } try: for seed in prc.extract_from_resource(harvest): seed["seed_resource"] = seed_resource results.append(seed) except ValueError as exc: logger.warning("Invalid XML:", exc, harvest.uri) return results
def extract_seeds(self, latest_update): queryset = self.get_queryset() \ .filter(since__date__gte=latest_update.date(), status=200) metadata_objective = { "@": "$.items", "external_id": "$.uuid", "state": BuasMetadataExtraction.get_record_state } metadata_objective.update(BuasMetadataExtraction.OBJECTIVE) extract_config = create_config("extract_processor", { "objective": metadata_objective }) prc = ExtractProcessor(config=extract_config) results = [] for harvest in queryset: seed_resource = { "resource": f"{harvest._meta.app_label}.{harvest._meta.model_name}", "id": harvest.id, "success": True } for seed in prc.extract_from_resource(harvest): seed["seed_resource"] = seed_resource results.append(seed) return results
def get_edurep_query_seeds(query): queryset = EdurepSearch.objects.filter(request__contains=query) api_objective = { "@": EdurepDataExtraction.get_api_records, "external_id": EdurepDataExtraction.get_api_external_id, "state": EdurepDataExtraction.get_api_record_state } api_objective.update(EDUREP_EXTRACTION_OBJECTIVE) extract_config = create_config("extract_processor", {"objective": api_objective}) prc = ExtractProcessor(config=extract_config) results = [] for search in queryset.filter(status=200): try: results += list(prc.extract_from_resource(search)) except ValueError as exc: err.warning("Invalid XML:", exc, search.uri) seeds = {} for seed in sorted(results, key=lambda rsl: rsl["publisher_date"] or ""): # Some records in Edurep do not have any known URL # As we can't possibly process those we ignore them (silently) # If we want to fix this it should happen on Edurep's or Sharekit's side # We informed Kirsten van Veelo and Martine Teirlinck about the situation. if not seed["url"]: continue # We adjust url's of seeds if the source files are not at the URL # We should improve data extraction to always get source files if seed["mime_type"] == "application/x-Wikiwijs-Arrangement": seed["package_url"] = seed["url"] seed["url"] += "?p=imscp" # And deduplicate entire seeds based on URL seeds[seed["url"]] = seed return seeds.values()
def get_edurep_oaipmh_seeds(set_specification, latest_update, include_deleted=True): queryset = EdurepOAIPMH.objects\ .filter(set_specification=set_specification, since__date__gte=latest_update.date(), status=200) oaipmh_objective = { "@": EdurepDataExtraction.get_oaipmh_records, "external_id": EdurepDataExtraction.get_oaipmh_external_id, "state": EdurepDataExtraction.get_oaipmh_record_state } oaipmh_objective.update(EDUREP_EXTRACTION_OBJECTIVE) extract_config = create_config("extract_processor", {"objective": oaipmh_objective}) prc = ExtractProcessor(config=extract_config) results = [] for harvest in queryset: try: results += list(prc.extract_from_resource(harvest)) except ValueError as exc: err.warning("Invalid XML:", exc, harvest.uri) seeds = [] for seed in results: # Some records in Edurep do not have any known URL # As we can't possibly process those we ignore them (silently) # If we want to fix this it should happen on Edurep's or Sharekit's side # We informed Kirsten van Veelo and Martine Teirlinck about the situation. if seed["state"] == "active" and not seed["url"]: continue # We adjust url's of seeds if the source files are not at the URL # We should improve data extraction to always get source files if seed["mime_type"] == "application/x-Wikiwijs-Arrangement" and seed.get( "url", None): seed["package_url"] = seed["url"] seed["url"] += "?p=imscp" # We deduplicate based on the external_id a UID by Edurep seeds.append(seed) # Now we'll mark any invalid seeds as deleted to make sure they disappear # Invalid seeds have a copyright or are of insufficient education level for seed in seeds: if not seed["copyright"] or seed["copyright"] == "no": seed["state"] = "deleted" if seed["lowest_educational_level"] < 1: # lower level than MBO seed["state"] = "deleted" # And we return the seeds based on whether to include deleted or not return seeds if include_deleted else \ [result for result in seeds if result.get("state", "active") == "active"]