def extract_seeds(self, set_specification, latest_update):
queryset = self.get_queryset().filter(
set_specification=set_specification,
since__date__gte=latest_update.date(),
status=200,
is_extracted=False)
oaipmh_objective = {
"@": EdurepDataExtraction.get_oaipmh_records,
"external_id": EdurepDataExtraction.get_oaipmh_external_id,
"state": EdurepDataExtraction.get_oaipmh_record_state
}
oaipmh_objective.update(EDUREP_EXTRACTION_OBJECTIVE)
extract_config = create_config("extract_processor",
{"objective": oaipmh_objective})
prc = ExtractProcessor(config=extract_config)
results = []
for harvest in queryset:
seed_resource = {
"resource":
f"{harvest._meta.app_label}.{harvest._meta.model_name}",
"id": harvest.id,
"success": True
}
try:
for seed in prc.extract_from_resource(harvest):
seed["seed_resource"] = seed_resource
results.append(seed)
except ValueError as exc:
logger.warning("Invalid XML:", exc, harvest.uri)
return results
def extract_seeds(self, latest_update):
queryset = self.get_queryset() \
.filter(since__date__gte=latest_update.date(), status=200)
metadata_objective = {
"@": "$.items",
"external_id": "$.uuid",
"state": BuasMetadataExtraction.get_record_state
}
metadata_objective.update(BuasMetadataExtraction.OBJECTIVE)
extract_config = create_config("extract_processor", {
"objective": metadata_objective
})
prc = ExtractProcessor(config=extract_config)
results = []
for harvest in queryset:
seed_resource = {
"resource": f"{harvest._meta.app_label}.{harvest._meta.model_name}",
"id": harvest.id,
"success": True
}
for seed in prc.extract_from_resource(harvest):
seed["seed_resource"] = seed_resource
results.append(seed)
return results
def get_edurep_query_seeds(query):
queryset = EdurepSearch.objects.filter(request__contains=query)
api_objective = {
"@": EdurepDataExtraction.get_api_records,
"external_id": EdurepDataExtraction.get_api_external_id,
"state": EdurepDataExtraction.get_api_record_state
}
api_objective.update(EDUREP_EXTRACTION_OBJECTIVE)
extract_config = create_config("extract_processor",
{"objective": api_objective})
prc = ExtractProcessor(config=extract_config)
results = []
for search in queryset.filter(status=200):
try:
results += list(prc.extract_from_resource(search))
except ValueError as exc:
err.warning("Invalid XML:", exc, search.uri)
seeds = {}
for seed in sorted(results, key=lambda rsl: rsl["publisher_date"] or ""):
# Some records in Edurep do not have any known URL
# As we can't possibly process those we ignore them (silently)
# If we want to fix this it should happen on Edurep's or Sharekit's side
# We informed Kirsten van Veelo and Martine Teirlinck about the situation.
if not seed["url"]:
continue
# We adjust url's of seeds if the source files are not at the URL
# We should improve data extraction to always get source files
if seed["mime_type"] == "application/x-Wikiwijs-Arrangement":
seed["package_url"] = seed["url"]
seed["url"] += "?p=imscp"
# And deduplicate entire seeds based on URL
seeds[seed["url"]] = seed
return seeds.values()
def get_edurep_oaipmh_seeds(set_specification,
latest_update,
include_deleted=True):
queryset = EdurepOAIPMH.objects\
.filter(set_specification=set_specification, since__date__gte=latest_update.date(), status=200)
oaipmh_objective = {
"@": EdurepDataExtraction.get_oaipmh_records,
"external_id": EdurepDataExtraction.get_oaipmh_external_id,
"state": EdurepDataExtraction.get_oaipmh_record_state
}
oaipmh_objective.update(EDUREP_EXTRACTION_OBJECTIVE)
extract_config = create_config("extract_processor",
{"objective": oaipmh_objective})
prc = ExtractProcessor(config=extract_config)
results = []
for harvest in queryset:
try:
results += list(prc.extract_from_resource(harvest))
except ValueError as exc:
err.warning("Invalid XML:", exc, harvest.uri)
seeds = []
for seed in results:
# Some records in Edurep do not have any known URL
# As we can't possibly process those we ignore them (silently)
# If we want to fix this it should happen on Edurep's or Sharekit's side
# We informed Kirsten van Veelo and Martine Teirlinck about the situation.
if seed["state"] == "active" and not seed["url"]:
continue
# We adjust url's of seeds if the source files are not at the URL
# We should improve data extraction to always get source files
if seed["mime_type"] == "application/x-Wikiwijs-Arrangement" and seed.get(
"url", None):
seed["package_url"] = seed["url"]
seed["url"] += "?p=imscp"
# We deduplicate based on the external_id a UID by Edurep
seeds.append(seed)
# Now we'll mark any invalid seeds as deleted to make sure they disappear
# Invalid seeds have a copyright or are of insufficient education level
for seed in seeds:
if not seed["copyright"] or seed["copyright"] == "no":
seed["state"] = "deleted"
if seed["lowest_educational_level"] < 1: # lower level than MBO
seed["state"] = "deleted"
# And we return the seeds based on whether to include deleted or not
return seeds if include_deleted else \
[result for result in seeds if result.get("state", "active") == "active"]