def generateIcesXml(self, logrow_id,
datatype_list, year_from, year_to, status, user):
""" """
# Load resource content to translate from DynTaxa to WoRMS.
self._translate_taxa = TranslateTaxa()
self._translate_taxa.loadTranslateTaxa('translate_dyntaxa_to_worms')
# Create target directory if not exists.
if not os.path.exists( self._export_dir_path):
os.makedirs( self._export_dir_path)
#
error_counter = 0
#
year_int = int(year_from)
year_to_int = int(year_to)
#
for datatype in datatype_list:
while year_int <= year_to_int:
""
self.generateOneIcesXml(logrow_id, error_counter,
datatype, unicode(year_int), status, user)
#
year_int += 1
#
if error_counter > 0:
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED (Errors: ' + unicode(error_counter) + ')')
else:
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
if settings.DEBUG: print('DEBUG: ICES-XML generation FINISHED')
def cleanUpSpeciesObs(self, log_row_id):
""" """
# # Remove deleted rows when there is no more need for them from the consumer side (SLU).
# speciesobs_models.SpeciesObs.objects.filter(status='DELETED').delete()
# Deletes all rows.
speciesobs_models.SpeciesObs.objects.all().delete()
#
admin_models.changeLogRowStatus(log_row_id, status = 'FINISHED')
def deleteExportFiles(request):
""" Deletes rows in the database. """
error_message = None
#
if request.method == "GET":
#
form = forms.DeleteExportFilesForm()
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("exportfiles_delete_all.html", contextinstance)
elif request.method == "POST":
#
form = forms.DeleteExportFilesForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = 'Not a valid user or password. Please try again...'
#
# if error_message == None:
# if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'):
# logrow_id = admin_models.createLogRow(command = 'Delete all datasets from FTP', status = 'RUNNING', user = user)
# try:
# error_message = dataset_utils.DatasetUtils().deleteAllFilesFromFtp()
# admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
# except:
# error_message = u"Can't delete datasets from the FTP area."
# admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
# admin_models.addResultLog(logrow_id, result_log = error_message)
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = 'Delete all ICES-XML files from DB', status = 'RUNNING', user = user)
try:
exportformats_models.ExportFiles.objects.all().delete()
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
except:
error_message = u"Can't delete datasets from the database."
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/sharkdataadmin")
#
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("exportfiles_delete_all.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/sharkdataadmin")
def validateIcesXml(self, logrow_id, datatype_list, user):
""" """
error_counter = 0
#
db_exports = export_models.ExportFiles.objects.all()
for db_export in db_exports:
if db_export.datatype in datatype_list:
#
error_counter = self.validateOneIcesXml(logrow_id, error_counter, db_export, user)
#
if error_counter > 0:
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED (Errors: ' + unicode(error_counter) + ')')
else:
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
if settings.DEBUG: print('DEBUG: ICES-XML generation FINISHED')
def deleteResource(request, resource_id):
""" Deletes one row in the database. The FTP area is not affected. """
resource = models.Resources.objects.get(id=resource_id)
#
if request.method == "GET":
form = forms.DeleteResourceForm()
contextinstance = {'form' : form,
'resource' : resource,
'error_message' : None}
contextinstance.update(csrf(request))
return render_to_response("delete_resource.html", contextinstance)
elif request.method == "POST":
# Reloads db-stored data.
resources_utils.ResourcesUtils().clear()
#
error_message = None # initially.
#
form = forms.DeleteResourceForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
if user not in settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.keys():
error_message = u'Not a valid user. Please try again...'
#
if error_message == None:
if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'):
logrow_id = admin_models.createLogRow(command = u'Delete resource (FTP)', status = u'RUNNING', user = user)
try:
resources_utils.ResourcesUtils().deleteFileFromFtp(resource.resource_file_name)
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't delete resources from the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = u'Delete resource (DB)', status = u'RUNNING', user = user)
try:
resource = models.Resources.objects.get(id=resource_id)
resource.delete()
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't delete resource from the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/resources")
#
contextinstance = {'form' : form,
'resource' : resource,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("delete_resource.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/resources")
def deleteResources(request):
""" Deletes all rows in the database. The FTP area is not affected. """
error_message = None # initially.
#
if request.method == "GET":
#
form = forms.DeleteAllResourcesForm()
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("resources_delete_all.html", contextinstance)
elif request.method == "POST":
# Reloads db-stored data.
resources_utils.ResourcesUtils().clear()
#
error_message = None # initially.
#
form = forms.DeleteAllResourcesForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = u'Not a valid user or password. Please try again...'
#
if error_message == None:
if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'):
logrow_id = admin_models.createLogRow(command = u'Delete all resources (FTP)', status = u'RUNNING', user = user)
try:
resources_utils.ResourcesUtils().deleteAllFilesFromFtp()
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't delete resources from the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = u'Delete all resources (DB)', status = u'RUNNING', user = user)
try:
resources_models.Resources.objects.all().delete()
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't delete resources from the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/sharkdataadmin")
#
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("resources_delete_all.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/sharkdataadmin")
def loadAllResources(request):
""" Updates the database from resources stored in the FTP area.
"""
error_message = None # initially.
#
if request.method == "GET":
form = forms.LoadAllResourcesForm()
contextinstance = {'form': form,
'error_message': error_message}
contextinstance.update(csrf(request))
return render_to_response("resources_load_all.html", contextinstance)
elif request.method == "POST":
# Reloads db-stored data.
resources_utils.ResourcesUtils().clear()
#
error_message = None # initially.
#
form = forms.LoadAllResourcesForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = u'Not a valid user or password. Please try again...'
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = u'Load all resources from FTP to DB', status = u'RUNNING', user = user)
try:
resources_utils.ResourcesUtils().writeResourcesInfoToDb(user)
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't load resources and save to the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/sharkdataadmin")
#
contextinstance = {'form': form,
'error_message': error_message}
contextinstance.update(csrf(request))
return render_to_response("resources_load_all.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/sharkdataadmin")
def deleteExportFile(request, export_name):
""" Deletes one row in the database. """
exportfile = models.ExportFiles.objects.get(export_name = export_name)
#
if request.method == "GET":
form = forms.DeleteExportFileForm()
contextinstance = {'form' : form,
'exportfile' : exportfile,
'error_message' : None}
contextinstance.update(csrf(request))
return render_to_response("delete_exportfile.html", contextinstance)
elif request.method == "POST":
error_message = None # initially.
#
form = forms.DeleteExportFileForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = 'Not a valid user or password. Please try again...'
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = 'Delete exportfile (DB)', status = 'RUNNING', user = user)
try:
exportfile = models.ExportFiles.objects.get(export_name = export_name)
exportfile.delete()
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
except:
error_message = u"Can't delete exportfile from the database."
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/exportformats")
#
contextinstance = {'form' : form,
'exportfile' : exportfile,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("delete_exportfile.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/exportformats")
def cleanUpSpeciesObs(request):
""" Removes species observations with status='DELETED'. """
error_message = None # initially.
#
if request.method == "GET":
#
form = forms.CleanUpSpeciesObsForm()
contextinstance = {'form': form,
'error_message': error_message}
contextinstance.update(csrf(request))
return render_to_response("speciesobs_cleanup.html", contextinstance)
elif request.method == "POST":
form = forms.CleanUpSpeciesObsForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = u'Not a valid user or password. Please try again...'
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = u'Clean up species observations', status = u'RUNNING', user = user)
try:
error_message = speciesobs_utils.SpeciesObsUtils().cleanUpSpeciesObsInThread(logrow_id)
# admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except:
error_message = u"Can't clean up species observations."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/sharkdataadmin")
#
contextinstance = {'form': form,
'error_message': error_message}
contextinstance.update(csrf(request))
return render_to_response("speciesobs_cleanup.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/sharkdataadmin")
def generateArchivesForAllDatasetsInThread(self, user):
""" """
logrow_id = admin_models.createLogRow(command = u'Generate archives (DwC, etc.)', status = u'RUNNING', user = user)
try:
# Check if generate_archives thread is running.
if self._generate_archives_thread:
if self._generate_archives_thread.is_alive():
error_message = u"Generate archives is already running. Please try again later."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
return
# Use a thread to relese the user. This will take some time.
self._generate_archives_thread = threading.Thread(target = self.generateArchivesForAllDatasets, args=(logrow_id, user, ))
self._generate_archives_thread.start()
except:
error_message = u"Can't generate_archives."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
return None # No error message.
def validateIcesXmlInThread(self, datatype_list, user):
""" """
logrow_id = admin_models.createLogRow(command = 'Validate ICES-XML file.', status = 'RUNNING', user = user)
try:
# Check if thread is running.
if self._validate_ices_xml_thread:
if self._validate_ices_xml_thread.is_alive():
error_message = u"Validate ICES-XML file is already running. Please try again later."
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
return
# Use a thread to relese the user.
self._validate_ices_xml_thread = threading.Thread(target = sharkdata_core.ValidateIcesXml().validateIcesXml,
args=(logrow_id, datatype_list, user ))
self._validate_ices_xml_thread.start()
except Exception as e:
error_message = u"Can't validate ICES-XML file." + '\nException: ' + unicode(e) + '\n'
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
#
return None # No error message.
def loadAllDatasets(request):
""" Updates the database from datasets stored in the FTP area.
I multiple versions of a dataset are in the FTP area only the latest
will be loaded.
"""
error_message = None # initially.
#
if request.method == "GET":
form = forms.LoadAllDatasetsForm()
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("datasets_load_all.html", contextinstance)
elif request.method == "POST":
error_message = None # initially.
#
form = forms.LoadAllDatasetsForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = u'Not a valid user or password. Please try again...'
#
#
#
if error_message == None:
logrow_id = admin_models.createLogRow(command = u'Load all datasets from FTP to DB', status = u'RUNNING', user = user)
try:
error_counter = dataset_utils.DatasetUtils().writeLatestDatasetsInfoToDb(logrow_id, user)
if error_counter > 0:
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')')
else:
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
except Exception as e:
error_message = u"Can't load datasets and save to the database."
admin_models.changeLogRowStatus(logrow_id, status = u'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
if settings.DEBUG: print(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n')
settings.LOGGER.error(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n')
#
# if error_message == None:
# if ('update_metadata' in request.POST) and (request.POST['update_metadata'] == u'on'):
# dataset_utils.DatasetUtils().updateMetadataForAllDatasetsInThread(user)
#
# if error_message == None:
# if ('generate_archives' in request.POST) and (request.POST['generate_archives'] == u'on'):
# dataset_utils.DatasetUtils().generateArchivesForAllDatasetsInThread(user)
# OK.
if error_message == None:
return HttpResponseRedirect("/sharkdataadmin")
#
contextinstance = {'form' : form,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("datasets_load_all.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/sharkdataadmin")
def generateIcesXmlInThread(self, datatype_list, year_from, year_to, status, user):
""" """
logrow_id = admin_models.createLogRow(command="Generate ICES-XML file.", status="RUNNING", user=user)
try:
# Check if thread is running.
if self._generate_ices_xml_thread:
if self._generate_ices_xml_thread.is_alive():
error_message = "Generate ICES-XML file is already running. Please try again later."
admin_models.changeLogRowStatus(logrow_id, status="FAILED")
admin_models.addResultLog(logrow_id, result_log=error_message)
#
return
# Use a thread to relese the user.
self._generate_ices_xml_thread = threading.Thread(
target=sharkdata_core.GenerateIcesXml().generateIcesXml,
args=(logrow_id, datatype_list, year_from, year_to, status, user),
)
self._generate_ices_xml_thread.start()
except Exception as e:
error_message = "Can't generate ICES-XML file." + "\nException: " + unicode(e) + "\n"
admin_models.changeLogRowStatus(logrow_id, status="FAILED")
admin_models.addResultLog(logrow_id, result_log=error_message)
#
return None # No error message.
def deleteDataset(request, dataset_id):
""" Deletes one row in the database. """
dataset = models.Datasets.objects.get(id=dataset_id)
#
if request.method == "GET":
form = forms.DeleteDatasetForm()
contextinstance = {'form' : form,
'dataset' : dataset,
'error_message' : None}
contextinstance.update(csrf(request))
return render_to_response("delete_dataset.html", contextinstance)
elif request.method == "POST":
error_message = None # initially.
#
form = forms.DeleteDatasetForm(request.POST)
if form.is_valid():
#
user = request.POST['user']
password = request.POST['password']
if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None):
error_message = 'Not a valid user or password. Please try again...'
#
if error_message == None:
if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'):
# Delete the marked dataset version. Earlier versions vill be used, if there are any.
logrow_id = admin_models.createLogRow(command = 'Delete dataset (FTP)', status = 'RUNNING', user = user)
try:
error_message = sharkdata_core.DatasetUtils().deleteFileFromFtp(dataset.dataset_file_name)
error_message = sharkdata_core.DatasetUtils().writeLatestDatasetsInfoToDb(user)
admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED')
except:
error_message = u"Can't delete dataset from the ftp."
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
else:
logrow_id = admin_models.createLogRow(command = 'Delete dataset (DB)', status = 'RUNNING', user = user)
try:
dataset = models.Datasets.objects.get(id=dataset_id)
dataset.delete()
except:
error_message = u"Can't delete dataset from the database."
admin_models.changeLogRowStatus(logrow_id, status = 'FAILED')
admin_models.addResultLog(logrow_id, result_log = error_message)
# OK.
if error_message == None:
return HttpResponseRedirect("/datasets")
#
contextinstance = {'form' : form,
'dataset' : dataset,
'error_message' : error_message}
contextinstance.update(csrf(request))
return render_to_response("delete_dataset.html", contextinstance)
# Not a valid request method.
return HttpResponseRedirect("/datasets")
def generateArchivesForAllDatasets(self, logrow_id, user):
""" """
# Load resource file containing WoRMS info for taxa.
worms_info_object = misc_utils.SpeciesWormsInfo()
worms_info_object.loadSpeciesFromResource()
#
error_counter = 0
datasets = models.Datasets.objects.all()
for dataset in datasets:
zip_file_name = dataset.dataset_file_name
#
admin_models.addResultLog(logrow_id, result_log = u'Generating archive file for: ' + zip_file_name + u'...')
if settings.DEBUG:
print(u'DEBUG: ===== Processing: ' + zip_file_name)
#
archive = None
dwca_config_dir = u''
if zip_file_name.startswith(u'SHARK_Zooplankton'):
archive = dwca_eurobis_zooplankton.DwcaEurObisZooplankton()
elif zip_file_name.startswith(u'SHARK_Phytoplankton'):
archive = dwca_eurobis_phytoplankton.DwcaEurObisPhytoplankton()
elif zip_file_name.startswith(u'SHARK_Zoobenthos'):
archive = dwca_eurobis_zoobenthos.DwcaEurObisZoobenthos()
elif zip_file_name.startswith(u'SHARK_Bacterioplankton'):
archive = dwca_eurobis_bacterioplankton.DwcaEurObisBacterioplankton()
#
if not archive:
continue # Skip if other datatypes.
# === Test for GBIF-Occurrence, GBIF-EurOBIS (EMODnet-Bio) and GBIF for Sample Data. ===
try:
dataset = misc_utils.Dataset()
dataset.loadDataFromZipFile(zip_file_name,
dataset_dir_path = self._ftp_dir_path,
encoding = u'cp1252')
# # === Test for GBIF-Occurrence. ===
# try:
# admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive.')
# archive = biological_data_exchange_util.DarwinCoreArchiveFormat(
# datatype,
# u'settings_dwca.json',
# dwca_config_dir,
# meta_file_name = u'meta.xml',
# eml_file_name = u'eml.xml',
# worms_info_object = worms_info_object)
# archive.createArchiveParts(dataset)
# # Save generated archive file.
# generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives')
# achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A.zip')
# if not os.path.exists(generated_archives_path):
# os.makedirs(generated_archives_path)
# archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path,
# settings_dir_path = dwca_config_dir)
# # Update database.
# db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name)
# db_dataset.dwc_archive_available = True
# db_dataset.dwc_archive_file_path = os.path.join(generated_archives_path, achive_file_name)
# db_dataset.save()
# except Exception as e:
# error_counter += 1
# admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A from: ' + zip_file_name + u'.')
# === Test for GBIF-EurOBIS (EMODnet-Bio). ===
try:
admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (EurOBIS format).')
# archive = biological_data_exchange_util.DarwinCoreArchiveFormatForEurObis(
# datatype,
# u'settings_dwca_eurobis.json',
# dwca_config_dir,
# meta_file_name = u'meta_eurobis.xml',
# eml_file_name = u'eml_eurobis.xml',
# worms_info_object = worms_info_object)
# Update database before.
db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name)
db_dataset.dwc_archive_eurobis_available = False
db_dataset.dwc_archive_eurobis_file_path = u''
db_dataset.save()
if worms_info_object:
archive.setWormsInfoObject(worms_info_object)
#
archive.createArchiveParts(dataset)
# Save generated archive file.
generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives')
achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-EurOBIS.zip')
if not os.path.exists(generated_archives_path):
os.makedirs(generated_archives_path)
archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path,
settings_dir_path = dwca_config_dir)
# Update database after.
db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name)
db_dataset.dwc_archive_eurobis_available = True
db_dataset.dwc_archive_eurobis_file_path = os.path.join(generated_archives_path, achive_file_name)
db_dataset.save()
except Exception as e:
error_counter += 1
print(e)
admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (eurOBIS format) from: ' + zip_file_name + u'.')
# # === Test for GBIF for Sample Data. ===
# try:
# admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (Sample data format).')
# archive = biological_data_exchange_util.DarwinCoreArchiveFormatForSampleData(
# datatype,
# u'settings_dwca_sampledata.json',
# dwca_config_dir,
# meta_file_name = u'meta_sampledata.xml',
# eml_file_name = u'eml_sampledata.xml',
# worms_info_object = worms_info_object)
# archive.createArchiveParts(dataset)
# # Save generated archive file.
# generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives')
# achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-SampleData.zip')
# if not os.path.exists(generated_archives_path):
# os.makedirs(generated_archives_path)
# archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path,
# settings_dir_path = dwca_config_dir)
# # Update database.
# db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name)
# db_dataset.dwc_archive_sampledata_available = True
# db_dataset.dwc_archive_sampledata_file_path = os.path.join(generated_archives_path, achive_file_name)
# db_dataset.save()
# except Exception as e:
# error_counter += 1
# admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (Sample data format) from: ' + zip_file_name + u'.')
except Exception as e:
error_counter += 1
traceback.print_exc()
admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate archive files from: ' + zip_file_name + u'.')
#
if error_counter > 0:
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')')
else:
admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED')
if settings.DEBUG:
print(u'DEBUG: Archive generation FINISHED')
def updateSpeciesObs(self, log_row_id):
""" """
#
print('Species observations update. Started.')
try:
# Load resource file containing WoRMS info for taxa.
worms_info_object = sharkdata_core.SpeciesWormsInfo()
worms_info_object.loadSpeciesFromResource()
# Mark all rows in db table before update starts.
speciesobs_models.SpeciesObs.objects.all().update(last_update_date = '0000-00-00')
# Loop over all datasets.
valid_datatypes = [
'Epibenthos',
'GreySeal',
'HarbourSeal',
'Phytoplankton',
'RingedSeal',
'Zoobenthos',
'Zooplankton',
###'Speciesobs',
]
#
for dataset_queryset in datasets_models.Datasets.objects.all():
if dataset_queryset.datatype in valid_datatypes:
print('Loading data from: ' + dataset_queryset.ftp_file_path)
else:
print('Skipped (wrong datatype): ' + dataset_queryset.ftp_file_path)
continue
#
admin_models.addResultLog(log_row_id, result_log = 'Extracting species obs from: ' + dataset_queryset.dataset_file_name + '...')
#
zipreader = sharkdata_core.SharkArchiveFileReader(dataset_queryset.ftp_file_path)
try:
zipreader.open()
data = zipreader.getDataAsText()
finally:
zipreader.close()
#
encoding = 'cp1252'
rowseparator = '\n'
fieldseparator = '\t'
#
data = unicode(data, encoding, 'strict')
datarows = (item.strip() for item in data.split(rowseparator)) # Generator instead of list.
#
for rowindex, datarow in enumerate(datarows):
#
try:
if len(datarow) == 0:
continue
#
row = [item.strip() for item in datarow.split(fieldseparator)]
if rowindex == 0:
header = row
else:
header = self.cleanUpHeader(header)
rowdict = dict(zip(header, row))
# Check if position is valid. Skip row if not.
# lat_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.')
# long_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.')
lat_dd = rowdict.get('latitude_dd', '').replace(',', '.')
long_dd = rowdict.get('longitude_dd', '').replace(',', '.')
# if self.isFloat(lat_dd) and self.isFloat(long_dd):
if True:
if (float(lat_dd) > 70.0) or (float(lat_dd) < 50.0) or (float(long_dd) > 25.0) or (float(long_dd) < 5.0):
# Don't add to SpeciesObs if lat_dd/long_dd is outside the box.
print('Row skipped, position outside box. Latitude: ' + lat_dd + ' Longitude: ' + long_dd + ' Row: ' + unicode(rowindex))
continue
else:
# Don't add to SpeciesObs if lat_dd/long_dd is invalid.
continue
#
tmp_date = rowdict.get('sampling_date', '')
tmp_year = ''
tmp_month = ''
tmp_day = ''
if len(tmp_date) >= 10:
tmp_year = tmp_date[0:4]
tmp_month = tmp_date[5:7]
tmp_day = tmp_date[8:10]
scientificname = rowdict.get('scientific_name', '-') if rowdict.get('scientific_name') else '-'
scientificauthority = rowdict.get('scientific_authority', '-') if rowdict.get('scientific_authority') else '-'
taxon_worms_info = worms_info_object.getTaxonInfoDict(scientificname)
if taxon_worms_info:
taxonkingdom = taxon_worms_info.get('kingdom', '-') if taxon_worms_info.get('kingdom') else '-'
taxonphylum = taxon_worms_info.get('phylum', '-') if taxon_worms_info.get('phylum') else '-'
taxonclass = taxon_worms_info.get('class', '-') if taxon_worms_info.get('class') else '-'
taxonorder = taxon_worms_info.get('order', '-') if taxon_worms_info.get('order') else '-'
taxonfamily = taxon_worms_info.get('family', '-') if taxon_worms_info.get('family') else '-'
taxongenus = taxon_worms_info.get('genus', '-') if taxon_worms_info.get('genus') else '-'
else:
taxonkingdom = '-'
taxonphylum = '-'
taxonclass = '-'
taxonorder = '-'
taxonfamily = '-'
taxongenus = '-'
speciesobs = speciesobs_models.SpeciesObs(
data_type = rowdict.get('data_type', ''),
scientific_name = scientificname,
scientific_authority = scientificauthority,
# latitude_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.'),
# longitude_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.'),
latitude_dd = rowdict.get('latitude_dd', '').replace(',', '.'),
longitude_dd = rowdict.get('longitude_dd', '').replace(',', '.'),
sampling_date = rowdict.get('sampling_date', ''),
sampling_year = tmp_year,
sampling_month = tmp_month,
sampling_day = tmp_day,
sample_min_depth = rowdict.get('sample_min_depth', ''),
sample_max_depth = rowdict.get('sample_max_depth', ''),
sampler_type = rowdict.get('sampler_type', ''),
dyntaxa_id = rowdict.get('dyntaxa_id', '') if rowdict.get('dyntaxa_id') else '-',
# taxon_kingdom = rowdict.get('taxon_kingdom', '-') if rowdict.get('taxon_kingdom') else '-',
# taxon_phylum = rowdict.get('taxon_phylum', '-') if rowdict.get('taxon_phylum') else '-',
# taxon_class = rowdict.get('taxon_class', '-') if rowdict.get('taxon_class') else '-',
# taxon_order = rowdict.get('taxon_order', '-') if rowdict.get('taxon_order') else '-',
# taxon_family = rowdict.get('taxon_family', '-') if rowdict.get('taxon_family') else '-',
# taxon_genus = rowdict.get('taxon_genus', '-') if rowdict.get('taxon_genus') else '-',
# taxon_species = rowdict.get('taxon_species', '-') if rowdict.get('taxon_species') else '-',
taxon_kingdom = taxonkingdom,
taxon_phylum = taxonphylum,
taxon_class = taxonclass,
taxon_order = taxonorder,
taxon_family = taxonfamily,
taxon_genus = taxongenus,
# taxon_species = rowdict.get('species', '-') if rowdict.get('species') else '-',
orderer = rowdict.get('orderer', '') if rowdict.get('orderer') else '-',
reporting_institute = rowdict.get('reporting_institute', '') if rowdict.get('reporting_institute') else '-',
sampling_laboratory = rowdict.get('sampling_laboratory', '') if rowdict.get('sampling_laboratory') else '-',
analytical_laboratory = rowdict.get('analytical_laboratory', '') if rowdict.get('analytical_laboratory') else '-',
#
occurrence_id = '', # Added below.
#
dataset_name = unicode(dataset_queryset.dataset_name),
dataset_file_name = unicode(dataset_queryset.dataset_file_name),
# ##### Example: 'POINT(-73.9869510 40.7560540)', Note: Order longitude - latitude.
# geometry = 'POINT(' + rowdict.get('longitude_dd', '0.0').replace(',', '.') + ' ' + rowdict.get('latitude_dd', '0.0').replace(',', '.') + ')',
)
# Calculate DarwinCore Observation Id.
generated_occurrence_id = speciesobs.calculateDarwinCoreObservationIdAsMD5()
#
# if speciesobs_models.SpeciesObs(
# speciesobs.save()
#speciesobs_models.SpeciesObs.objects.filter(status='DELETED').delete()
try:
obs_existing_row = speciesobs_models.SpeciesObs.objects.get(occurrence_id = generated_occurrence_id)
except:
obs_existing_row = None # Does not exist.
#
current_date = unicode(time.strftime('%Y-%m-%d'))
#
if obs_existing_row:
if obs_existing_row.status == 'ACTIVE':
obs_existing_row.last_update_date = current_date
else:
obs_existing_row.status= 'ACTIVE'
obs_existing_row.last_update_date = current_date
obs_existing_row.last_status_change_date = current_date
#
obs_existing_row.save()
else:
speciesobs.occurrence_id = generated_occurrence_id
speciesobs.status = 'ACTIVE'
speciesobs.last_update_date = current_date
speciesobs.last_status_change_date = current_date
#
speciesobs.save()
except Exception as e:
admin_models.addResultLog(log_row_id, result_log = '- Error in row ' + unicode(rowindex) + ': ' + unicode(e))
#
print('Species observations update. Mark not updated rows as DELETED.')
#
current_date = unicode(time.strftime('%Y-%m-%d'))
# speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(status = 'DELETED')
# speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(last_update_date = current_date)
#
datarows = speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00')
for datarow in datarows:
if datarow.status == 'DELETED':
datarow.last_update_date = current_date
else:
datarow.status = 'DELETED'
datarow.last_update_date = current_date
datarow.last_status_change_date = current_date
#
datarow.save()
#
admin_models.changeLogRowStatus(log_row_id, status = 'FINISHED')
#
print('Species observations update. Finished.')
#
except Exception as e:
admin_models.addResultLog(log_row_id, result_log = '- Failed. Error: ' + unicode(e))
admin_models.changeLogRowStatus(log_row_id, status = 'FAILED')
