def generateIcesXml(self, logrow_id, datatype_list, year_from, year_to, status, user): """ """ # Load resource content to translate from DynTaxa to WoRMS. self._translate_taxa = TranslateTaxa() self._translate_taxa.loadTranslateTaxa('translate_dyntaxa_to_worms') # Create target directory if not exists. if not os.path.exists( self._export_dir_path): os.makedirs( self._export_dir_path) # error_counter = 0 # year_int = int(year_from) year_to_int = int(year_to) # for datatype in datatype_list: while year_int <= year_to_int: "" self.generateOneIcesXml(logrow_id, error_counter, datatype, unicode(year_int), status, user) # year_int += 1 # if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED (Errors: ' + unicode(error_counter) + ')') else: admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') if settings.DEBUG: print('DEBUG: ICES-XML generation FINISHED')
def cleanUpSpeciesObs(self, log_row_id): """ """ # # Remove deleted rows when there is no more need for them from the consumer side (SLU). # speciesobs_models.SpeciesObs.objects.filter(status='DELETED').delete() # Deletes all rows. speciesobs_models.SpeciesObs.objects.all().delete() # admin_models.changeLogRowStatus(log_row_id, status = 'FINISHED')
def deleteExportFiles(request): """ Deletes rows in the database. """ error_message = None # if request.method == "GET": # form = forms.DeleteExportFilesForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("exportfiles_delete_all.html", contextinstance) elif request.method == "POST": # form = forms.DeleteExportFilesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # # if error_message == None: # if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'): # logrow_id = admin_models.createLogRow(command = 'Delete all datasets from FTP', status = 'RUNNING', user = user) # try: # error_message = dataset_utils.DatasetUtils().deleteAllFilesFromFtp() # admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') # except: # error_message = u"Can't delete datasets from the FTP area." # admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') # admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = 'Delete all ICES-XML files from DB', status = 'RUNNING', user = user) try: exportformats_models.ExportFiles.objects.all().delete() admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete datasets from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("exportfiles_delete_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def validateIcesXml(self, logrow_id, datatype_list, user): """ """ error_counter = 0 # db_exports = export_models.ExportFiles.objects.all() for db_export in db_exports: if db_export.datatype in datatype_list: # error_counter = self.validateOneIcesXml(logrow_id, error_counter, db_export, user) # if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED (Errors: ' + unicode(error_counter) + ')') else: admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') if settings.DEBUG: print('DEBUG: ICES-XML generation FINISHED')
def deleteResource(request, resource_id): """ Deletes one row in the database. The FTP area is not affected. """ resource = models.Resources.objects.get(id=resource_id) # if request.method == "GET": form = forms.DeleteResourceForm() contextinstance = {'form' : form, 'resource' : resource, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_resource.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.DeleteResourceForm(request.POST) if form.is_valid(): # user = request.POST['user'] if user not in settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.keys(): error_message = u'Not a valid user. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'): logrow_id = admin_models.createLogRow(command = u'Delete resource (FTP)', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().deleteFileFromFtp(resource.resource_file_name) admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Delete resource (DB)', status = u'RUNNING', user = user) try: resource = models.Resources.objects.get(id=resource_id) resource.delete() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resource from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/resources") # contextinstance = {'form' : form, 'resource' : resource, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_resource.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/resources")
def deleteResources(request): """ Deletes all rows in the database. The FTP area is not affected. """ error_message = None # initially. # if request.method == "GET": # form = forms.DeleteAllResourcesForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("resources_delete_all.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.DeleteAllResourcesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'): logrow_id = admin_models.createLogRow(command = u'Delete all resources (FTP)', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().deleteAllFilesFromFtp() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Delete all resources (DB)', status = u'RUNNING', user = user) try: resources_models.Resources.objects.all().delete() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("resources_delete_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def loadAllResources(request): """ Updates the database from resources stored in the FTP area. """ error_message = None # initially. # if request.method == "GET": form = forms.LoadAllResourcesForm() contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("resources_load_all.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.LoadAllResourcesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Load all resources from FTP to DB', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().writeResourcesInfoToDb(user) admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't load resources and save to the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("resources_load_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def deleteExportFile(request, export_name): """ Deletes one row in the database. """ exportfile = models.ExportFiles.objects.get(export_name = export_name) # if request.method == "GET": form = forms.DeleteExportFileForm() contextinstance = {'form' : form, 'exportfile' : exportfile, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_exportfile.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.DeleteExportFileForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = 'Delete exportfile (DB)', status = 'RUNNING', user = user) try: exportfile = models.ExportFiles.objects.get(export_name = export_name) exportfile.delete() admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete exportfile from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/exportformats") # contextinstance = {'form' : form, 'exportfile' : exportfile, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_exportfile.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/exportformats")
def cleanUpSpeciesObs(request): """ Removes species observations with status='DELETED'. """ error_message = None # initially. # if request.method == "GET": # form = forms.CleanUpSpeciesObsForm() contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("speciesobs_cleanup.html", contextinstance) elif request.method == "POST": form = forms.CleanUpSpeciesObsForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Clean up species observations', status = u'RUNNING', user = user) try: error_message = speciesobs_utils.SpeciesObsUtils().cleanUpSpeciesObsInThread(logrow_id) # admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't clean up species observations." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("speciesobs_cleanup.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def generateArchivesForAllDatasetsInThread(self, user): """ """ logrow_id = admin_models.createLogRow(command = u'Generate archives (DwC, etc.)', status = u'RUNNING', user = user) try: # Check if generate_archives thread is running. if self._generate_archives_thread: if self._generate_archives_thread.is_alive(): error_message = u"Generate archives is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return # Use a thread to relese the user. This will take some time. self._generate_archives_thread = threading.Thread(target = self.generateArchivesForAllDatasets, args=(logrow_id, user, )) self._generate_archives_thread.start() except: error_message = u"Can't generate_archives." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return None # No error message.
def validateIcesXmlInThread(self, datatype_list, user): """ """ logrow_id = admin_models.createLogRow(command = 'Validate ICES-XML file.', status = 'RUNNING', user = user) try: # Check if thread is running. if self._validate_ices_xml_thread: if self._validate_ices_xml_thread.is_alive(): error_message = u"Validate ICES-XML file is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return # Use a thread to relese the user. self._validate_ices_xml_thread = threading.Thread(target = sharkdata_core.ValidateIcesXml().validateIcesXml, args=(logrow_id, datatype_list, user )) self._validate_ices_xml_thread.start() except Exception as e: error_message = u"Can't validate ICES-XML file." + '\nException: ' + unicode(e) + '\n' admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return None # No error message.
def loadAllDatasets(request): """ Updates the database from datasets stored in the FTP area. I multiple versions of a dataset are in the FTP area only the latest will be loaded. """ error_message = None # initially. # if request.method == "GET": form = forms.LoadAllDatasetsForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("datasets_load_all.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.LoadAllDatasetsForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # # # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Load all datasets from FTP to DB', status = u'RUNNING', user = user) try: error_counter = dataset_utils.DatasetUtils().writeLatestDatasetsInfoToDb(logrow_id, user) if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')') else: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except Exception as e: error_message = u"Can't load datasets and save to the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) if settings.DEBUG: print(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n') settings.LOGGER.error(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n') # # if error_message == None: # if ('update_metadata' in request.POST) and (request.POST['update_metadata'] == u'on'): # dataset_utils.DatasetUtils().updateMetadataForAllDatasetsInThread(user) # # if error_message == None: # if ('generate_archives' in request.POST) and (request.POST['generate_archives'] == u'on'): # dataset_utils.DatasetUtils().generateArchivesForAllDatasetsInThread(user) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("datasets_load_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def generateIcesXmlInThread(self, datatype_list, year_from, year_to, status, user): """ """ logrow_id = admin_models.createLogRow(command="Generate ICES-XML file.", status="RUNNING", user=user) try: # Check if thread is running. if self._generate_ices_xml_thread: if self._generate_ices_xml_thread.is_alive(): error_message = "Generate ICES-XML file is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status="FAILED") admin_models.addResultLog(logrow_id, result_log=error_message) # return # Use a thread to relese the user. self._generate_ices_xml_thread = threading.Thread( target=sharkdata_core.GenerateIcesXml().generateIcesXml, args=(logrow_id, datatype_list, year_from, year_to, status, user), ) self._generate_ices_xml_thread.start() except Exception as e: error_message = "Can't generate ICES-XML file." + "\nException: " + unicode(e) + "\n" admin_models.changeLogRowStatus(logrow_id, status="FAILED") admin_models.addResultLog(logrow_id, result_log=error_message) # return None # No error message.
def deleteDataset(request, dataset_id): """ Deletes one row in the database. """ dataset = models.Datasets.objects.get(id=dataset_id) # if request.method == "GET": form = forms.DeleteDatasetForm() contextinstance = {'form' : form, 'dataset' : dataset, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_dataset.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.DeleteDatasetForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'): # Delete the marked dataset version. Earlier versions vill be used, if there are any. logrow_id = admin_models.createLogRow(command = 'Delete dataset (FTP)', status = 'RUNNING', user = user) try: error_message = sharkdata_core.DatasetUtils().deleteFileFromFtp(dataset.dataset_file_name) error_message = sharkdata_core.DatasetUtils().writeLatestDatasetsInfoToDb(user) admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete dataset from the ftp." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) else: logrow_id = admin_models.createLogRow(command = 'Delete dataset (DB)', status = 'RUNNING', user = user) try: dataset = models.Datasets.objects.get(id=dataset_id) dataset.delete() except: error_message = u"Can't delete dataset from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/datasets") # contextinstance = {'form' : form, 'dataset' : dataset, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_dataset.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/datasets")
def generateArchivesForAllDatasets(self, logrow_id, user): """ """ # Load resource file containing WoRMS info for taxa. worms_info_object = misc_utils.SpeciesWormsInfo() worms_info_object.loadSpeciesFromResource() # error_counter = 0 datasets = models.Datasets.objects.all() for dataset in datasets: zip_file_name = dataset.dataset_file_name # admin_models.addResultLog(logrow_id, result_log = u'Generating archive file for: ' + zip_file_name + u'...') if settings.DEBUG: print(u'DEBUG: ===== Processing: ' + zip_file_name) # archive = None dwca_config_dir = u'' if zip_file_name.startswith(u'SHARK_Zooplankton'): archive = dwca_eurobis_zooplankton.DwcaEurObisZooplankton() elif zip_file_name.startswith(u'SHARK_Phytoplankton'): archive = dwca_eurobis_phytoplankton.DwcaEurObisPhytoplankton() elif zip_file_name.startswith(u'SHARK_Zoobenthos'): archive = dwca_eurobis_zoobenthos.DwcaEurObisZoobenthos() elif zip_file_name.startswith(u'SHARK_Bacterioplankton'): archive = dwca_eurobis_bacterioplankton.DwcaEurObisBacterioplankton() # if not archive: continue # Skip if other datatypes. # === Test for GBIF-Occurrence, GBIF-EurOBIS (EMODnet-Bio) and GBIF for Sample Data. === try: dataset = misc_utils.Dataset() dataset.loadDataFromZipFile(zip_file_name, dataset_dir_path = self._ftp_dir_path, encoding = u'cp1252') # # === Test for GBIF-Occurrence. === # try: # admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive.') # archive = biological_data_exchange_util.DarwinCoreArchiveFormat( # datatype, # u'settings_dwca.json', # dwca_config_dir, # meta_file_name = u'meta.xml', # eml_file_name = u'eml.xml', # worms_info_object = worms_info_object) # archive.createArchiveParts(dataset) # # Save generated archive file. # generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') # achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A.zip') # if not os.path.exists(generated_archives_path): # os.makedirs(generated_archives_path) # archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, # settings_dir_path = dwca_config_dir) # # Update database. # db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) # db_dataset.dwc_archive_available = True # db_dataset.dwc_archive_file_path = os.path.join(generated_archives_path, achive_file_name) # db_dataset.save() # except Exception as e: # error_counter += 1 # admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A from: ' + zip_file_name + u'.') # === Test for GBIF-EurOBIS (EMODnet-Bio). === try: admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (EurOBIS format).') # archive = biological_data_exchange_util.DarwinCoreArchiveFormatForEurObis( # datatype, # u'settings_dwca_eurobis.json', # dwca_config_dir, # meta_file_name = u'meta_eurobis.xml', # eml_file_name = u'eml_eurobis.xml', # worms_info_object = worms_info_object) # Update database before. db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) db_dataset.dwc_archive_eurobis_available = False db_dataset.dwc_archive_eurobis_file_path = u'' db_dataset.save() if worms_info_object: archive.setWormsInfoObject(worms_info_object) # archive.createArchiveParts(dataset) # Save generated archive file. generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-EurOBIS.zip') if not os.path.exists(generated_archives_path): os.makedirs(generated_archives_path) archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, settings_dir_path = dwca_config_dir) # Update database after. db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) db_dataset.dwc_archive_eurobis_available = True db_dataset.dwc_archive_eurobis_file_path = os.path.join(generated_archives_path, achive_file_name) db_dataset.save() except Exception as e: error_counter += 1 print(e) admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (eurOBIS format) from: ' + zip_file_name + u'.') # # === Test for GBIF for Sample Data. === # try: # admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (Sample data format).') # archive = biological_data_exchange_util.DarwinCoreArchiveFormatForSampleData( # datatype, # u'settings_dwca_sampledata.json', # dwca_config_dir, # meta_file_name = u'meta_sampledata.xml', # eml_file_name = u'eml_sampledata.xml', # worms_info_object = worms_info_object) # archive.createArchiveParts(dataset) # # Save generated archive file. # generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') # achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-SampleData.zip') # if not os.path.exists(generated_archives_path): # os.makedirs(generated_archives_path) # archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, # settings_dir_path = dwca_config_dir) # # Update database. # db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) # db_dataset.dwc_archive_sampledata_available = True # db_dataset.dwc_archive_sampledata_file_path = os.path.join(generated_archives_path, achive_file_name) # db_dataset.save() # except Exception as e: # error_counter += 1 # admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (Sample data format) from: ' + zip_file_name + u'.') except Exception as e: error_counter += 1 traceback.print_exc() admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate archive files from: ' + zip_file_name + u'.') # if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')') else: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') if settings.DEBUG: print(u'DEBUG: Archive generation FINISHED')
def updateSpeciesObs(self, log_row_id): """ """ # print('Species observations update. Started.') try: # Load resource file containing WoRMS info for taxa. worms_info_object = sharkdata_core.SpeciesWormsInfo() worms_info_object.loadSpeciesFromResource() # Mark all rows in db table before update starts. speciesobs_models.SpeciesObs.objects.all().update(last_update_date = '0000-00-00') # Loop over all datasets. valid_datatypes = [ 'Epibenthos', 'GreySeal', 'HarbourSeal', 'Phytoplankton', 'RingedSeal', 'Zoobenthos', 'Zooplankton', ###'Speciesobs', ] # for dataset_queryset in datasets_models.Datasets.objects.all(): if dataset_queryset.datatype in valid_datatypes: print('Loading data from: ' + dataset_queryset.ftp_file_path) else: print('Skipped (wrong datatype): ' + dataset_queryset.ftp_file_path) continue # admin_models.addResultLog(log_row_id, result_log = 'Extracting species obs from: ' + dataset_queryset.dataset_file_name + '...') # zipreader = sharkdata_core.SharkArchiveFileReader(dataset_queryset.ftp_file_path) try: zipreader.open() data = zipreader.getDataAsText() finally: zipreader.close() # encoding = 'cp1252' rowseparator = '\n' fieldseparator = '\t' # data = unicode(data, encoding, 'strict') datarows = (item.strip() for item in data.split(rowseparator)) # Generator instead of list. # for rowindex, datarow in enumerate(datarows): # try: if len(datarow) == 0: continue # row = [item.strip() for item in datarow.split(fieldseparator)] if rowindex == 0: header = row else: header = self.cleanUpHeader(header) rowdict = dict(zip(header, row)) # Check if position is valid. Skip row if not. # lat_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.') # long_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.') lat_dd = rowdict.get('latitude_dd', '').replace(',', '.') long_dd = rowdict.get('longitude_dd', '').replace(',', '.') # if self.isFloat(lat_dd) and self.isFloat(long_dd): if True: if (float(lat_dd) > 70.0) or (float(lat_dd) < 50.0) or (float(long_dd) > 25.0) or (float(long_dd) < 5.0): # Don't add to SpeciesObs if lat_dd/long_dd is outside the box. print('Row skipped, position outside box. Latitude: ' + lat_dd + ' Longitude: ' + long_dd + ' Row: ' + unicode(rowindex)) continue else: # Don't add to SpeciesObs if lat_dd/long_dd is invalid. continue # tmp_date = rowdict.get('sampling_date', '') tmp_year = '' tmp_month = '' tmp_day = '' if len(tmp_date) >= 10: tmp_year = tmp_date[0:4] tmp_month = tmp_date[5:7] tmp_day = tmp_date[8:10] scientificname = rowdict.get('scientific_name', '-') if rowdict.get('scientific_name') else '-' scientificauthority = rowdict.get('scientific_authority', '-') if rowdict.get('scientific_authority') else '-' taxon_worms_info = worms_info_object.getTaxonInfoDict(scientificname) if taxon_worms_info: taxonkingdom = taxon_worms_info.get('kingdom', '-') if taxon_worms_info.get('kingdom') else '-' taxonphylum = taxon_worms_info.get('phylum', '-') if taxon_worms_info.get('phylum') else '-' taxonclass = taxon_worms_info.get('class', '-') if taxon_worms_info.get('class') else '-' taxonorder = taxon_worms_info.get('order', '-') if taxon_worms_info.get('order') else '-' taxonfamily = taxon_worms_info.get('family', '-') if taxon_worms_info.get('family') else '-' taxongenus = taxon_worms_info.get('genus', '-') if taxon_worms_info.get('genus') else '-' else: taxonkingdom = '-' taxonphylum = '-' taxonclass = '-' taxonorder = '-' taxonfamily = '-' taxongenus = '-' speciesobs = speciesobs_models.SpeciesObs( data_type = rowdict.get('data_type', ''), scientific_name = scientificname, scientific_authority = scientificauthority, # latitude_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.'), # longitude_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.'), latitude_dd = rowdict.get('latitude_dd', '').replace(',', '.'), longitude_dd = rowdict.get('longitude_dd', '').replace(',', '.'), sampling_date = rowdict.get('sampling_date', ''), sampling_year = tmp_year, sampling_month = tmp_month, sampling_day = tmp_day, sample_min_depth = rowdict.get('sample_min_depth', ''), sample_max_depth = rowdict.get('sample_max_depth', ''), sampler_type = rowdict.get('sampler_type', ''), dyntaxa_id = rowdict.get('dyntaxa_id', '') if rowdict.get('dyntaxa_id') else '-', # taxon_kingdom = rowdict.get('taxon_kingdom', '-') if rowdict.get('taxon_kingdom') else '-', # taxon_phylum = rowdict.get('taxon_phylum', '-') if rowdict.get('taxon_phylum') else '-', # taxon_class = rowdict.get('taxon_class', '-') if rowdict.get('taxon_class') else '-', # taxon_order = rowdict.get('taxon_order', '-') if rowdict.get('taxon_order') else '-', # taxon_family = rowdict.get('taxon_family', '-') if rowdict.get('taxon_family') else '-', # taxon_genus = rowdict.get('taxon_genus', '-') if rowdict.get('taxon_genus') else '-', # taxon_species = rowdict.get('taxon_species', '-') if rowdict.get('taxon_species') else '-', taxon_kingdom = taxonkingdom, taxon_phylum = taxonphylum, taxon_class = taxonclass, taxon_order = taxonorder, taxon_family = taxonfamily, taxon_genus = taxongenus, # taxon_species = rowdict.get('species', '-') if rowdict.get('species') else '-', orderer = rowdict.get('orderer', '') if rowdict.get('orderer') else '-', reporting_institute = rowdict.get('reporting_institute', '') if rowdict.get('reporting_institute') else '-', sampling_laboratory = rowdict.get('sampling_laboratory', '') if rowdict.get('sampling_laboratory') else '-', analytical_laboratory = rowdict.get('analytical_laboratory', '') if rowdict.get('analytical_laboratory') else '-', # occurrence_id = '', # Added below. # dataset_name = unicode(dataset_queryset.dataset_name), dataset_file_name = unicode(dataset_queryset.dataset_file_name), # ##### Example: 'POINT(-73.9869510 40.7560540)', Note: Order longitude - latitude. # geometry = 'POINT(' + rowdict.get('longitude_dd', '0.0').replace(',', '.') + ' ' + rowdict.get('latitude_dd', '0.0').replace(',', '.') + ')', ) # Calculate DarwinCore Observation Id. generated_occurrence_id = speciesobs.calculateDarwinCoreObservationIdAsMD5() # # if speciesobs_models.SpeciesObs( # speciesobs.save() #speciesobs_models.SpeciesObs.objects.filter(status='DELETED').delete() try: obs_existing_row = speciesobs_models.SpeciesObs.objects.get(occurrence_id = generated_occurrence_id) except: obs_existing_row = None # Does not exist. # current_date = unicode(time.strftime('%Y-%m-%d')) # if obs_existing_row: if obs_existing_row.status == 'ACTIVE': obs_existing_row.last_update_date = current_date else: obs_existing_row.status= 'ACTIVE' obs_existing_row.last_update_date = current_date obs_existing_row.last_status_change_date = current_date # obs_existing_row.save() else: speciesobs.occurrence_id = generated_occurrence_id speciesobs.status = 'ACTIVE' speciesobs.last_update_date = current_date speciesobs.last_status_change_date = current_date # speciesobs.save() except Exception as e: admin_models.addResultLog(log_row_id, result_log = '- Error in row ' + unicode(rowindex) + ': ' + unicode(e)) # print('Species observations update. Mark not updated rows as DELETED.') # current_date = unicode(time.strftime('%Y-%m-%d')) # speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(status = 'DELETED') # speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(last_update_date = current_date) # datarows = speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00') for datarow in datarows: if datarow.status == 'DELETED': datarow.last_update_date = current_date else: datarow.status = 'DELETED' datarow.last_update_date = current_date datarow.last_status_change_date = current_date # datarow.save() # admin_models.changeLogRowStatus(log_row_id, status = 'FINISHED') # print('Species observations update. Finished.') # except Exception as e: admin_models.addResultLog(log_row_id, result_log = '- Failed. Error: ' + unicode(e)) admin_models.changeLogRowStatus(log_row_id, status = 'FAILED')