def loadAllDatasets(request): """ Updates the database from datasets stored in the FTP area. I multiple versions of a dataset are in the FTP area only the latest will be loaded. """ error_message = None # initially. # if request.method == "GET": form = forms.LoadAllDatasetsForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("datasets_load_all.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.LoadAllDatasetsForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # # # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Load all datasets from FTP to DB', status = u'RUNNING', user = user) try: error_counter = dataset_utils.DatasetUtils().writeLatestDatasetsInfoToDb(logrow_id, user) if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')') else: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except Exception as e: error_message = u"Can't load datasets and save to the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) if settings.DEBUG: print(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n') settings.LOGGER.error(u'\nError: ' + error_message + u'\nException: ' + unicode(e) + u'\n') # # if error_message == None: # if ('update_metadata' in request.POST) and (request.POST['update_metadata'] == u'on'): # dataset_utils.DatasetUtils().updateMetadataForAllDatasetsInThread(user) # # if error_message == None: # if ('generate_archives' in request.POST) and (request.POST['generate_archives'] == u'on'): # dataset_utils.DatasetUtils().generateArchivesForAllDatasetsInThread(user) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("datasets_load_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def deleteResource(request, resource_id): """ Deletes one row in the database. The FTP area is not affected. """ resource = models.Resources.objects.get(id=resource_id) # if request.method == "GET": form = forms.DeleteResourceForm() contextinstance = {'form' : form, 'resource' : resource, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_resource.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.DeleteResourceForm(request.POST) if form.is_valid(): # user = request.POST['user'] if user not in settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.keys(): error_message = u'Not a valid user. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'): logrow_id = admin_models.createLogRow(command = u'Delete resource (FTP)', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().deleteFileFromFtp(resource.resource_file_name) admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Delete resource (DB)', status = u'RUNNING', user = user) try: resource = models.Resources.objects.get(id=resource_id) resource.delete() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resource from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/resources") # contextinstance = {'form' : form, 'resource' : resource, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_resource.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/resources")
def deleteResources(request): """ Deletes all rows in the database. The FTP area is not affected. """ error_message = None # initially. # if request.method == "GET": # form = forms.DeleteAllResourcesForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("resources_delete_all.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.DeleteAllResourcesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == u'on'): logrow_id = admin_models.createLogRow(command = u'Delete all resources (FTP)', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().deleteAllFilesFromFtp() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Delete all resources (DB)', status = u'RUNNING', user = user) try: resources_models.Resources.objects.all().delete() admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't delete resources from the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("resources_delete_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def deleteDataset(request, dataset_id): """ Deletes one row in the database. """ dataset = models.Datasets.objects.get(id=dataset_id) # if request.method == "GET": form = forms.DeleteDatasetForm() contextinstance = {'form' : form, 'dataset' : dataset, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_dataset.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.DeleteDatasetForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # if error_message == None: if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'): # Delete the marked dataset version. Earlier versions vill be used, if there are any. logrow_id = admin_models.createLogRow(command = 'Delete dataset (FTP)', status = 'RUNNING', user = user) try: error_message = sharkdata_core.DatasetUtils().deleteFileFromFtp(dataset.dataset_file_name) error_message = sharkdata_core.DatasetUtils().writeLatestDatasetsInfoToDb(user) admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete dataset from the ftp." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) else: logrow_id = admin_models.createLogRow(command = 'Delete dataset (DB)', status = 'RUNNING', user = user) try: dataset = models.Datasets.objects.get(id=dataset_id) dataset.delete() except: error_message = u"Can't delete dataset from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/datasets") # contextinstance = {'form' : form, 'dataset' : dataset, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_dataset.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/datasets")
def writeFileInfoToDb(self, file_name, logrow_id = None, user = ''): """ Extracts info from the dataset filename and from the zip file content and adds to database. """ try: # ftp_file_path = os.path.join(self._ftp_dir_path, file_name) # Extract info from file name. dataset_name, datatype, version = self.splitFilename(file_name) # Extract metadata parts. metadata = '' metadata_auto = '' columndata_available = False # zipreader = sharkdata_core.SharkArchiveFileReader(file_name, self._ftp_dir_path) try: zipreader.open() # try: metadata = zipreader.getMetadataAsText() encoding = 'cp1252' metadata = unicode(metadata, encoding, 'strict') except Exception as e: admin_models.addResultLog(logrow_id, result_log = 'WARNING: ' + unicode(e)) # try: metadata_auto = zipreader.getMetadataAutoAsText() encoding = 'cp1252' metadata_auto = unicode(metadata_auto, encoding, 'strict') except Exception as e: admin_models.addResultLog(logrow_id, result_log = 'WARNING: ' + unicode(e)) # columndata_available = zipreader.isDataColumnsAvailable() finally: zipreader.close() # Save to db. dataset = datasets_models.Datasets( dataset_name = dataset_name, datatype = datatype, version = version, dataset_file_name = file_name, ftp_file_path = ftp_file_path, content_data = 'NOT USED', content_metadata = metadata, content_metadata_auto = metadata_auto, # column_data_available = columndata_available, dwc_archive_eurobis_available = False, dwc_archive_eurobis_file_path = '', ) dataset.save() # return None # No error message. # except Exception as e: return unicode(e)
def deleteExportFiles(request): """ Deletes rows in the database. """ error_message = None # if request.method == "GET": # form = forms.DeleteExportFilesForm() contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("exportfiles_delete_all.html", contextinstance) elif request.method == "POST": # form = forms.DeleteExportFilesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # # if error_message == None: # if ('delete_ftp' in request.POST) and (request.POST['delete_ftp'] == 'on'): # logrow_id = admin_models.createLogRow(command = 'Delete all datasets from FTP', status = 'RUNNING', user = user) # try: # error_message = dataset_utils.DatasetUtils().deleteAllFilesFromFtp() # admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') # except: # error_message = u"Can't delete datasets from the FTP area." # admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') # admin_models.addResultLog(logrow_id, result_log = error_message) # if error_message == None: logrow_id = admin_models.createLogRow(command = 'Delete all ICES-XML files from DB', status = 'RUNNING', user = user) try: exportformats_models.ExportFiles.objects.all().delete() admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete datasets from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form' : form, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("exportfiles_delete_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def loadAllResources(request): """ Updates the database from resources stored in the FTP area. """ error_message = None # initially. # if request.method == "GET": form = forms.LoadAllResourcesForm() contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("resources_load_all.html", contextinstance) elif request.method == "POST": # Reloads db-stored data. resources_utils.ResourcesUtils().clear() # error_message = None # initially. # form = forms.LoadAllResourcesForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Load all resources from FTP to DB', status = u'RUNNING', user = user) try: resources_utils.ResourcesUtils().writeResourcesInfoToDb(user) admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't load resources and save to the database." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("resources_load_all.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def deleteExportFile(request, export_name): """ Deletes one row in the database. """ exportfile = models.ExportFiles.objects.get(export_name = export_name) # if request.method == "GET": form = forms.DeleteExportFileForm() contextinstance = {'form' : form, 'exportfile' : exportfile, 'error_message' : None} contextinstance.update(csrf(request)) return render_to_response("delete_exportfile.html", contextinstance) elif request.method == "POST": error_message = None # initially. # form = forms.DeleteExportFileForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = 'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = 'Delete exportfile (DB)', status = 'RUNNING', user = user) try: exportfile = models.ExportFiles.objects.get(export_name = export_name) exportfile.delete() admin_models.changeLogRowStatus(logrow_id, status = 'FINISHED') except: error_message = u"Can't delete exportfile from the database." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/exportformats") # contextinstance = {'form' : form, 'exportfile' : exportfile, 'error_message' : error_message} contextinstance.update(csrf(request)) return render_to_response("delete_exportfile.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/exportformats")
def cleanUpSpeciesObs(request): """ Removes species observations with status='DELETED'. """ error_message = None # initially. # if request.method == "GET": # form = forms.CleanUpSpeciesObsForm() contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("speciesobs_cleanup.html", contextinstance) elif request.method == "POST": form = forms.CleanUpSpeciesObsForm(request.POST) if form.is_valid(): # user = request.POST['user'] password = request.POST['password'] if password != settings.APPS_VALID_USERS_AND_PASSWORDS_FOR_TEST.get(user, None): error_message = u'Not a valid user or password. Please try again...' # if error_message == None: logrow_id = admin_models.createLogRow(command = u'Clean up species observations', status = u'RUNNING', user = user) try: error_message = speciesobs_utils.SpeciesObsUtils().cleanUpSpeciesObsInThread(logrow_id) # admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') except: error_message = u"Can't clean up species observations." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # OK. if error_message == None: return HttpResponseRedirect("/sharkdataadmin") # contextinstance = {'form': form, 'error_message': error_message} contextinstance.update(csrf(request)) return render_to_response("speciesobs_cleanup.html", contextinstance) # Not a valid request method. return HttpResponseRedirect("/sharkdataadmin")
def writeLatestDatasetsInfoToDb(self, logrow_id = None, user = u''): """ Updates the database from datasets stored in the FTP area. I multiple versions of a dataset are in the FTP area only the latest will be loaded. """ error_counter = 0 # Remove all db rows. datasets_models.Datasets.objects.all().delete() # Get latest datasets from FTP archive. archive = shark_utils.SharkArchive(self._ftp_dir_path) for file_name in sorted(archive.getLatestSharkArchiveFilenames()): if logrow_id: admin_models.addResultLog(logrow_id, result_log = u'Loading file: ' + file_name + u'...') try: self.writeFileInfoToDb(file_name, user) except Exception as e: error_counter += 1 admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to load: ' + file_name + u'.') # return error_counter
def generateArchivesForAllDatasetsInThread(self, user): """ """ logrow_id = admin_models.createLogRow(command = u'Generate archives (DwC, etc.)', status = u'RUNNING', user = user) try: # Check if generate_archives thread is running. if self._generate_archives_thread: if self._generate_archives_thread.is_alive(): error_message = u"Generate archives is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return # Use a thread to relese the user. This will take some time. self._generate_archives_thread = threading.Thread(target = self.generateArchivesForAllDatasets, args=(logrow_id, user, )) self._generate_archives_thread.start() except: error_message = u"Can't generate_archives." admin_models.changeLogRowStatus(logrow_id, status = u'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return None # No error message.
def validateIcesXmlInThread(self, datatype_list, user): """ """ logrow_id = admin_models.createLogRow(command = 'Validate ICES-XML file.', status = 'RUNNING', user = user) try: # Check if thread is running. if self._validate_ices_xml_thread: if self._validate_ices_xml_thread.is_alive(): error_message = u"Validate ICES-XML file is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return # Use a thread to relese the user. self._validate_ices_xml_thread = threading.Thread(target = sharkdata_core.ValidateIcesXml().validateIcesXml, args=(logrow_id, datatype_list, user )) self._validate_ices_xml_thread.start() except Exception as e: error_message = u"Can't validate ICES-XML file." + '\nException: ' + unicode(e) + '\n' admin_models.changeLogRowStatus(logrow_id, status = 'FAILED') admin_models.addResultLog(logrow_id, result_log = error_message) # return None # No error message.
def generateIcesXmlInThread(self, datatype_list, year_from, year_to, status, user): """ """ logrow_id = admin_models.createLogRow(command="Generate ICES-XML file.", status="RUNNING", user=user) try: # Check if thread is running. if self._generate_ices_xml_thread: if self._generate_ices_xml_thread.is_alive(): error_message = "Generate ICES-XML file is already running. Please try again later." admin_models.changeLogRowStatus(logrow_id, status="FAILED") admin_models.addResultLog(logrow_id, result_log=error_message) # return # Use a thread to relese the user. self._generate_ices_xml_thread = threading.Thread( target=sharkdata_core.GenerateIcesXml().generateIcesXml, args=(logrow_id, datatype_list, year_from, year_to, status, user), ) self._generate_ices_xml_thread.start() except Exception as e: error_message = "Can't generate ICES-XML file." + "\nException: " + unicode(e) + "\n" admin_models.changeLogRowStatus(logrow_id, status="FAILED") admin_models.addResultLog(logrow_id, result_log=error_message) # return None # No error message.
def updateSpeciesObs(self, log_row_id): """ """ # print('Species observations update. Started.') try: # Load resource file containing WoRMS info for taxa. worms_info_object = sharkdata_core.SpeciesWormsInfo() worms_info_object.loadSpeciesFromResource() # Mark all rows in db table before update starts. speciesobs_models.SpeciesObs.objects.all().update(last_update_date = '0000-00-00') # Loop over all datasets. valid_datatypes = [ 'Epibenthos', 'GreySeal', 'HarbourSeal', 'Phytoplankton', 'RingedSeal', 'Zoobenthos', 'Zooplankton', ###'Speciesobs', ] # for dataset_queryset in datasets_models.Datasets.objects.all(): if dataset_queryset.datatype in valid_datatypes: print('Loading data from: ' + dataset_queryset.ftp_file_path) else: print('Skipped (wrong datatype): ' + dataset_queryset.ftp_file_path) continue # admin_models.addResultLog(log_row_id, result_log = 'Extracting species obs from: ' + dataset_queryset.dataset_file_name + '...') # zipreader = sharkdata_core.SharkArchiveFileReader(dataset_queryset.ftp_file_path) try: zipreader.open() data = zipreader.getDataAsText() finally: zipreader.close() # encoding = 'cp1252' rowseparator = '\n' fieldseparator = '\t' # data = unicode(data, encoding, 'strict') datarows = (item.strip() for item in data.split(rowseparator)) # Generator instead of list. # for rowindex, datarow in enumerate(datarows): # try: if len(datarow) == 0: continue # row = [item.strip() for item in datarow.split(fieldseparator)] if rowindex == 0: header = row else: header = self.cleanUpHeader(header) rowdict = dict(zip(header, row)) # Check if position is valid. Skip row if not. # lat_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.') # long_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.') lat_dd = rowdict.get('latitude_dd', '').replace(',', '.') long_dd = rowdict.get('longitude_dd', '').replace(',', '.') # if self.isFloat(lat_dd) and self.isFloat(long_dd): if True: if (float(lat_dd) > 70.0) or (float(lat_dd) < 50.0) or (float(long_dd) > 25.0) or (float(long_dd) < 5.0): # Don't add to SpeciesObs if lat_dd/long_dd is outside the box. print('Row skipped, position outside box. Latitude: ' + lat_dd + ' Longitude: ' + long_dd + ' Row: ' + unicode(rowindex)) continue else: # Don't add to SpeciesObs if lat_dd/long_dd is invalid. continue # tmp_date = rowdict.get('sampling_date', '') tmp_year = '' tmp_month = '' tmp_day = '' if len(tmp_date) >= 10: tmp_year = tmp_date[0:4] tmp_month = tmp_date[5:7] tmp_day = tmp_date[8:10] scientificname = rowdict.get('scientific_name', '-') if rowdict.get('scientific_name') else '-' scientificauthority = rowdict.get('scientific_authority', '-') if rowdict.get('scientific_authority') else '-' taxon_worms_info = worms_info_object.getTaxonInfoDict(scientificname) if taxon_worms_info: taxonkingdom = taxon_worms_info.get('kingdom', '-') if taxon_worms_info.get('kingdom') else '-' taxonphylum = taxon_worms_info.get('phylum', '-') if taxon_worms_info.get('phylum') else '-' taxonclass = taxon_worms_info.get('class', '-') if taxon_worms_info.get('class') else '-' taxonorder = taxon_worms_info.get('order', '-') if taxon_worms_info.get('order') else '-' taxonfamily = taxon_worms_info.get('family', '-') if taxon_worms_info.get('family') else '-' taxongenus = taxon_worms_info.get('genus', '-') if taxon_worms_info.get('genus') else '-' else: taxonkingdom = '-' taxonphylum = '-' taxonclass = '-' taxonorder = '-' taxonfamily = '-' taxongenus = '-' speciesobs = speciesobs_models.SpeciesObs( data_type = rowdict.get('data_type', ''), scientific_name = scientificname, scientific_authority = scientificauthority, # latitude_dd = rowdict.get('sample_latitude_dd', '').replace(',', '.'), # longitude_dd = rowdict.get('sample_longitude_dd', '').replace(',', '.'), latitude_dd = rowdict.get('latitude_dd', '').replace(',', '.'), longitude_dd = rowdict.get('longitude_dd', '').replace(',', '.'), sampling_date = rowdict.get('sampling_date', ''), sampling_year = tmp_year, sampling_month = tmp_month, sampling_day = tmp_day, sample_min_depth = rowdict.get('sample_min_depth', ''), sample_max_depth = rowdict.get('sample_max_depth', ''), sampler_type = rowdict.get('sampler_type', ''), dyntaxa_id = rowdict.get('dyntaxa_id', '') if rowdict.get('dyntaxa_id') else '-', # taxon_kingdom = rowdict.get('taxon_kingdom', '-') if rowdict.get('taxon_kingdom') else '-', # taxon_phylum = rowdict.get('taxon_phylum', '-') if rowdict.get('taxon_phylum') else '-', # taxon_class = rowdict.get('taxon_class', '-') if rowdict.get('taxon_class') else '-', # taxon_order = rowdict.get('taxon_order', '-') if rowdict.get('taxon_order') else '-', # taxon_family = rowdict.get('taxon_family', '-') if rowdict.get('taxon_family') else '-', # taxon_genus = rowdict.get('taxon_genus', '-') if rowdict.get('taxon_genus') else '-', # taxon_species = rowdict.get('taxon_species', '-') if rowdict.get('taxon_species') else '-', taxon_kingdom = taxonkingdom, taxon_phylum = taxonphylum, taxon_class = taxonclass, taxon_order = taxonorder, taxon_family = taxonfamily, taxon_genus = taxongenus, # taxon_species = rowdict.get('species', '-') if rowdict.get('species') else '-', orderer = rowdict.get('orderer', '') if rowdict.get('orderer') else '-', reporting_institute = rowdict.get('reporting_institute', '') if rowdict.get('reporting_institute') else '-', sampling_laboratory = rowdict.get('sampling_laboratory', '') if rowdict.get('sampling_laboratory') else '-', analytical_laboratory = rowdict.get('analytical_laboratory', '') if rowdict.get('analytical_laboratory') else '-', # occurrence_id = '', # Added below. # dataset_name = unicode(dataset_queryset.dataset_name), dataset_file_name = unicode(dataset_queryset.dataset_file_name), # ##### Example: 'POINT(-73.9869510 40.7560540)', Note: Order longitude - latitude. # geometry = 'POINT(' + rowdict.get('longitude_dd', '0.0').replace(',', '.') + ' ' + rowdict.get('latitude_dd', '0.0').replace(',', '.') + ')', ) # Calculate DarwinCore Observation Id. generated_occurrence_id = speciesobs.calculateDarwinCoreObservationIdAsMD5() # # if speciesobs_models.SpeciesObs( # speciesobs.save() #speciesobs_models.SpeciesObs.objects.filter(status='DELETED').delete() try: obs_existing_row = speciesobs_models.SpeciesObs.objects.get(occurrence_id = generated_occurrence_id) except: obs_existing_row = None # Does not exist. # current_date = unicode(time.strftime('%Y-%m-%d')) # if obs_existing_row: if obs_existing_row.status == 'ACTIVE': obs_existing_row.last_update_date = current_date else: obs_existing_row.status= 'ACTIVE' obs_existing_row.last_update_date = current_date obs_existing_row.last_status_change_date = current_date # obs_existing_row.save() else: speciesobs.occurrence_id = generated_occurrence_id speciesobs.status = 'ACTIVE' speciesobs.last_update_date = current_date speciesobs.last_status_change_date = current_date # speciesobs.save() except Exception as e: admin_models.addResultLog(log_row_id, result_log = '- Error in row ' + unicode(rowindex) + ': ' + unicode(e)) # print('Species observations update. Mark not updated rows as DELETED.') # current_date = unicode(time.strftime('%Y-%m-%d')) # speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(status = 'DELETED') # speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00').update(last_update_date = current_date) # datarows = speciesobs_models.SpeciesObs.objects.filter(last_update_date = '0000-00-00') for datarow in datarows: if datarow.status == 'DELETED': datarow.last_update_date = current_date else: datarow.status = 'DELETED' datarow.last_update_date = current_date datarow.last_status_change_date = current_date # datarow.save() # admin_models.changeLogRowStatus(log_row_id, status = 'FINISHED') # print('Species observations update. Finished.') # except Exception as e: admin_models.addResultLog(log_row_id, result_log = '- Failed. Error: ' + unicode(e)) admin_models.changeLogRowStatus(log_row_id, status = 'FAILED')
def generateOneIcesXml(self, logrow_id, error_counter, datatype, year, status, user): """ """ # Add all rows from all datasets that match datatype and year. icesxmlgenerator = IcesXmlGenerator(self._translate_taxa) # db_datasets = datasets_models.Datasets.objects.all() for db_dataset in db_datasets: if db_dataset.datatype.upper() != datatype.upper(): continue # try: zip_file_name = db_dataset.dataset_file_name admin_models.addResultLog(logrow_id, result_log = 'Reading archive file: ' + zip_file_name + '...') if settings.DEBUG: if settings.DEBUG: print('DEBUG: ICES-ZIP processing: ' + zip_file_name) # dataset = sharkdata_core.Dataset() dataset.loadDataFromZipFile(zip_file_name, dataset_dir_path = self._ftp_dir_path, encoding = 'cp1252') # dataheader = dataset.data_header if settings.DEBUG: print(dataheader) # datarows = dataset.data_rows for datarow in datarows: datarow_dict = dict(zip(dataheader, map(unicode, datarow))) # if datarow_dict.get('visit_year', '') == year: # TODO: Should be codes. national_monitoring_list = ['National marine monitoring', 'Nationella programmet Bottniska Viken från 2015', 'Nationella programmet Egentliga Östersjön, Utsjön 2007-'] # if datarow_dict.get('sample_project_name_sv', '') in national_monitoring_list: icesxmlgenerator.add_row(datarow_dict) # except Exception as e: error_counter += 1 traceback.print_exc() admin_models.addResultLog(logrow_id, result_log = 'ERROR: Failed to generate ICES-XML from: ' + zip_file_name + '.') # try: # Create and save the result. out_rows = icesxmlgenerator.create_xml() # export_name = 'ICES-XML' + '_SMHI_' + datatype + '_' + year export_file_name = export_name + '.xml' export_file_path = os.path.join(self._export_dir_path, export_file_name) error_log_file = export_name + '_log.txt' error_log_file_path = os.path.join(self._export_dir_path, error_log_file) # icesxmlgenerator.save_xml_file(out_rows, export_file_path) # Update database. # Delete row if exists. export_db_rows = export_models.ExportFiles.objects.filter(export_name = export_name) for db_row in export_db_rows: db_row.delete() # approved = False if status == 'Checked by DC': approved = True # Will not be validated via DATSU. # Write row. dbrow = export_models.ExportFiles( format = 'ICES-XML', datatype = datatype, year = year, approved = approved, status = status, export_name = export_name, export_file_name = export_file_name, export_file_path = export_file_path, error_log_file = error_log_file, error_log_file_path = error_log_file_path, generated_by = user, ) dbrow.save() # Log file. log_rows = [] log_rows.append('') log_rows.append('') log_rows.append('Generate ICES-XML files. ' + unicode(datetime.datetime.now())) log_rows.append('') log_rows.append('- Format: ' + dbrow.format) log_rows.append('- Datatype: ' + unicode(dbrow.datatype)) log_rows.append('- Year: ' + unicode(dbrow.year)) log_rows.append('- Status: ' + unicode(dbrow.status)) log_rows.append('- Approved: ' + unicode(dbrow.approved)) log_rows.append('- Export name: ' + unicode(dbrow.export_name)) log_rows.append('- Export file name: ' + unicode(dbrow.export_file_name)) log_rows.append('') # icesxmlgenerator.save_log_file(log_rows, error_log_file_path) # except Exception as e: error_counter += 1 traceback.print_exc() admin_models.addResultLog(logrow_id, result_log = 'ERROR: Failed to generate ICES-XML files. Exception: ' + unicode(e))
def generateArchivesForAllDatasets(self, logrow_id, user): """ """ # Load resource file containing WoRMS info for taxa. worms_info_object = misc_utils.SpeciesWormsInfo() worms_info_object.loadSpeciesFromResource() # error_counter = 0 datasets = models.Datasets.objects.all() for dataset in datasets: zip_file_name = dataset.dataset_file_name # admin_models.addResultLog(logrow_id, result_log = u'Generating archive file for: ' + zip_file_name + u'...') if settings.DEBUG: print(u'DEBUG: ===== Processing: ' + zip_file_name) # archive = None dwca_config_dir = u'' if zip_file_name.startswith(u'SHARK_Zooplankton'): archive = dwca_eurobis_zooplankton.DwcaEurObisZooplankton() elif zip_file_name.startswith(u'SHARK_Phytoplankton'): archive = dwca_eurobis_phytoplankton.DwcaEurObisPhytoplankton() elif zip_file_name.startswith(u'SHARK_Zoobenthos'): archive = dwca_eurobis_zoobenthos.DwcaEurObisZoobenthos() elif zip_file_name.startswith(u'SHARK_Bacterioplankton'): archive = dwca_eurobis_bacterioplankton.DwcaEurObisBacterioplankton() # if not archive: continue # Skip if other datatypes. # === Test for GBIF-Occurrence, GBIF-EurOBIS (EMODnet-Bio) and GBIF for Sample Data. === try: dataset = misc_utils.Dataset() dataset.loadDataFromZipFile(zip_file_name, dataset_dir_path = self._ftp_dir_path, encoding = u'cp1252') # # === Test for GBIF-Occurrence. === # try: # admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive.') # archive = biological_data_exchange_util.DarwinCoreArchiveFormat( # datatype, # u'settings_dwca.json', # dwca_config_dir, # meta_file_name = u'meta.xml', # eml_file_name = u'eml.xml', # worms_info_object = worms_info_object) # archive.createArchiveParts(dataset) # # Save generated archive file. # generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') # achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A.zip') # if not os.path.exists(generated_archives_path): # os.makedirs(generated_archives_path) # archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, # settings_dir_path = dwca_config_dir) # # Update database. # db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) # db_dataset.dwc_archive_available = True # db_dataset.dwc_archive_file_path = os.path.join(generated_archives_path, achive_file_name) # db_dataset.save() # except Exception as e: # error_counter += 1 # admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A from: ' + zip_file_name + u'.') # === Test for GBIF-EurOBIS (EMODnet-Bio). === try: admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (EurOBIS format).') # archive = biological_data_exchange_util.DarwinCoreArchiveFormatForEurObis( # datatype, # u'settings_dwca_eurobis.json', # dwca_config_dir, # meta_file_name = u'meta_eurobis.xml', # eml_file_name = u'eml_eurobis.xml', # worms_info_object = worms_info_object) # Update database before. db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) db_dataset.dwc_archive_eurobis_available = False db_dataset.dwc_archive_eurobis_file_path = u'' db_dataset.save() if worms_info_object: archive.setWormsInfoObject(worms_info_object) # archive.createArchiveParts(dataset) # Save generated archive file. generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-EurOBIS.zip') if not os.path.exists(generated_archives_path): os.makedirs(generated_archives_path) archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, settings_dir_path = dwca_config_dir) # Update database after. db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) db_dataset.dwc_archive_eurobis_available = True db_dataset.dwc_archive_eurobis_file_path = os.path.join(generated_archives_path, achive_file_name) db_dataset.save() except Exception as e: error_counter += 1 print(e) admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (eurOBIS format) from: ' + zip_file_name + u'.') # # === Test for GBIF for Sample Data. === # try: # admin_models.addResultLog(logrow_id, result_log = u' - Darwin Core Archive (Sample data format).') # archive = biological_data_exchange_util.DarwinCoreArchiveFormatForSampleData( # datatype, # u'settings_dwca_sampledata.json', # dwca_config_dir, # meta_file_name = u'meta_sampledata.xml', # eml_file_name = u'eml_sampledata.xml', # worms_info_object = worms_info_object) # archive.createArchiveParts(dataset) # # Save generated archive file. # generated_archives_path = os.path.join(settings.APP_DATASETS_FTP_PATH, u'generated_archives') # achive_file_name = zip_file_name.replace(u'.zip', u'_DwC-A-SampleData.zip') # if not os.path.exists(generated_archives_path): # os.makedirs(generated_archives_path) # archive.saveToArchiveFile(achive_file_name, zip_dir_path = generated_archives_path, # settings_dir_path = dwca_config_dir) # # Update database. # db_dataset = models.Datasets.objects.get(dataset_file_name = zip_file_name) # db_dataset.dwc_archive_sampledata_available = True # db_dataset.dwc_archive_sampledata_file_path = os.path.join(generated_archives_path, achive_file_name) # db_dataset.save() # except Exception as e: # error_counter += 1 # admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate DwC-A (Sample data format) from: ' + zip_file_name + u'.') except Exception as e: error_counter += 1 traceback.print_exc() admin_models.addResultLog(logrow_id, result_log = u'ERROR: Failed to generate archive files from: ' + zip_file_name + u'.') # if error_counter > 0: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED (Errors: ' + unicode(error_counter) + u')') else: admin_models.changeLogRowStatus(logrow_id, status = u'FINISHED') if settings.DEBUG: print(u'DEBUG: Archive generation FINISHED')
def validateOneIcesXml(self, logrow_id, error_counter, db_export, user): """ Status = Checked by DC (No DATSU check) --> Approved = True Status = Not checked --> Status = DATSU-ok --> Approved = True Status = Not checked --> Status = DATSU-failed --> Approved = False Status = Test --> Status = Test-DATSU-ok --> Approved = False Status = Test --> Status = Test-DATSU-failed --> Approved = False """ try: # if (db_export.status == 'Checked by DC'): # # Don't perform DATSU check. # db_export.approved = True # db_export.save() # self.append_to_log_file(db_export, datsu_response = None) # el if (db_export.status == 'Not checked'): # http://datsu.ices.dk/DatsuRest/api/ScreenFile/test.sharkdata,se!exportformats!ICES-XML_SMHI_zoobenthos_2005,xml/carlos!ices,dk/zb # http://datsu.ices.dk/DatsuRest/api/ScreenFile/test.sharkdata,se!exportformats!ICES-XML_SMHI_zoobenthos_2014,xml/arnold,andreasson!smhi,se/zb url_part_1 = 'http://datsu.ices.dk/DatsuRest/api/ScreenFile/test,sharkdata,se!exportformats!' url_part_2 = db_export.export_file_name.replace('.', ',') url_part_3 = '/arnold,andreasson!smhi,se' # TODO: [email protected] url_part_4 = '/zb' # if settings.DEBUG: print(url_part_1 + url_part_2 + url_part_3 + url_part_4) # datsu_response_json = urllib2.urlopen(url_part_1 + url_part_2 + url_part_3 + url_part_4) datsu_response = json.load(datsu_response_json) # # For test: # datsu_response = dict({u'SessionID': u'484', u'NoErrors': -1, u'ScreenResultURL': u'datsu.ices.dk/test/ScreenResult.aspx?groupError=0&sessionid=484'}) if settings.DEBUG: print('DEBUG: \n' + json.dumps(datsu_response, sort_keys = True, indent = 2)) # if datsu_response['NoErrors'] == -1: db_export.status = 'DATSU-ok' db_export.approved = True else: db_export.status = 'DATSU-failed' db_export.approved = False # db_export.save() self.append_to_log_file(db_export, datsu_response = datsu_response) elif (db_export.status == 'Test'): # http://datsu.ices.dk/DatsuRest/api/ScreenFile/test.sharkdata,se!exportformats!ICES-XML_SMHI_zoobenthos_2005,xml/carlos!ices,dk/zb # http://datsu.ices.dk/DatsuRest/api/ScreenFile/test.sharkdata,se!exportformats!ICES-XML_SMHI_zoobenthos_2014,xml/arnold,andreasson!smhi,se/zb url_part_1 = 'http://datsu.ices.dk/DatsuRest/api/ScreenFile/test,sharkdata,se!exportformats!' url_part_2 = db_export.export_file_name.replace('.', ',') url_part_3 = '/arnold,andreasson!smhi,se' # TODO: [email protected] url_part_4 = '/zb' # if settings.DEBUG: print(url_part_1 + url_part_2 + url_part_3 + url_part_4) # datsu_response_json = urllib2.urlopen(url_part_1 + url_part_2 + url_part_3 + url_part_4) datsu_response = json.load(datsu_response_json) # # For test: # datsu_response = dict({u'SessionID': u'484', u'NoErrors': -1, u'ScreenResultURL': u'datsu.ices.dk/test/ScreenResult.aspx?groupError=0&sessionid=484'}) if settings.DEBUG: print('DEBUG: \n' + json.dumps(datsu_response, sort_keys = True, indent = 2)) # if datsu_response['NoErrors'] == -1: db_export.status = 'Test-DATSU-ok' db_export.approved = False else: db_export.status = 'Test-DATSU-failed' db_export.approved = False # Logging. admin_models.addResultLog(logrow_id, result_log = 'ERROR: Failed to validating ICES-XML file. DATSU errors URL: ' + datsu_response['ScreenResultURL']) # db_export.save() self.append_to_log_file(db_export, datsu_response = datsu_response) else: if settings.DEBUG: print('DEBUG: ' + db_export.export_file_name + ' ' + db_export.status + ' ' + unicode(db_export.approved)) except Exception as e: error_counter += 1 traceback.print_exc() admin_models.addResultLog(logrow_id, result_log = 'ERROR: Failed to validating ICES-XML files. Exception: ' + unicode(e)) return error_counter