def _process_uploaded_file(resource, validate_files_dict):
log = logging.getLogger()
# find a tif file or a zip file
res_file = None
for r_file in resource.files.all():
if r_file.extension.lower() in ('.tif', '.tiff', '.zip'):
res_file = r_file
break
if res_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
validation_results = raster.raster_file_validation(
raster_file=temp_file, resource=resource)
if not validation_results['error_info']:
log.info("Geo raster file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == os.path.splitext(f)[1]
].pop()
metadata = raster.extract_metadata(temp_vrt_file_path)
# delete the original resource file if it is a zip file
if res_file.extension.lower() == '.zip':
file_name = delete_resource_file_only(resource, res_file)
delete_format_metadata_after_delete_file(resource, file_name)
# add all extracted files (tif and vrt)
for f in validation_results['new_resource_files_to_add']:
uploaded_file = UploadedFile(file=open(f, 'rb'),
name=os.path.basename(f))
utils.add_file_to_resource(resource, uploaded_file)
# use the extracted metadata to populate resource metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
log_msg = "Geo raster resource (ID:{}) - extracted metadata was saved to DB"
log_msg = log_msg.format(resource.short_id)
log.info(log_msg)
else:
# delete all the files in the resource
for res_file in resource.files.all():
delete_resource_file_only(resource, res_file)
validate_files_dict['are_files_valid'] = False
err_msg = "Uploaded file was not added to the resource. "
err_msg += ", ".join(msg
for msg in validation_results['error_info'])
validate_files_dict['message'] = err_msg
log_msg = "File validation failed for raster resource (ID:{})."
log_msg = log_msg.format(resource.short_id)
log.error(log_msg)
# cleanup the temp file directory
if os.path.exists(temp_file):
shutil.rmtree(os.path.dirname(temp_file))
def _process_uploaded_file(resource, res_file, err_msg):
files_failed_validation = []
try:
# validate files and extract metadata
meta_dict, shape_files, shp_res_files = geofeature.extract_metadata_and_files(
resource, res_file, file_type=False)
except Exception as ex:
# need to delete all resource files as these files failed validation
for f in resource.files.all():
files_failed_validation.append(f.file_name)
f.delete()
err_msg = err_msg.format(reason=ex.message,
files=", ".join(files_failed_validation))
raise ValidationError(err_msg)
# upload generated files in case of zip file
if res_file.extension.lower() == '.zip':
with transaction.atomic():
# delete the zip file
res_file.delete()
# uploaded the extracted files
for fl in shape_files:
uploaded_file = UploadedFile(file=open(fl, 'rb'),
name=os.path.basename(fl))
utils.add_file_to_resource(resource, uploaded_file)
# add extracted metadata to the resource
xml_file = ''
for f in shape_files:
if f.lower().endswith('.shp.xml'):
xml_file = f
break
geofeature.add_metadata(resource, meta_dict, xml_file)
def _process_uploaded_file(resource, res_file, err_msg):
files_failed_validation = []
try:
# validate files and extract metadata
meta_dict, shape_files, shp_res_files = geofeature.extract_metadata_and_files(
resource, res_file, file_type=False)
except Exception as ex:
# need to delete all resource files as these files failed validation
for f in resource.files.all():
files_failed_validation.append(f.file_name)
f.delete()
err_msg = err_msg.format(reason=ex.message, files=", ".join(files_failed_validation))
raise ValidationError(err_msg)
# upload generated files in case of zip file
if res_file.extension.lower() == '.zip':
with transaction.atomic():
# delete the zip file
res_file.delete()
# uploaded the extracted files
for fl in shape_files:
uploaded_file = UploadedFile(file=open(fl, 'rb'),
name=os.path.basename(fl))
utils.add_file_to_resource(resource, uploaded_file)
# add extracted metadata to the resource
xml_file = ''
for f in shape_files:
if f.lower().endswith('.shp.xml'):
xml_file = f
break
geofeature.add_metadata(resource, meta_dict, xml_file)
def _process_uploaded_file(resource, validate_files_dict):
log = logging.getLogger()
# find a tif file or a zip file
res_file = None
for r_file in resource.files.all():
if r_file.extension.lower() in ('.tif', '.tiff', '.zip'):
res_file = r_file
break
if res_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
validation_results = raster.raster_file_validation(raster_file=temp_file,
resource=resource)
if not validation_results['error_info']:
log.info("Geo raster file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if
'.vrt' == os.path.splitext(f)[1]].pop()
metadata = raster.extract_metadata(temp_vrt_file_path)
# delete the original resource file if it is a zip file
if res_file.extension.lower() == '.zip':
file_name = delete_resource_file_only(resource, res_file)
delete_format_metadata_after_delete_file(resource, file_name)
# add all extracted files (tif and vrt)
for f in validation_results['new_resource_files_to_add']:
uploaded_file = UploadedFile(file=open(f, 'rb'),
name=os.path.basename(f))
utils.add_file_to_resource(resource, uploaded_file)
# use the extracted metadata to populate resource metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
log_msg = "Geo raster resource (ID:{}) - extracted metadata was saved to DB"
log_msg = log_msg.format(resource.short_id)
log.info(log_msg)
else:
# delete all the files in the resource
for res_file in resource.files.all():
delete_resource_file_only(resource, res_file)
validate_files_dict['are_files_valid'] = False
err_msg = "Uploaded file was not added to the resource. "
err_msg += ", ".join(msg for msg in validation_results['error_info'])
validate_files_dict['message'] = err_msg
log_msg = "File validation failed for raster resource (ID:{})."
log_msg = log_msg.format(resource.short_id)
log.error(log_msg)
# cleanup the temp file directory
if os.path.exists(temp_file):
shutil.rmtree(os.path.dirname(temp_file))
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in HydroShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique HydroShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects added to the resource
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
This does **not** handle mutability; changes to immutable resources should be denied elsewhere.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
source_names = kwargs.pop('source_names', [])
if __debug__:
assert(isinstance(source_names, list))
move = kwargs.pop('move', False)
folder = kwargs.pop('folder', None)
if __debug__: # assure that there are no spurious kwargs left.
for k in kwargs:
print("kwargs[{}]".format(k))
assert len(kwargs) == 0
for f in files:
ret.append(utils.add_file_to_resource(resource, f, folder=folder))
if len(source_names) > 0:
for ifname in source_names:
ret.append(utils.add_file_to_resource(resource, None,
folder=folder,
source_name=ifname,
move=move))
if not ret:
# no file has been added, make sure data/contents directory exists if no file is added
utils.create_empty_contents_directory(resource)
else:
# some file(s) added, need to update quota usage
update_quota_usage(resource)
return ret
def add_zip_file_contents_to_resource(pk, zip_file_path):
"""Add zip file to existing resource and remove tmp zip file."""
zfile = None
resource = None
try:
resource = utils.get_resource_by_shortkey(pk, or_404=False)
zfile = zipfile.ZipFile(zip_file_path)
num_files = len(zfile.infolist())
zcontents = utils.ZipContents(zfile)
files = zcontents.get_files()
resource.file_unpack_status = 'Running'
resource.save()
for i, f in enumerate(files):
logger.debug("Adding file {0} to resource {1}".format(f.name, pk))
utils.add_file_to_resource(resource, f)
resource.file_unpack_message = "Imported {0} of about {1} file(s) ...".format(
i, num_files)
resource.save()
# This might make the resource unsuitable for public consumption
resource.update_public_and_discoverable()
# TODO: this is a bit of a lie because a different user requested the bag overwrite
utils.resource_modified(resource,
resource.creator,
overwrite_bag=False)
# Call success callback
resource.file_unpack_message = None
resource.file_unpack_status = 'Done'
resource.save()
except BaseResource.DoesNotExist:
msg = "Unable to add zip file contents to non-existent resource {pk}."
msg = msg.format(pk=pk)
logger.error(msg)
except:
exc_info = "".join(traceback.format_exception(*sys.exc_info()))
if resource:
resource.file_unpack_status = 'Error'
resource.file_unpack_message = exc_info
resource.save()
if zfile:
zfile.close()
logger.error(exc_info)
finally:
# Delete upload file
os.unlink(zip_file_path)
def add_zip_file_contents_to_resource(pk, zip_file_path):
"""Add zip file to existing resource and remove tmp zip file."""
zfile = None
resource = None
try:
resource = utils.get_resource_by_shortkey(pk, or_404=False)
zfile = zipfile.ZipFile(zip_file_path)
num_files = len(zfile.infolist())
zcontents = utils.ZipContents(zfile)
files = zcontents.get_files()
resource.file_unpack_status = 'Running'
resource.save()
for i, f in enumerate(files):
logger.debug("Adding file {0} to resource {1}".format(f.name, pk))
utils.add_file_to_resource(resource, f)
resource.file_unpack_message = "Imported {0} of about {1} file(s) ...".format(
i, num_files)
resource.save()
# This might make the resource unsuitable for public consumption
resource.update_public_and_discoverable()
# TODO: this is a bit of a lie because a different user requested the bag overwrite
utils.resource_modified(resource, resource.creator, overwrite_bag=False)
# Call success callback
resource.file_unpack_message = None
resource.file_unpack_status = 'Done'
resource.save()
except BaseResource.DoesNotExist:
msg = "Unable to add zip file contents to non-existent resource {pk}."
msg = msg.format(pk=pk)
logger.error(msg)
except:
exc_info = "".join(traceback.format_exception(*sys.exc_info()))
if resource:
resource.file_unpack_status = 'Error'
resource.file_unpack_message = exc_info
resource.save()
if zfile:
zfile.close()
logger.error(exc_info)
finally:
# Delete upload file
os.unlink(zip_file_path)
def add_file_to_resource(self, file_to_add, upload_folder=None):
file_to_upload = UploadedFile(file=open(file_to_add, 'rb'),
name=os.path.basename(file_to_add))
new_res_file = add_file_to_resource(
self.composite_resource, file_to_upload, folder=upload_folder, check_target_folder=True
)
return new_res_file
def add_file_to_resource(self, file_to_add, upload_folder=None):
file_to_upload = UploadedFile(file=open(file_to_add, 'rb'),
name=os.path.basename(file_to_add))
new_res_file = add_file_to_resource(
self.composite_resource, file_to_upload, folder=upload_folder, check_target_folder=True
)
return new_res_file
def add_zip_file_contents_to_resource(pk, zip_file_path):
zfile = None
resource = None
try:
resource = utils.get_resource_by_shortkey(pk, or_404=False)
zfile = zipfile.ZipFile(zip_file_path)
num_files = len(zfile.infolist())
zcontents = utils.ZipContents(zfile)
files = zcontents.get_files()
resource.file_unpack_status = 'Running'
resource.save()
for i, f in enumerate(files):
logger.debug("Adding file {0} to resource {1}".format(f.name, pk))
utils.add_file_to_resource(resource, f)
resource.file_unpack_message = "Imported {0} of about {1} file(s) ...".format(
i, num_files)
resource.save()
# Call success callback
resource.file_unpack_message = None
resource.file_unpack_status = 'Done'
resource.save()
except BaseResource.DoesNotExist:
msg = "Unable to add zip file contents to non-existent resource {pk}."
msg = msg.format(pk=pk)
logger.error(msg)
except:
exc_info = "".join(traceback.format_exception(*sys.exc_info()))
if resource:
resource.file_unpack_status = 'Error'
resource.file_unpack_message = exc_info
resource.save()
if zfile:
zfile.close()
logger.error(exc_info)
finally:
# Delete upload file
os.unlink(zip_file_path)
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
# base file name (no path included)
file_name = utils.get_resource_file_name_and_extension(res_file)[1]
# file name without the extension
file_name = file_name[:-len(res_file.extension)]
file_folder = res_file.file_folder
upload_folder = ''
if res_file is not None and res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
# validate the file
error_info, files_to_add_to_resource = raster_file_validation(
raster_file=temp_file)
if not error_info:
log.info("Geo raster file type file validation successful.")
# extract metadata
temp_dir = os.path.dirname(temp_file)
temp_vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == os.path.splitext(f)[1]
].pop()
metadata = extract_metadata(temp_vrt_file_path)
log.info(
"Geo raster file type metadata extraction was successful.")
with transaction.atomic():
# create a geo raster logical file object to be associated with resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = file_name
logical_file.save()
try:
# create a folder for the raster file type using the base file name as the
# name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, file_name)
log.info("Folder created:{}".format(new_folder_path))
create_folder(resource.short_id, new_folder_path)
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
# the added resource file will be part of a new generic logical file
# by default
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# delete the generic logical file object
if new_res_file.logical_file is not None:
# deleting the file level metadata object will delete the associated
# logical file object
new_res_file.logical_file.metadata.delete()
# make each resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"Geo raster file type - new files were added to the resource."
)
# use the extracted metadata to populate file metadata
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
logical_file.metadata.create_element(k, **v)
log.info(
"Geo raster file type - metadata was saved to DB")
# set resource to private if logical file is missing required metadata
resource.update_public_and_discoverable()
# delete the original resource file
delete_resource_file(resource.short_id, res_file.id,
user)
log.info("Deleted original resource file.")
except Exception as ex:
msg = "Geo raster file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
if upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join(
'data', 'contents', upload_folder)
remove_folder(user, resource.short_id,
folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
else:
err_msg = "Geo raster file type file validation failed.{}".format(
' '.join(error_info))
log.info(err_msg)
raise ValidationError(err_msg)
else:
if res_file is None:
err_msg = "Failed to set Geo raster file type. " \
"Resource doesn't have the specified file."
log.error(err_msg)
raise ValidationError(err_msg)
else:
err_msg = "Failed to set Geo raster file type." \
"The specified file doesn't have a generic logical file type."
log.error(err_msg)
raise ValidationError(err_msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a tif or zip raster resource file to GeoRasterFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoRasterFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None:
raise ValidationError("File not found.")
if res_file.extension != '.nc':
raise ValidationError("Not a NetCDF file.")
# base file name (no path included)
file_name = res_file.file_name
# file name without the extension
nc_file_name = file_name.split(".")[0]
resource_metadata = []
file_type_metadata = []
files_to_add_to_resource = []
if res_file.has_generic_logical_file:
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
files_to_add_to_resource.append(temp_file)
# file validation and metadata extraction
nc_dataset = nc_utils.get_nc_dataset(temp_file)
if isinstance(nc_dataset, netCDF4.Dataset):
# Extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(
temp_file)
# populate resource_metadata and file_type_metadata lists with extracted metadata
add_metadata_to_list(resource_metadata, res_dublin_core_meta,
res_type_specific_meta,
file_type_metadata, resource)
# create the ncdump text file
dump_file = create_header_info_txt_file(
temp_file, nc_file_name)
files_to_add_to_resource.append(dump_file)
file_folder = res_file.file_folder
with transaction.atomic():
# first delete the netcdf file that we retrieved from irods
# for setting it to netcdf file type
delete_resource_file(resource.short_id, res_file.id, user)
# create a netcdf logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type unless the extracted metadata
# has a value for title
dataset_title = res_dublin_core_meta.get('title', None)
if dataset_title is not None:
logical_file.dataset_name = dataset_title
else:
logical_file.dataset_name = nc_file_name
logical_file.save()
try:
# create a folder for the netcdf file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, nc_file_name)
fed_file_full_path = ''
if resource.resource_federation_path:
fed_file_full_path = os.path.join(
resource.root_path, new_folder_path)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(
file_folder, new_folder_name)
# add all new files to the resource
for f in files_to_add_to_resource:
uploaded_file = UploadedFile(
file=open(f, 'rb'), name=os.path.basename(f))
new_res_file = utils.add_file_to_resource(
resource,
uploaded_file,
folder=upload_folder,
fed_res_file_name_or_path=fed_file_full_path)
# make each resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"NetCDF file type - new files were added to the resource."
)
except Exception as ex:
msg = "NetCDF file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
# TODO: in case of any error put the original file back and
# delete the folder that was created
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
log.info("NetCDF file type was created.")
# use the extracted metadata to populate resource metadata
for element in resource_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element(
'title', title_element.id, **v)
else:
resource.metadata.create_element(k, **v)
log.info("Resource - metadata was saved to DB")
# use the extracted metadata to populate file metadata
for element in file_type_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'subject':
logical_file.metadata.keywords = v
logical_file.metadata.save()
else:
logical_file.metadata.create_element(k, **v)
log.info("NetCDF file type - metadata was saved to DB")
else:
err_msg = "Not a valid NetCDF file. File type file validation failed."
log.error(err_msg)
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
raise ValidationError(err_msg)
def set_file_type(cls, resource, user, file_id=None, folder_path=None):
""" Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file
or a folder """
log = logging.getLogger()
res_file, folder_path = cls._validate_set_file_type_inputs(resource, file_id, folder_path)
# base file name (no path included)
file_name = res_file.file_name
# file name without the extension - needed for naming the new aggregation folder
nc_file_name = file_name[:-len(res_file.extension)]
resource_metadata = []
file_type_metadata = []
upload_folder = ''
res_files_to_delete = []
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
# file validation and metadata extraction
nc_dataset = nc_utils.get_nc_dataset(temp_file)
if isinstance(nc_dataset, netCDF4.Dataset):
msg = "NetCDF aggregation. Error when creating aggregation. Error:{}"
file_type_success = False
# extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(temp_file)
# populate resource_metadata and file_type_metadata lists with extracted metadata
add_metadata_to_list(resource_metadata, res_dublin_core_meta,
res_type_specific_meta, file_type_metadata, resource)
# create the ncdump text file
dump_file = create_header_info_txt_file(temp_file, nc_file_name)
file_folder = res_file.file_folder
aggregation_folder_created = False
create_new_folder = cls._check_create_aggregation_folder(
selected_res_file=res_file, selected_folder=folder_path,
aggregation_file_count=1)
with transaction.atomic():
# create a netcdf logical file object to be associated with
# resource files
dataset_title = res_dublin_core_meta.get('title', nc_file_name)
logical_file = cls.initialize(dataset_title, resource)
try:
if folder_path is None:
# we are here means aggregation is being created by selecting a file
# create a folder for the netcdf file type using the base file
# name as the name for the new folder if the file is not already in a folder
if create_new_folder:
upload_folder = cls._create_aggregation_folder(resource, file_folder,
nc_file_name)
aggregation_folder_created = True
log.info("NetCDF Aggregation creation - folder created:{}".format(
upload_folder))
else:
# selected nc file is already in a folder
upload_folder = file_folder
# create logical file record in DB
logical_file.save()
if aggregation_folder_created:
# copy the nc file to the new aggregation folder and make it part
# of the logical file
tgt_folder = upload_folder
files_to_copy = [res_file]
logical_file.copy_resource_files(resource, files_to_copy,
tgt_folder)
res_files_to_delete.append(res_file)
else:
# make the selected nc file as part of the aggregation/file type
logical_file.add_resource_file(res_file)
else:
# logical file record gets created in DB
logical_file.save()
# folder has been selected to create aggregation
upload_folder = folder_path
# make the .nc file part of the aggregation
logical_file.add_resource_file(res_file)
# add the new dump txt file to the resource
uploaded_file = UploadedFile(file=open(dump_file, 'rb'),
name=os.path.basename(dump_file))
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder, add_to_aggregation=False
)
# make this new resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info("NetCDF aggregation creation - a new file was added to the resource.")
# use the extracted metadata to populate resource metadata
for element in resource_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element('title', title_element.id, **v)
else:
resource.metadata.create_element(k, **v)
log.info("NetCDF Aggregation creation - Resource metadata was saved to DB")
# use the extracted metadata to populate file metadata
for element in file_type_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'subject':
logical_file.metadata.keywords = v
logical_file.metadata.save()
# update resource level keywords
resource_keywords = [subject.value.lower() for subject in
resource.metadata.subjects.all()]
for kw in logical_file.metadata.keywords:
if kw.lower() not in resource_keywords:
resource.metadata.create_element('subject', value=kw)
else:
logical_file.metadata.create_element(k, **v)
log.info("NetCDF aggregation - metadata was saved in aggregation")
logical_file._finalize(user, resource,
folder_created=aggregation_folder_created,
res_files_to_delete=res_files_to_delete)
file_type_success = True
post_add_netcdf_aggregation.send(
sender=AbstractLogicalFile,
resource=resource,
file=logical_file
)
except Exception as ex:
msg = msg.format(ex.message)
log.exception(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
if not file_type_success:
aggregation_from_folder = folder_path is not None
cls._cleanup_on_fail_to_create_aggregation(user, resource, upload_folder,
file_folder, aggregation_from_folder)
raise ValidationError(msg)
else:
err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed."
log.error(err_msg)
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
raise ValidationError(err_msg)
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in CommonsShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique CommonsShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects added to the resource
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
This does **not** handle mutability; changes to immutable resources should be denied elsewhere.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
source_names = kwargs.pop('source_names', [])
source_sizes = kwargs.pop('source_sizes', [])
is_file_reference = kwargs.pop('is_file_reference', False)
if __debug__:
assert (isinstance(source_names, list))
move = kwargs.pop('move', False)
folder = kwargs.pop('folder', None)
if __debug__: # assure that there are no spurious kwargs left.
for k in kwargs:
print("kwargs[{}]".format(k))
assert len(kwargs) == 0
for f in files:
ret.append(utils.add_file_to_resource(resource, f, folder=folder))
if len(source_names) > 0:
if len(source_names) != len(source_sizes):
# if length is not equal, there is an issue with source_sizes input parameter, so it will not be
# used by setting it to be empty
source_sizes = []
idx = 0
for ifname in source_names:
s_size = source_sizes[idx] if source_sizes else 0
idx += 1
ret.append(
utils.add_file_to_resource(
resource,
None,
folder=folder,
source_name=ifname,
source_size=s_size,
move=move,
is_file_reference=is_file_reference))
# make sure data/contents directory exists if not exist already
utils.create_empty_contents_directory(resource)
return ret
def netcdf_post_create_resource(sender, **kwargs):
log = logging.getLogger()
resource = kwargs['resource']
validate_files_dict = kwargs['validate_files']
res_file = resource.files.all().first()
if res_file:
temp_file = utils.get_file_from_irods(res_file)
nc_dataset = nc_utils.get_nc_dataset(temp_file)
nc_file_name = res_file.file_name
if isinstance(nc_dataset, netCDF4.Dataset):
# Extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(
temp_file)
# populate metadata list with extracted metadata
metadata = []
add_metadata_to_list(metadata, res_dublin_core_meta,
res_type_specific_meta)
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = list(element.items())[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element('title', title_element.id,
**v)
elif k == 'rights':
rights_element = resource.metadata.rights
resource.metadata.update_element('rights',
rights_element.id, **v)
elif k == 'creator':
resource.metadata.creators.all().delete()
resource.metadata.create_element('creator', **v)
else:
resource.metadata.create_element(k, **v)
# create the ncdump text file
dump_file = create_header_info_txt_file(temp_file, nc_file_name)
dump_file_name = nc_file_name + '_header_info.txt'
uploaded_file = UploadedFile(file=open(dump_file, mode="rb"),
name=dump_file_name)
utils.add_file_to_resource(resource, uploaded_file)
else:
delete_resource_file_only(resource, res_file)
validate_files_dict['are_files_valid'] = False
err_msg = "Uploaded file was not added to the resource." \
" Please provide a valid NetCDF file. "
validate_files_dict['message'] = err_msg
log_msg = "File validation failed for netcdf resource (ID:{})."
log_msg = log_msg.format(resource.short_id)
log.error(log_msg)
# cleanup the temp file directory
if os.path.exists(temp_file):
shutil.rmtree(os.path.dirname(temp_file))
# set metadata is dirty flag as false for resource creation
metadata = resource.metadata
metadata.is_dirty = False
metadata.save()
# since we are extracting metadata after resource creation
# metadata xml files need to be regenerated - so need to set the
# dirty bag flags
if resource.files.all().count() > 0:
utils.set_dirty_bag_flag(resource)
def netcdf_post_create_resource(sender, **kwargs):
log = logging.getLogger()
resource = kwargs['resource']
validate_files_dict = kwargs['validate_files']
res_file = resource.files.all().first()
if res_file:
temp_file = utils.get_file_from_irods(res_file)
nc_dataset = nc_utils.get_nc_dataset(temp_file)
nc_file_name = res_file.file_name
if isinstance(nc_dataset, netCDF4.Dataset):
# Extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(temp_file)
# populate metadata list with extracted metadata
metadata = []
add_metadata_to_list(metadata, res_dublin_core_meta, res_type_specific_meta)
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element('title', title_element.id, **v)
elif k == 'rights':
rights_element = resource.metadata.rights
resource.metadata.update_element('rights', rights_element.id, **v)
elif k == 'creator':
resource.metadata.creators.all().delete()
resource.metadata.create_element('creator', **v)
else:
resource.metadata.create_element(k, **v)
# create the ncdump text file
dump_file = create_header_info_txt_file(temp_file, nc_file_name)
dump_file_name = nc_file_name + '_header_info.txt'
uploaded_file = UploadedFile(file=open(dump_file), name=dump_file_name)
utils.add_file_to_resource(resource, uploaded_file)
else:
delete_resource_file_only(resource, res_file)
validate_files_dict['are_files_valid'] = False
err_msg = "Uploaded file was not added to the resource." \
" Please provide a valid NetCDF file. "
validate_files_dict['message'] = err_msg
log_msg = "File validation failed for netcdf resource (ID:{})."
log_msg = log_msg.format(resource.short_id)
log.error(log_msg)
# cleanup the temp file directory
if os.path.exists(temp_file):
shutil.rmtree(os.path.dirname(temp_file))
# set metadata is dirty flag as false for resource creation
metadata = resource.metadata
metadata.is_dirty = False
metadata.save()
# since we are extracting metadata after resource creation
# metadata xml files need to be regenerated - so need to set the
# dirty bag flags
if resource.files.all().count() > 0:
utils.set_dirty_bag_flag(resource)
def set_file_type(cls, resource, user, file_id=None, folder_path=None):
""" Creates a NetCDFLogicalFile (aggregation) from a netcdf file (.nc) resource file
or a folder """
log = logging.getLogger()
res_file, folder_path = cls._validate_set_file_type_inputs(
resource, file_id, folder_path)
# base file name (no path included)
file_name = res_file.file_name
# file name without the extension - needed for naming the new aggregation folder
nc_file_name = file_name[:-len(res_file.extension)]
resource_metadata = []
file_type_metadata = []
upload_folder = ''
res_files_to_delete = []
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
# file validation and metadata extraction
nc_dataset = nc_utils.get_nc_dataset(temp_file)
if isinstance(nc_dataset, netCDF4.Dataset):
msg = "NetCDF aggregation. Error when creating aggregation. Error:{}"
file_type_success = False
# extract the metadata from netcdf file
res_dublin_core_meta, res_type_specific_meta = nc_meta.get_nc_meta_dict(
temp_file)
# populate resource_metadata and file_type_metadata lists with extracted metadata
add_metadata_to_list(resource_metadata, res_dublin_core_meta,
res_type_specific_meta, file_type_metadata,
resource)
# create the ncdump text file
dump_file = create_header_info_txt_file(temp_file, nc_file_name)
file_folder = res_file.file_folder
aggregation_folder_created = False
create_new_folder = cls._check_create_aggregation_folder(
selected_res_file=res_file,
selected_folder=folder_path,
aggregation_file_count=1)
with transaction.atomic():
# create a netcdf logical file object to be associated with
# resource files
dataset_title = res_dublin_core_meta.get('title', nc_file_name)
logical_file = cls.initialize(dataset_title, resource)
try:
if folder_path is None:
# we are here means aggregation is being created by selecting a file
# create a folder for the netcdf file type using the base file
# name as the name for the new folder if the file is not already in a folder
if create_new_folder:
upload_folder = cls._create_aggregation_folder(
resource, file_folder, nc_file_name)
aggregation_folder_created = True
log.info(
"NetCDF Aggregation creation - folder created:{}"
.format(upload_folder))
else:
# selected nc file is already in a folder
upload_folder = file_folder
# create logical file record in DB
logical_file.save()
if aggregation_folder_created:
# copy the nc file to the new aggregation folder and make it part
# of the logical file
tgt_folder = upload_folder
files_to_copy = [res_file]
logical_file.copy_resource_files(
resource, files_to_copy, tgt_folder)
res_files_to_delete.append(res_file)
else:
# make the selected nc file as part of the aggregation/file type
logical_file.add_resource_file(res_file)
else:
# logical file record gets created in DB
logical_file.save()
# folder has been selected to create aggregation
upload_folder = folder_path
# make the .nc file part of the aggregation
logical_file.add_resource_file(res_file)
# add the new dump txt file to the resource
uploaded_file = UploadedFile(
file=open(dump_file, 'rb'),
name=os.path.basename(dump_file))
new_res_file = utils.add_file_to_resource(
resource,
uploaded_file,
folder=upload_folder,
add_to_aggregation=False)
# make this new resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"NetCDF aggregation creation - a new file was added to the resource."
)
# use the extracted metadata to populate resource metadata
for element in resource_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'title':
# update title element
title_element = resource.metadata.title
resource.metadata.update_element(
'title', title_element.id, **v)
else:
resource.metadata.create_element(k, **v)
log.info(
"NetCDF Aggregation creation - Resource metadata was saved to DB"
)
# use the extracted metadata to populate file metadata
for element in file_type_metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
if k == 'subject':
logical_file.metadata.keywords = v
logical_file.metadata.save()
# update resource level keywords
resource_keywords = [
subject.value.lower() for subject in
resource.metadata.subjects.all()
]
for kw in logical_file.metadata.keywords:
if kw.lower() not in resource_keywords:
resource.metadata.create_element('subject',
value=kw)
else:
logical_file.metadata.create_element(k, **v)
log.info(
"NetCDF aggregation - metadata was saved in aggregation"
)
logical_file._finalize(
user,
resource,
folder_created=aggregation_folder_created,
res_files_to_delete=res_files_to_delete)
file_type_success = True
except Exception as ex:
msg = msg.format(ex.message)
log.exception(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
if not file_type_success:
aggregation_from_folder = folder_path is not None
cls._cleanup_on_fail_to_create_aggregation(
user, resource, upload_folder, file_folder,
aggregation_from_folder)
raise ValidationError(msg)
else:
err_msg = "Not a valid NetCDF file. NetCDF aggregation validation failed."
log.error(err_msg)
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
raise ValidationError(err_msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a .shp or .zip resource file to GeoFeatureFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as GeoFeatureFile type
:param user: user who is setting the file type
:return:
"""
# had to import it here to avoid import loop
from hs_core.views.utils import create_folder, remove_folder
log = logging.getLogger()
# get the file from irods
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None or not res_file.exists:
raise ValidationError("File not found.")
if res_file.extension.lower() not in ('.zip', '.shp'):
raise ValidationError("Not a valid geographic feature file.")
if not res_file.has_generic_logical_file:
raise ValidationError(
"Selected file must be part of a generic file type.")
try:
meta_dict, shape_files, shp_res_files = extract_metadata_and_files(
resource, res_file)
except ValidationError as ex:
log.exception(ex.message)
raise ex
# hold on to temp dir for final clean up
temp_dir = os.path.dirname(shape_files[0])
file_name = res_file.file_name
# file name without the extension
base_file_name = file_name[:-len(res_file.extension)]
xml_file = ''
for f in shape_files:
if f.lower().endswith('.shp.xml'):
xml_file = f
break
file_folder = res_file.file_folder
file_type_success = False
upload_folder = ''
msg = "GeoFeature file type. Error when setting file type. Error:{}"
with transaction.atomic():
# create a GeoFeature logical file object to be associated with
# resource files
logical_file = cls.create()
# by default set the dataset_name attribute of the logical file to the
# name of the file selected to set file type
logical_file.dataset_name = base_file_name
logical_file.save()
try:
# create a folder for the geofeature file type using the base file
# name as the name for the new folder
new_folder_path = cls.compute_file_type_folder(
resource, file_folder, base_file_name)
create_folder(resource.short_id, new_folder_path)
log.info("Folder created:{}".format(new_folder_path))
new_folder_name = new_folder_path.split('/')[-1]
if file_folder is None:
upload_folder = new_folder_name
else:
upload_folder = os.path.join(file_folder, new_folder_name)
# add all new files to the resource
files_to_add_to_resource = shape_files
for fl in files_to_add_to_resource:
uploaded_file = UploadedFile(file=open(fl, 'rb'),
name=os.path.basename(fl))
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=upload_folder)
# make each resource file we added part of the logical file
logical_file.add_resource_file(new_res_file)
log.info(
"GeoFeature file type - files were added to the file type."
)
add_metadata(resource, meta_dict, xml_file, logical_file)
log.info(
"GeoFeature file type and resource level metadata updated."
)
# delete the original resource files used as part of setting file type
for fl in shp_res_files:
delete_resource_file(resource.short_id, fl.id, user)
log.info("Deleted original resource files.")
file_type_success = True
except Exception as ex:
msg = msg.format(ex.message)
log.exception(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
if not file_type_success and upload_folder:
# delete any new files uploaded as part of setting file type
folder_to_remove = os.path.join('data', 'contents', upload_folder)
remove_folder(user, resource.short_id, folder_to_remove)
log.info("Deleted newly created file type folder")
raise ValidationError(msg)
def set_file_type(cls, resource, file_id, user):
"""
Sets a json resource file to RefTimeseriesFile type
:param resource: an instance of resource type CompositeResource
:param file_id: id of the resource file to be set as RefTimeSeriesFile type
:param user: user who is setting the file type
:return:
"""
log = logging.getLogger()
# get the the selected resource file object
res_file = utils.get_resource_file_by_id(resource, file_id)
if res_file is None:
raise ValidationError("File not found.")
if res_file.extension != '.refts':
raise ValidationError("Not a Ref Time Series file.")
files_to_add_to_resource = []
if res_file.has_generic_logical_file:
try:
json_file_content = _validate_json_file(res_file)
except Exception as ex:
raise ValidationError(ex.message)
# get the file from irods to temp dir
temp_file = utils.get_file_from_irods(res_file)
temp_dir = os.path.dirname(temp_file)
files_to_add_to_resource.append(temp_file)
file_folder = res_file.file_folder
with transaction.atomic():
# first delete the json file that we retrieved from irods
# for setting it to reftimeseries file type
delete_resource_file(resource.short_id, res_file.id, user)
# create a reftiemseries logical file object to be associated with
# resource files
logical_file = cls.create()
logical_file.metadata.json_file_content = json_file_content
logical_file.metadata.save()
try:
# add the json file back to the resource
uploaded_file = UploadedFile(
file=open(temp_file, 'rb'),
name=os.path.basename(temp_file))
# the added resource file will be part of a new generic logical file by default
new_res_file = utils.add_file_to_resource(
resource, uploaded_file, folder=file_folder)
# delete the generic logical file object
if new_res_file.logical_file is not None:
# deleting the file level metadata object will delete the associated
# logical file object
new_res_file.logical_file.metadata.delete()
# make the resource file we added as part of the logical file
logical_file.add_resource_file(new_res_file)
logical_file.metadata.save()
logical_file.dataset_name = logical_file.metadata.get_title_from_json(
)
logical_file.save()
# extract metadata
_extract_metadata(resource, logical_file)
log.info(
"RefTimeseries file type - json file was added to the resource."
)
except Exception as ex:
msg = "RefTimeseries file type. Error when setting file type. Error:{}"
msg = msg.format(ex.message)
log.exception(msg)
raise ValidationError(msg)
finally:
# remove temp dir
if os.path.isdir(temp_dir):
shutil.rmtree(temp_dir)
log.info("RefTimeseries file type was created.")
else:
err_msg = "Selected file is not part of a GenericLogical file."
log.error(err_msg)
raise ValidationError(err_msg)
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in HydroShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique HydroShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects added to the resource
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
This does **not** handle mutability; changes to immutable resources should be denied elsewhere.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
source_names = kwargs.pop('source_names', [])
full_paths = kwargs.pop('full_paths', {})
auto_aggregate = kwargs.pop('auto_aggregate', True)
if __debug__:
assert(isinstance(source_names, list))
folder = kwargs.pop('folder', None)
if __debug__: # assure that there are no spurious kwargs left.
for k in kwargs:
print("kwargs[{}]".format(k))
assert len(kwargs) == 0
prefix_path = 'data/contents'
if folder is None or folder == prefix_path:
base_dir = ""
elif folder.startswith(prefix_path):
base_dir = folder[len(prefix_path) + 1:]
else:
base_dir = folder
new_folders = set()
for f in files:
full_dir = base_dir
if f in full_paths:
# TODO, put this in it's own method?
full_path = full_paths[f]
dir_name = os.path.dirname(full_path)
# Only do join if dir_name is not empty, otherwise, it'd result in a trailing slash
full_dir = os.path.join(base_dir, dir_name) if dir_name else base_dir
if full_dir:
new_folders.add(os.path.join(resource.file_path, full_dir))
ret.append(utils.add_file_to_resource(resource, f, folder=full_dir))
else:
ret.append(utils.add_file_to_resource(resource, f, folder=None))
if len(source_names) > 0:
for ifname in source_names:
ret.append(utils.add_file_to_resource(resource, None,
folder=folder,
source_name=ifname))
if not ret:
# no file has been added, make sure data/contents directory exists if no file is added
utils.create_empty_contents_directory(resource)
else:
if resource.resource_type == "CompositeResource" and auto_aggregate:
utils.check_aggregations(resource, new_folders, ret)
# some file(s) added, need to update quota usage
update_quota_usage(res=resource)
return ret
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in HydroShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique HydroShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
For mutable resources (resources not formally published), the update adds the file that is
passed to this method to the resource. For immutable resources (formally published resources),
this method creates a new resource that is a new version of the formally published resource.
HydroShare will record the update by storing the SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy fields for the respective resources in their system metadata
HydroShare MUST check or set the values of SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new
and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified
system metadata entries must then be available in HydroShare.listObjects() to ensure that any
cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified.
A formally published resource can only be obsoleted by one newer version. Once a resource is
obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
fed_res_file_names = kwargs.pop('fed_res_file_names', '')
ret = []
fed_zone_home_path = kwargs.pop('fed_zone_home_path', '')
# for adding files to existing resources, the default action is copy
fed_copy_or_move = kwargs.pop('fed_copy_or_move', 'copy')
folder = kwargs.pop('folder', None)
for f in files:
if fed_zone_home_path:
# user has selected files from a federated iRODS zone, so files uploaded from local disk
# need to be stored to the federated iRODS zone rather than HydroShare zone as well
# TODO: why do we allow the federation prefix to vary in this statement?
ret.append(utils.add_file_to_resource(resource, f, folder=folder,
fed_res_file_name_or_path=fed_zone_home_path))
elif resource.resource_federation_path:
# file needs to be added to a resource in a federated zone
# TODO: why do we allow the federation prefix to vary in this statement?
ret.append(utils.add_file_to_resource(
resource, f, folder=folder,
fed_res_file_name_or_path=resource.resource_federation_path))
else:
ret.append(utils.add_file_to_resource(resource, f, folder=folder))
if fed_res_file_names:
if isinstance(fed_res_file_names, basestring):
ifnames = string.split(fed_res_file_names, ',')
elif isinstance(fed_res_file_names, list):
ifnames = fed_res_file_names
else:
return ret
for ifname in ifnames:
ret.append(utils.add_file_to_resource(resource, None, folder=folder,
fed_res_file_name_or_path=ifname,
fed_copy_or_move=fed_copy_or_move))
if not ret:
# no file has been added, make sure data/contents directory exists if no file is added
utils.create_empty_contents_directory(resource)
return ret
