def handle(self, *args, **options):
if len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
try:
resource = get_resource_by_shortkey(rid)
except BaseResource.DoesNotExist:
msg = "Resource with id {} not found in Django Resources".format(rid)
print(msg)
continue
print("LOOKING FOR RELATION ERRORS FOR RESOURCE {}".format(rid))
check_relations(resource)
else: # check all resources
print("LOOKING FOR RELATION ERRORS FOR ALL RESOURCES")
for r in BaseResource.objects.all():
try:
resource = get_resource_by_shortkey(r.short_id)
except BaseResource.DoesNotExist:
msg = "Resource with id {} not found in Django Resources".format(rid)
print(msg)
continue
print("LOOKING FOR RELATION ERRORS FOR RESOURCE {}".format(rid))
check_relations(resource)
def handle(self, *args, **options):
logger = logging.getLogger(__name__)
log_errors = options['log']
echo_errors = not options['log']
if len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
try:
resource = get_resource_by_shortkey(rid)
except BaseResource.NotFoundException:
msg = "resource {} not found".format(rid)
print(msg)
continue
_, count = repair_resource(resource,
echo_errors=echo_errors,
log_errors=log_errors,
return_errors=False)
if count:
msg = "... affected resource {} has type {}, title '{}'"\
.format(resource.short_id, resource.resource_type,
resource.title.encode('ascii', 'replace'))
if log_errors:
logger.info(msg)
if echo_errors:
print(msg)
else: # check all resources
print("REPAIRING ALL RESOURCES")
for r in BaseResource.objects.all():
try:
resource = get_resource_by_shortkey(r.short_id)
except BaseResource.NotFoundException:
msg = "resource {} not found".format(rid)
print(msg)
continue
_, count = repair_resource(resource,
echo_errors=echo_errors,
log_errors=log_errors,
return_errors=False)
if count:
msg = "... affected resource {} has type {}, title '{}'"\
.format(resource.short_id, resource.resource_type,
resource.title.encode('ascii', 'replace'))
if log_errors:
logger.info(msg)
if echo_errors:
print(msg)
def test_resource_create_with_core_and_extra_metadata(self):
rtype = 'GenericResource'
title = 'My Test resource'
metadata = []
metadata.append({'coverage': {'type': 'period', 'value': {'start': '01/01/2000',
'end': '12/12/2010'}}})
extra_metadata = {'latitude': '40', 'longitude': '-110'}
params = {'resource_type': rtype,
'title': title,
'metadata': json.dumps(metadata),
'extra_metadata': json.dumps(extra_metadata)}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
# test core metadata
content = json.loads(response.content)
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
self.assertEqual(resource.metadata.coverages.all().count(), 1)
self.assertEqual(resource.metadata.coverages.filter(type='period').count(), 1)
coverage = resource.metadata.coverages.all().first()
self.assertEqual(parser.parse(coverage.value['start']).date(),
parser.parse('01/01/2000').date())
self.assertEqual(parser.parse(coverage.value['end']).date(),
parser.parse('12/12/2010').date())
# test extra metadata
self.assertEquals(resource.extra_metadata.get('latitude'), '40')
self.assertEquals(resource.extra_metadata.get('longitude'), '-110')
self.resources_to_delete.append(res_id)
def write_metadata_to_resource(self, resource):
"""
Write metadata to resource
:param resource: ModelInstanceResource instance
"""
super(ModelInstanceResourceMeta, self).write_metadata_to_resource(resource)
resource.metadata._model_output.update(includes_output=self.has_model_output)
if self.executed_by_uri:
uri_stripped = self.executed_by_uri.strip('/')
short_id = os.path.basename(uri_stripped)
if short_id == '':
msg = "ExecutedBy URL {0} does not contain a model program resource ID, "
msg += "for resource {1}"
msg = msg.format(self.executed_by_uri, self.root_uri)
raise GenericResourceMeta.ResourceMetaException(msg)
# Make sure the resource specified by ExecutedBy exists
try:
executed_by_resource = get_resource_by_shortkey(short_id,
or_404=False)
except BaseResource.DoesNotExist:
msg = "ExecutedBy resource {0} does not exist.".format(short_id)
raise HsDeserializationDependencyException(short_id, msg)
executed_by = resource.metadata.executed_by
if not executed_by:
# Create
ExecutedBy.create(content_object=resource.metadata,
model_name=short_id)
else:
# Update
ExecutedBy.update(executed_by.element_id,
model_name=short_id)
def delete_resource(pk):
"""
Deletes a resource managed by HydroShare. The caller must be an owner of the resource or an administrator to perform
this function. The operation removes the resource from further interaction with HydroShare services and interfaces. The
implementation may delete the resource bytes, and should do so since a delete operation may be in response to a problem
with the resource (e.g., it contains malicious content, is inappropriate, or is subject to a legal request). If the
resource does not exist, the Exceptions.NotFound exception is raised.
REST URL: DELETE /resource/{pid}
Parameters:
pid - The unique HydroShare identifier of the resource to be deleted
Returns:
The pid of the resource that was deleted
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
Note: Only HydroShare administrators will be able to delete formally published resour
"""
#utils.get_resource_by_shortkey(pk).delete()
res = utils.get_resource_by_shortkey(pk)
res.delete()
return pk
def publish_resource(pk):
"""
Formally publishes a resource in HydroShare. Triggers the creation of a DOI for the resource, and triggers the
exposure of the resource to the HydroShare DataONE Member Node. The user must be an owner of a resource or an
adminstrator to perform this action.
REST URL: PUT /publishResource/{pid}
Parameters: pid - Unique HydroShare identifier for the resource to be formally published.
Returns: The pid of the resource that was published
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
Note: This is different than just giving public access to a resource via access control rul
"""
resource = utils.get_resource_by_shortkey(pk)
resource.published_and_frozen = True
resource.frozen = True
resource.edit_users = []
resource.edit_groups = []
resource.save()
def update_resource_file(pk, filename, f):
"""
Called by clients to update an individual file within a HydroShare resource.
REST URL: PUT /resource/{pid}/files/{filename}
Parameters:
pid - Unique HydroShare identifier for the resource from which the file will be extracted.
filename - The data bytes of the file that will be extracted from the resource identified by pid
file - the data bytes of the file to update
Returns: The bytes of the file extracted from the resource
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified does not exist or the file identified by filename does not exist
Exception.ServiceFailure - The service is unable to process the request
"""
resource = utils.get_resource_by_shortkey(pk)
for rf in ResourceFile.objects.filter(object_id=resource.id):
if os.path.basename(rf.resource_file.name) == filename:
rf.resource_file.delete()
rf.resource_file = File(f) if not isinstance(f, UploadedFile) else f
rf.save()
return rf
else:
raise ObjectDoesNotExist(filename)
def short_url(request, *args, **kwargs):
try:
shortkey = kwargs['shortkey']
except KeyError:
raise TypeError('shortkey must be specified...')
m = get_resource_by_shortkey(shortkey)
return HttpResponseRedirect(m.get_absolute_url())
def get(self, request, pk):
""" Get resource system metadata, as well as URLs to the bag and science metadata
"""
view_utils.authorize(request, pk, view=True, full=True)
res = get_resource_by_shortkey(pk)
ser = self.get_serializer_class()(self.resourceToResourceListItem(res))
return Response(data=ser.data, status=status.HTTP_200_OK)
def test_refts_creation_via_rest_api(self):
rtype = 'RefTimeSeriesResource'
title = 'My Test RefTS res'
ref_url = "http://data.iutahepscor.org/LoganRiverWOF/REST/waterml_1_1.svc/datavalues?" \
"location=iutah:LR_WaterLab_AA&variable=iutah:WaterTemp_EXO&" \
"startDate=2014-12-02T19:45:00Z&endDate=2014-12-05T19:45:00Z"
ref_type = "rest"
# test the "ref_url" rest endpoint is up
response = requests.get(ref_url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
metadata = []
# add core metadata
metadata.append({'relation': {'type': 'isPartOf',
'value': 'http://hydroshare.org/resource/001'}})
# add refts-specific metadata
metadata.append({"referenceurl":
{"value": ref_url,
"type": ref_type}})
# post to rest api
params = {'resource_type': rtype,
'title': title,
'metadata': json.dumps(metadata),
}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
content = json.loads(response.content)
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
# test core metadata
self.assertEqual(resource.metadata.relations.all().count(), 1)
relation = resource.metadata.relations.all().first()
self.assertEqual(relation.type, 'isPartOf')
self.assertEqual(relation.value, 'http://hydroshare.org/resource/001')
# test resource-specific metadata
self.assertEqual(resource.resource_type.lower(), "reftimeseriesresource")
self.assertEqual(resource.metadata.referenceURLs.all().count(), 1)
referenceURLs = resource.metadata.referenceURLs.all().first()
self.assertEquals(referenceURLs.value, ref_url)
self.assertEquals(referenceURLs.type, ref_type)
self.assertEqual(resource.metadata.sites.all().count(), 1)
sites = resource.metadata.sites.all().first()
self.assertEquals(sites.code, "LR_WaterLab_AA")
self.assertEqual(resource.metadata.variables.all().count(), 1)
variables = resource.metadata.variables.all().first()
self.assertEquals(variables.code, "WaterTemp_EXO")
self.resources_to_delete.append(res_id)
def get_science_metadata_xml(resource_short_id):
"""
Gets science metadata as an xml string for a specified resource
:param resource_short_id: id of the resource
:return: science metadata as an xml string
"""
res = utils.get_resource_by_shortkey(resource_short_id)
return res.metadata.get_xml()
def handle(self, *args, **options):
if len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
resource = get_resource_by_shortkey(rid)
self.repair_filtered_solr(resource, options)
else:
for resource in BaseResource.objects.all():
self.repair_filtered_solr(resource, options)
def copy_res_to_specified_federated_irods_server_as_needed(app_shortkey, res_shortkey, user):
"""
When app resource has iRODS federation target path and target resource defined as
extended metadata, the resource needs to be pushed to the specified iRODS federation path
in the specified iRODS storage resource.
:param app_shortkey: shortkey of the app tool resource to be launched
:param res_shortkey: shortkey of the resource launching the app tool resource
:param user: requesting user
:return:
"""
# check whether irods_path_key and irods_resc_key are added as extended metadata of the
# app tool resource, and if they are, push resource to specified iRODS target accordingly
filterd_app_obj = BaseResource.objects.filter(short_id=app_shortkey).filter(
extra_metadata__has_key=irods_path_key).filter(
extra_metadata__has_key=irods_resc_key).first()
if filterd_app_obj:
try:
res = get_resource_by_shortkey(res_shortkey)
app_res = get_resource_by_shortkey(app_shortkey)
user_auth_key_exist = BaseResource.objects.filter(short_id=app_shortkey).filter(
extra_metadata__has_key=user_auth_flag_key).first()
if user_auth_key_exist:
req_user_auth = True \
if app_res.extra_metadata[user_auth_flag_key].lower() == 'true' \
else False
if req_user_auth and not user.is_authenticated():
err_msg = "Only authorized users can launch the web app tool - " \
"Please sign in first."
raise WebAppLaunchException(err_msg)
irods_path = app_res.extra_metadata[irods_path_key]
irods_resc = app_res.extra_metadata[irods_resc_key]
istorage = res.get_irods_storage()
src_path = res.root_path
# delete all temporary resources copied to this user's space before pushing resource
dest_path = os.path.join(irods_path, user.username)
if istorage.exists(dest_path):
istorage.delete(dest_path)
dest_path = os.path.join(dest_path, res_shortkey)
istorage.copyFiles(src_path, dest_path, irods_resc)
except SessionException as ex:
raise WebAppLaunchException(ex.stderr)
except Exception as ex:
raise WebAppLaunchException(ex.message)
def delete_metadata_element(resource_short_id, element_model_name, element_id):
"""
Deletes a specific type of metadata element for a specified resource
:param resource_short_id: id of the resource for which metadata element to be deleted
:param element_model_name: metadata element name (e.g., creator)
:param element_id: id of the metadata element to be deleted
:return:
"""
res = utils.get_resource_by_shortkey(resource_short_id)
res.metadata.delete_element(element_model_name, element_id)
def create_metadata_element(resource_short_id, element_model_name, **kwargs):
"""
Creates a specific type of metadata element for a given resource
:param resource_short_id: id of the resource for which a metadata element needs to be created
:param element_model_name: metadata element name (e.g., creator)
:param kwargs: metadata element attribute name/value pairs for all those attributes that require a value
:return:
"""
res = utils.get_resource_by_shortkey(resource_short_id)
res.metadata.create_element(element_model_name, **kwargs)
def handle(self, *args, **options):
if len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
resource = get_resource_by_shortkey(rid)
if (options['type'] is None or resource.resource_type == options['type']) and \
(options['storage'] is None or resource.storage_type == options['storage']):
CheckResource(rid).test()
else:
for resource in BaseResource.objects.all():
if (options['type'] is None or resource.resource_type == options['type']) and \
(options['storage'] is None or resource.storage_type == options['storage']):
CheckResource(resource.short_id).test()
def update_metadata_element(resource_short_id, element_model_name, element_id, **kwargs):
"""
Updates the data associated with a metadata element for a specified resource
:param resource_short_id: id of the resource for which a metadata element needs to be updated
:param element_model_name: metadata element name (e.g., creator)
:param element_id: id of the metadata element to be updated
:param kwargs: metadata element attribute name/value pairs for all those attributes that need update
:return:
"""
res = utils.get_resource_by_shortkey(resource_short_id)
res.metadata.update_element(element_model_name, element_id, **kwargs)
def update_science_metadata(pk, metadata=None, keywords=None, **kwargs):
"""
Called by clients to update the science metadata for a resource in HydroShare.
REST URL: PUT /scimeta/{pid}
Parameters:
pid - Unique HydroShare identifier for the resource that is to be updated.
ScienceMetadata - The data bytes of the ScienceMetadata that will update the existing Science Metadata for the resource
identified by pid
Returns: The pid assigned to the resource whose Science Metadata was updated
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the resource is incomplete
Exception.ServiceFailure - The service is unable to process the request
Notes:
For mutable resources (resources that have not been formally published), the update overwrites existing Science
Metadata using the ScienceMetadata that is passed to this method. For immutable resources (formally published
resources), this method creates a new resource that is a new version of the formally published resource. HydroShare
will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the
respective resources in their system metadata. HydroShare MUST check or set the values of SystemMetadata.obsoletes
and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects.
hydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be
available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on
SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version.
Once a resource is obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
if keywords:
AssignedKeyword.objects.filter(object_pk=resource.id).delete()
ks = [Keyword.objects.get_or_create(title=k) for k in keywords]
ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse
for k in ks:
AssignedKeyword.objects.create(content_object=resource.id, keyword=k)
# for creating metadata elements based on the new metadata implementation
if metadata:
_update_science_metadata(resource, metadata)
if kwargs:
for field, value in kwargs.items():
setattr(resource, field, value)
resource.save()
def add_zip_file_contents_to_resource(pk, zip_file_path):
"""Add zip file to existing resource and remove tmp zip file."""
zfile = None
resource = None
try:
resource = utils.get_resource_by_shortkey(pk, or_404=False)
zfile = zipfile.ZipFile(zip_file_path)
num_files = len(zfile.infolist())
zcontents = utils.ZipContents(zfile)
files = zcontents.get_files()
resource.file_unpack_status = 'Running'
resource.save()
for i, f in enumerate(files):
logger.debug("Adding file {0} to resource {1}".format(f.name, pk))
utils.add_file_to_resource(resource, f)
resource.file_unpack_message = "Imported {0} of about {1} file(s) ...".format(
i, num_files)
resource.save()
# This might make the resource unsuitable for public consumption
resource.update_public_and_discoverable()
# TODO: this is a bit of a lie because a different user requested the bag overwrite
utils.resource_modified(resource, resource.creator, overwrite_bag=False)
# Call success callback
resource.file_unpack_message = None
resource.file_unpack_status = 'Done'
resource.save()
except BaseResource.DoesNotExist:
msg = "Unable to add zip file contents to non-existent resource {pk}."
msg = msg.format(pk=pk)
logger.error(msg)
except:
exc_info = "".join(traceback.format_exception(*sys.exc_info()))
if resource:
resource.file_unpack_status = 'Error'
resource.file_unpack_message = exc_info
resource.save()
if zfile:
zfile.close()
logger.error(exc_info)
finally:
# Delete upload file
os.unlink(zip_file_path)
def put(self, request, pk):
""" Update access rules
"""
view_utils.authorize(request, pk, edit=True, full=True)
access_rules_validator = serializers.AccessRulesRequestValidator(data=request.data)
if not access_rules_validator.is_valid():
raise ValidationError(detail=access_rules_validator.errors)
validated_request_data = access_rules_validator.validated_data
res = get_resource_by_shortkey(pk)
res.public = validated_request_data["public"]
res.save()
return Response(data={"resource_id": pk}, status=status.HTTP_200_OK)
def update_web_services(services_url, api_token, timeout, publish_urls, res_id):
"""Update web services hosted by GeoServer and HydroServer.
This function sends a resource id to the HydroShare web services manager
application, which will check the current status of the resource and register
or unregister services hosted by GeoServer and HydroServer.
The HydroShare web services manager will return a list of endpoint URLs
for both the resource and individual aggregations. If publish_urls is set to
True, these endpoints will be added to the extra metadata fields of the
resource and aggregations.
"""
session = requests.Session()
session.headers.update(
{"Authorization": " ".join(("Token", str(api_token)))}
)
rest_url = str(services_url) + "/" + str(res_id) + "/"
try:
response = session.post(rest_url, timeout=timeout)
if publish_urls and response.status_code == status.HTTP_201_CREATED:
try:
resource = utils.get_resource_by_shortkey(res_id)
response_content = json.loads(response.content)
for key, value in response_content["resource"].iteritems():
resource.extra_metadata[key] = value
resource.save()
for url in response_content["content"]:
lf = resource.logical_files[[i.aggregation_name for i in
resource.logical_files].index(
url["layer_name"].encode("utf-8")
)]
lf.metadata.extra_metadata["Web Services URL"] = url["message"]
lf.metadata.save()
except Exception as e:
logger.error(e)
return e
return response
except (requests.exceptions.RequestException, ValueError) as e:
logger.error(e)
return e
def add_resource_files(pk, *files):
"""
Called by clients to update a resource in HydroShare by adding a single file.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pid - Unique HydroShare identifier for the resource that is to be updated.
file - The data bytes of the file that will be added to the existing resource identified by pid
Returns: The pid assigned to the updated resource
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
For mutable resources (resources not formally published), the update adds the file that is passed
to this method to the resource. For immutable resources (formally published resources), this method creates a new
resource that is a new version of the formally published resource. HydroShare will record the update by storing the
SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata
HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they
accurately represent the relationship between the new and old objects. HydroShare MUST also set
SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in
HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on
SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version.
Once a resource is obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
for file in files:
ret.append(ResourceFile.objects.create(
content_object=resource,
resource_file=File(file) if not isinstance(file, UploadedFile) else file
))
# add format metadata element if necessary
file_format_type = utils.get_file_mime_type(file.name)
if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]:
resource.metadata.create_element('format', value=file_format_type)
return ret
def get_capabilities(pk):
"""
Describes API services exposed for a resource. If there are extra capabilites for a particular resource type over
and above the standard Hydroshare API, then this API call will list these
REST URL: GET /capabilites/{pid}
Parameters: Unique HydroShare identifier for the resource whose capabilites are to be retrieved.
Return Type: Capabilites
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
"""
res = utils.get_resource_by_shortkey(pk)
return getattr(res, 'extra_capabilities', lambda: None)()
def put(self, request, pk):
""" Update access rules
"""
# only resource owners are allowed to change resource flags (e.g., public)
view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.SET_RESOURCE_FLAG)
access_rules_validator = serializers.AccessRulesRequestValidator(data=request.data)
if not access_rules_validator.is_valid():
raise ValidationError(detail=access_rules_validator.errors)
validated_request_data = access_rules_validator.validated_data
res = get_resource_by_shortkey(pk)
try:
res.set_public(validated_request_data['public'], request.user)
except CoreValidationError:
return Response(data={'resource_id': pk}, status=status.HTTP_403_FORBIDDEN)
return Response(data={'resource_id': pk}, status=status.HTTP_200_OK)
def get_system_metadata(pk):
"""
Describes the resource identified by the pid by returning the associated system metadata object. If the resource
does not exist, Exceptions.NotFound must be raised.
REST URL: GET /sysmeta/{pid}
Parameters: pk - Unique HydroShare identifier for the resource whose system metadata is to be retrieved.
Returns: System metadata document describing the resource.
Return Type: SystemMetadata
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
"""
return utils.get_resource_by_shortkey(pk)
def get_revisions(pk):
"""
Returns a list of pids for resources that are revisions of the resource identified by the specified pid.
REST URL: GET /revisions/{pid}
Parameters: pid - Unique HydroShare identifier for the resource whose revisions are to be retrieved.
Returns: List of pids for resources that are revisions of the specified resource.
Return Type: List of pids
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The Resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
"""
return utils.get_resource_by_shortkey(pk).bags.all()
def create_temp_zip(resource_id, input_path, output_path, sf_aggregation, sf_zip=False):
""" Create temporary zip file from input_path and store in output_path
:param input_path: full irods path of input starting with federation path
:param output_path: full irods path of output starting with federation path
:param sf_aggregation: if True, include logical metadata files
"""
from hs_core.hydroshare.utils import get_resource_by_shortkey
res = get_resource_by_shortkey(resource_id)
istorage = res.get_irods_storage() # invoke federated storage as necessary
if res.resource_type == "CompositeResource":
if '/data/contents/' in input_path:
short_path = input_path.split('/data/contents/')[1] # strip /data/contents/
res.create_aggregation_xml_documents(aggregation_name=short_path)
else: # all metadata included, e.g., /data/*
res.create_aggregation_xml_documents()
try:
if sf_zip:
# input path points to single file aggregation
# ensure that foo.zip contains aggregation metadata
# by copying these into a temp subdirectory foo/foo parallel to where foo.zip is stored
temp_folder_name, ext = os.path.splitext(output_path) # strip zip to get scratch dir
head, tail = os.path.split(temp_folder_name) # tail is unqualified folder name "foo"
out_with_folder = os.path.join(temp_folder_name, tail) # foo/foo is subdir to zip
istorage.copyFiles(input_path, out_with_folder)
if sf_aggregation:
try:
istorage.copyFiles(input_path + '_resmap.xml', out_with_folder + '_resmap.xml')
except SessionException:
logger.error("cannot copy {}".format(input_path + '_resmap.xml'))
try:
istorage.copyFiles(input_path + '_meta.xml', out_with_folder + '_meta.xml')
except SessionException:
logger.error("cannot copy {}".format(input_path + '_meta.xml'))
istorage.zipup(temp_folder_name, output_path)
istorage.delete(temp_folder_name) # delete working directory; this isn't the zipfile
else: # regular folder to zip
istorage.zipup(input_path, output_path)
except SessionException as ex:
logger.error(ex.stderr)
return False
return True
def get_resource_map(pk):
"""
Describes the resource identified by the pid by returning the associated resource map document. If the resource does
not exist, Exceptions.NotFound must be raised.
REST URL: GET /resourcemap/{pid}
Parameters: pid - Unique HydroShare identifier for the resource whose resource map is to be retrieved.
Returns: Resource map document describing the resource.
Return Type: ResourceMap
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
"""
return utils.get_resource_by_shortkey(pk)
def handle(self, *args, **options):
if options['username'] is None or options['resource_id'] is None:
usage()
exit(1)
username = options['username']
resource_id = options['resource_id']
user = user_from_name(username)
if user is None:
usage()
exit(1)
try:
resource = get_resource_by_shortkey(resource_id, or_404=False)
except BaseResource.DoesNotExist:
print("No such resource {}.".format(resource_id))
usage()
exit(1)
print(access_provenance(user, resource))
def test_resource_create_with_extra_metadata(self):
rtype = 'GenericResource'
title = 'My Test resource'
extra_metadata = {'latitude': '40', 'longitude': '-110'}
params = {'resource_type': rtype,
'title': title,
'extra_metadata': json.dumps(extra_metadata)}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
content = json.loads(response.content)
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
# test extra metadata
self.assertEquals(resource.extra_metadata.get('latitude'), '40')
self.assertEquals(resource.extra_metadata.get('longitude'), '-110')
self.resources_to_delete.append(res_id)
def write_metadata_to_resource(self, resource):
"""
Write metadata to resource
:param resource: ModelInstanceResource instance
"""
super(ModelInstanceResourceMeta,
self).write_metadata_to_resource(resource)
resource.metadata._model_output.update(
includes_output=self.has_model_output)
if self.executed_by_uri:
uri_stripped = self.executed_by_uri.strip('/')
short_id = os.path.basename(uri_stripped)
if short_id == '':
msg = "ExecutedBy URL {0} does not contain a model program resource ID, "
msg += "for resource {1}"
msg = msg.format(self.executed_by_uri, self.root_uri)
raise GenericResourceMeta.ResourceMetaException(msg)
# Make sure the resource specified by ExecutedBy exists
try:
executed_by_resource = get_resource_by_shortkey(short_id,
or_404=False)
except BaseResource.DoesNotExist:
msg = "ExecutedBy resource {0} does not exist.".format(
short_id)
raise HsDeserializationDependencyException(short_id, msg)
executed_by = resource.metadata.executed_by
if not executed_by:
# Create
ExecutedBy.create(content_object=resource.metadata,
model_name=short_id)
else:
# Update
ExecutedBy.update(executed_by.element_id, model_name=short_id)
def create_temp_zip(resource_id,
input_path,
output_path,
aggregation_name=None,
sf_zip=False):
""" Create temporary zip file from input_path and store in output_path
:param resource_id: the short_id of a resource
:param input_path: full irods path of input starting with federation path
:param output_path: full irods path of output starting with federation path
:param aggregation_name: The name of the aggregation to zip
:param sf_zip: signals a single file to zip
"""
from hs_core.hydroshare.utils import get_resource_by_shortkey
res = get_resource_by_shortkey(resource_id)
aggregation = None
if aggregation_name:
aggregation = res.get_aggregation_by_aggregation_name(aggregation_name)
istorage = res.get_irods_storage() # invoke federated storage as necessary
if res.resource_type == "CompositeResource":
if '/data/contents/' in input_path:
short_path = input_path.split('/data/contents/')[
1] # strip /data/contents/
res.create_aggregation_xml_documents(path=short_path)
else: # all metadata included, e.g., /data/*
res.create_aggregation_xml_documents()
try:
if aggregation or sf_zip:
# input path points to single file aggregation
# ensure that foo.zip contains aggregation metadata
# by copying these into a temp subdirectory foo/foo parallel to where foo.zip is stored
temp_folder_name, ext = os.path.splitext(
output_path) # strip zip to get scratch dir
head, tail = os.path.split(
temp_folder_name) # tail is unqualified folder name "foo"
out_with_folder = os.path.join(temp_folder_name,
tail) # foo/foo is subdir to zip
istorage.copyFiles(input_path, out_with_folder)
if aggregation:
try:
istorage.copyFiles(aggregation.map_file_path,
temp_folder_name)
except SessionException:
logger.error("cannot copy {}".format(
aggregation.map_file_path))
try:
istorage.copyFiles(aggregation.metadata_file_path,
temp_folder_name)
except SessionException:
logger.error("cannot copy {}".format(
aggregation.metadata_file_path))
for file in aggregation.files.all():
try:
istorage.copyFiles(file.storage_path, temp_folder_name)
except SessionException:
logger.error("cannot copy {}".format(
file.storage_path))
istorage.zipup(temp_folder_name, output_path)
istorage.delete(
temp_folder_name
) # delete working directory; this isn't the zipfile
else: # regular folder to zip
istorage.zipup(input_path, output_path)
except SessionException as ex:
logger.error(ex.stderr)
return False
return True
def create_bag_by_irods(resource_id):
"""Create a resource bag on iRODS side by running the bagit rule and ibun zip.
This function runs as a celery task, invoked asynchronously so that it does not
block the main web thread when it creates bags for very large files which will take some time.
:param
resource_id: the resource uuid that is used to look for the resource to create the bag for.
:return: True if bag creation operation succeeds;
False if there is an exception raised or resource does not exist.
"""
from hs_core.hydroshare.utils import get_resource_by_shortkey
res = get_resource_by_shortkey(resource_id)
istorage = res.get_irods_storage()
metadata_dirty = istorage.getAVU(res.root_path, 'metadata_dirty')
# if metadata has been changed, then regenerate metadata xml files
if metadata_dirty is None or metadata_dirty.lower() == "true":
try:
create_bag_files(res)
except Exception as ex:
logger.error('Failed to create bag files. Error:{}'.format(
ex.message))
return False
bag_full_name = 'bags/{res_id}.zip'.format(res_id=resource_id)
if res.resource_federation_path:
irods_bagit_input_path = os.path.join(res.resource_federation_path,
resource_id)
is_exist = istorage.exists(irods_bagit_input_path)
# check to see if bagit readme.txt file exists or not
bagit_readme_file = '{fed_path}/{res_id}/readme.txt'.format(
fed_path=res.resource_federation_path, res_id=resource_id)
is_bagit_readme_exist = istorage.exists(bagit_readme_file)
bagit_input_path = "*BAGITDATA='{path}'".format(
path=irods_bagit_input_path)
bagit_input_resource = "*DESTRESC='{def_res}'".format(
def_res=settings.HS_IRODS_USER_ZONE_DEF_RES)
bag_full_name = os.path.join(res.resource_federation_path,
bag_full_name)
bagit_files = [
'{fed_path}/{res_id}/bagit.txt'.format(
fed_path=res.resource_federation_path, res_id=resource_id),
'{fed_path}/{res_id}/manifest-md5.txt'.format(
fed_path=res.resource_federation_path, res_id=resource_id),
'{fed_path}/{res_id}/tagmanifest-md5.txt'.format(
fed_path=res.resource_federation_path,
res_id=resource_id), '{fed_path}/bags/{res_id}.zip'.format(
fed_path=res.resource_federation_path, res_id=resource_id)
]
else:
is_exist = istorage.exists(resource_id)
# check to see if bagit readme.txt file exists or not
bagit_readme_file = '{res_id}/readme.txt'.format(res_id=resource_id)
is_bagit_readme_exist = istorage.exists(bagit_readme_file)
irods_dest_prefix = "/" + settings.IRODS_ZONE + "/home/" + settings.IRODS_USERNAME
irods_bagit_input_path = os.path.join(irods_dest_prefix, resource_id)
bagit_input_path = "*BAGITDATA='{path}'".format(
path=irods_bagit_input_path)
bagit_input_resource = "*DESTRESC='{def_res}'".format(
def_res=settings.IRODS_DEFAULT_RESOURCE)
bagit_files = [
'{res_id}/bagit.txt'.format(res_id=resource_id),
'{res_id}/manifest-md5.txt'.format(res_id=resource_id),
'{res_id}/tagmanifest-md5.txt'.format(res_id=resource_id),
'bags/{res_id}.zip'.format(res_id=resource_id)
]
# only proceed when the resource is not deleted potentially by another request
# when being downloaded
if is_exist:
# if bagit readme.txt does not exist, add it.
if not is_bagit_readme_exist:
from_file_name = getattr(settings,
'HS_BAGIT_README_FILE_WITH_PATH',
'docs/bagit/readme.txt')
istorage.saveFile(from_file_name, bagit_readme_file, True)
# call iRODS bagit rule here
bagit_rule_file = getattr(settings, 'IRODS_BAGIT_RULE',
'hydroshare/irods/ruleGenerateBagIt_HS.r')
try:
# call iRODS run and ibun command to create and zip the bag, ignore SessionException
# for now as a workaround which could be raised from potential race conditions when
# multiple ibun commands try to create the same zip file or the very same resource
# gets deleted by another request when being downloaded
istorage.runBagitRule(bagit_rule_file, bagit_input_path,
bagit_input_resource)
istorage.zipup(irods_bagit_input_path, bag_full_name)
istorage.setAVU(irods_bagit_input_path, 'bag_modified', "false")
return True
except SessionException as ex:
# if an exception occurs, delete incomplete files potentially being generated by
# iRODS bagit rule and zipping operations
for fname in bagit_files:
if istorage.exists(fname):
istorage.delete(fname)
logger.error(ex.stderr)
return False
else:
logger.error('Resource does not exist.')
return False
def test_get_resource_by_shortkey(self):
self.assertEqual(
utils.get_resource_by_shortkey(self.res.short_id),
self.res
)
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in HydroShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique HydroShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects added to the resource
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
This does **not** handle mutability; changes to immutable resources should be denied elsewhere.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
source_names = kwargs.pop('source_names', [])
full_paths = kwargs.pop('full_paths', {})
auto_aggregate = kwargs.pop('auto_aggregate', True)
if __debug__:
assert(isinstance(source_names, list))
folder = kwargs.pop('folder', None)
if __debug__: # assure that there are no spurious kwargs left.
for k in kwargs:
print("kwargs[{}]".format(k))
assert len(kwargs) == 0
prefix_path = 'data/contents'
if folder is None or folder == prefix_path:
base_dir = ""
elif folder.startswith(prefix_path):
base_dir = folder[len(prefix_path) + 1:]
else:
base_dir = folder
new_folders = set()
for f in files:
full_dir = base_dir
if f in full_paths:
# TODO, put this in it's own method?
full_path = full_paths[f]
dir_name = os.path.dirname(full_path)
# Only do join if dir_name is not empty, otherwise, it'd result in a trailing slash
full_dir = os.path.join(base_dir, dir_name) if dir_name else base_dir
if full_dir:
new_folders.add(os.path.join(resource.file_path, full_dir))
ret.append(utils.add_file_to_resource(resource, f, folder=full_dir))
else:
ret.append(utils.add_file_to_resource(resource, f, folder=None))
if len(source_names) > 0:
for ifname in source_names:
ret.append(utils.add_file_to_resource(resource, None,
folder=folder,
source_name=ifname))
if not ret:
# no file has been added, make sure data/contents directory exists if no file is added
utils.create_empty_contents_directory(resource)
else:
if resource.resource_type == "CompositeResource" and auto_aggregate:
utils.check_aggregations(resource, new_folders, ret)
# some file(s) added, need to update quota usage
update_quota_usage(res=resource)
return ret
def test_resource_create_with_extended_metadata(self):
"""
The followings are the extended metadata elements for the NetCDF resource that can be
passed as part of the 'metadata' parameter when creating a resource:
originalcoverage
variable
"""
rtype = 'NetcdfResource'
title = 'My Test resource'
metadata = []
# originalcover
value = {"northlimit": 12, "projection": "transverse_mercator", "units": "meter",
"southlimit": 10, "eastlimit": 23, "westlimit": 2}
metadata.append({'originalcoverage': {'value': value,
'projection_string_text': '+proj=tmerc +lon_0=-111.0 '
'+lat_0=0.0 +x_0=500000.0 '
'+y_0=0.0 +k_0=0.9996',
'projection_string_type': 'Proj4 String'}})
# variable (this element is defined in multiple resource types including
# NetcdfResource type)
var_name = 'SWE'
var_type = 'Float'
var_shape = 'y,x,time'
var_unit = 'm'
var_missing_value = '-9999'
var_des_name = 'Snow water equivalent'
var_method = 'model simulation of UEB'
metadata.append({'variable': {'name': var_name, 'type': var_type, 'shape': var_shape,
'unit': var_unit, 'missing_value': var_missing_value,
'descriptive_name': var_des_name, 'method': var_method}})
params = {'resource_type': rtype,
'title': title,
'metadata': json.dumps(metadata),
}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
content = json.loads(response.content.decode())
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
# there should be 1 originalcoverage element
self.assertEqual(resource.metadata.ori_coverage.all().count(), 1)
ori_coverage = resource.metadata.ori_coverage.all().first()
self.assertEqual(ori_coverage.value, value)
self.assertEqual(ori_coverage.projection_string_text, '+proj=tmerc +lon_0=-111.0 '
'+lat_0=0.0 +x_0=500000.0 '
'+y_0=0.0 +k_0=0.9996')
self.assertEqual(ori_coverage.projection_string_type, 'Proj4 String')
# there should be 1 variable element
self.assertEqual(resource.metadata.variables.all().count(), 1)
variable = resource.metadata.variables.all().first()
self.assertEqual(variable.name, var_name)
self.assertEqual(variable.type, var_type)
self.assertEqual(variable.shape, var_shape)
self.assertEqual(variable.unit, var_unit)
self.assertEqual(variable.missing_value, var_missing_value)
self.assertEqual(variable.descriptive_name, var_des_name)
self.assertEqual(variable.method, var_method)
self.resources_to_delete.append(res_id)
def delete_resource_file(pk, filename_or_id, user, delete_logical_file=True):
"""
Deletes an individual file from a HydroShare resource. If the file does not exist,
the Exceptions.NotFound exception is raised.
REST URL: DELETE /resource/{pid}/files/{filename}
Parameters:
pk - The unique HydroShare identifier for the resource from which the file will be deleted
filename_or_id - Name of the file or id of the file to be deleted from the resource
user - requesting user
delete_logical_file - If True then the ResourceFile object to be deleted if it is part of a
LogicalFile object then the LogicalFile object will be deleted which deletes all associated
ResourceFile objects and file type metadata objects.
Returns: The name or id of the file which was deleted
Return Type: string or integer
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist or the file identified by
file does not exist
Exception.ServiceFailure - The service is unable to process the request
Note: For mutable resources (resources that have not been formally published), this method
modifies the resource by deleting the file. For immutable resources (formally published
resources), this method creates a new resource that is a new version of the formally published
resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy fields for the respective resources in their system metadata
HydroShare MUST check or set the values of SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new
and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified.
The modified system metadata entries must then be available in HydroShare.listObjects() to
ensure that any cataloging systems pick up the changes when filtering on
SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by
one newer version. Once a resource is obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
res_cls = resource.__class__
fed_path = resource.resource_federation_path
for f in ResourceFile.objects.filter(object_id=resource.id):
if filter_condition(filename_or_id, fed_path, f):
if delete_logical_file:
if f.logical_file is not None:
# logical_delete() calls this function (delete_resource_file())
# to delete each of its contained ResourceFile objects
f.logical_file.logical_delete(user)
return filename_or_id
# send signal
signals.pre_delete_file_from_resource.send(sender=res_cls, file=f, resource=resource,
user=user)
file_name = delete_resource_file_only(resource, f)
delete_format_metadata_after_delete_file(resource, file_name)
break
else:
raise ObjectDoesNotExist(filename_or_id)
if resource.raccess.public or resource.raccess.discoverable:
if not resource.can_be_public_or_discoverable:
resource.raccess.public = False
resource.raccess.discoverable = False
resource.raccess.save()
signals.post_delete_file_from_resource.send(sender=res_cls, resource=resource)
# generate bag
utils.resource_modified(resource, user, overwrite_bag=False)
return filename_or_id
def create_new_version_resource_task(ori_res_id, username, new_res_id=None):
"""
Task for creating a new version of a resource
Args:
ori_res_id: the original resource id that is to be versioned.
new_res_id: the new versioned resource id from the original resource. If None, a
new resource will be created.
username: the requesting user's username
Returns:
the new versioned resource url as the payload
"""
try:
new_res = None
if not new_res_id:
new_res = create_empty_resource(ori_res_id, username)
new_res_id = new_res.short_id
utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id)
# copy metadata from source resource to target new-versioned resource except three elements
ori_res = utils.get_resource_by_shortkey(ori_res_id)
if not new_res:
new_res = utils.get_resource_by_shortkey(new_res_id)
utils.copy_and_create_metadata(ori_res, new_res)
# add or update Relation element to link source and target resources
hs_identifier = new_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
ori_res.metadata.create_element('relation',
type='isReplacedBy',
value=hs_identifier.url)
if new_res.metadata.relations.all().filter(
type='isVersionOf').exists():
# the original resource is already a versioned resource, and its isVersionOf relation
# element is copied over to this new version resource, needs to delete this element so
# it can be created to link to its original resource correctly
eid = new_res.metadata.relations.all().filter(
type='isVersionOf').first().id
new_res.metadata.delete_element('relation', eid)
hs_identifier = ori_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
new_res.metadata.create_element('relation',
type='isVersionOf',
value=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new version collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
create_bag(new_res)
# since an isReplaceBy relation element is added to original resource, needs to call
# resource_modified() for original resource
utils.resource_modified(ori_res, by_user=username, overwrite_bag=False)
# if everything goes well up to this point, set original resource to be immutable so that
# obsoleted resources cannot be modified from REST API
ori_res.raccess.immutable = True
ori_res.raccess.save()
ori_res.save()
return new_res.get_absolute_url()
except Exception as ex:
if new_res:
new_res.delete()
raise utils.ResourceVersioningException(str(ex))
finally:
# release the lock regardless
ori_res.locked_time = None
ori_res.save()
def test_refts_creation_via_rest_api(self):
rtype = 'RefTimeSeriesResource'
title = 'My Test RefTS res'
ref_url = "http://data.iutahepscor.org/LoganRiverWOF/REST/waterml_1_1.svc/datavalues?" \
"location=iutah:LR_WaterLab_AA&variable=iutah:WaterTemp_EXO&" \
"startDate=2014-12-02T19:45:00Z&endDate=2014-12-05T19:45:00Z"
ref_type = "rest"
# test the "ref_url" rest endpoint is up
response = requests.get(ref_url)
self.assertEqual(response.status_code, status.HTTP_200_OK)
metadata = []
# add core metadata
metadata.append({
'relation': {
'type': 'isPartOf',
'value': 'http://hydroshare.org/resource/001'
}
})
# add refts-specific metadata
metadata.append({"referenceurl": {"value": ref_url, "type": ref_type}})
# post to rest api
params = {
'resource_type': rtype,
'title': title,
'metadata': json.dumps(metadata),
}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
content = json.loads(response.content)
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
# test core metadata
self.assertEqual(resource.metadata.relations.all().count(), 1)
relation = resource.metadata.relations.all().first()
self.assertEqual(relation.type, 'isPartOf')
self.assertEqual(relation.value, 'http://hydroshare.org/resource/001')
# test resource-specific metadata
self.assertEqual(resource.resource_type.lower(),
"reftimeseriesresource")
self.assertEqual(resource.metadata.referenceURLs.all().count(), 1)
referenceURLs = resource.metadata.referenceURLs.all().first()
self.assertEquals(referenceURLs.value, ref_url)
self.assertEquals(referenceURLs.type, ref_type)
self.assertEqual(resource.metadata.sites.all().count(), 1)
sites = resource.metadata.sites.all().first()
self.assertEquals(sites.code, "LR_WaterLab_AA")
self.assertEqual(resource.metadata.variables.all().count(), 1)
variables = resource.metadata.variables.all().first()
self.assertEquals(variables.code, "WaterTemp_EXO")
self.resources_to_delete.append(res_id)
def update_resource(
pk,
edit_users=None, view_users=None, edit_groups=None, view_groups=None,
keywords=None, metadata=None,
*files, **kwargs):
"""
Called by clients to update a resource in HydroShare.
REST URL: PUT /resource/{pid}
Parameters:
pid - Unique HydroShare identifier for the resource that is to be updated.
resource - The data bytes of the resource that will update the existing resource identified by pid
Returns: The pid assigned to the updated resource
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the resource is incomplete
Exception.ServiceFailure - The service is unable to process the request
Notes:
For mutable resources (resources that have not been formally published), the update overwrites existing data and
metadata using the resource that is passed to this method. If a user wants to create a copy or modified version of a
mutable resource this should be done using HydroShare.createResource().
For immutable resources (formally published resources), this method creates a new resource that is a new version of
formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy fields for the respective resources in their system metadata.HydroShare MUST check or set
the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the
relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The
modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging
systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource
can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
if files:
ResourceFile.objects.filter(object_id=resource.id).delete()
for file in files:
ResourceFile.objects.create(
content_object=resource,
resource_file=File(file) if not isinstance(file, UploadedFile) else file
)
if 'owner' in kwargs:
owner = utils.user_from_id(kwargs['owner'])
resource.owners.add(owner)
if edit_users:
resource.edit_users.clear()
for user in edit_users:
user = utils.user_from_id(user)
resource.edit_users.add(user)
resource.view_users.add(user)
if view_users:
resource.view_users.clear()
for user in view_users:
user = utils.user_from_id(user)
resource.view_users.add(user)
if edit_groups:
resource.edit_groups.clear()
for group in edit_groups:
group = utils.group_from_id(group)
resource.edit_groups.add(group)
resource.view_groups.add(group)
if view_groups:
resource.edit_groups.clear()
for group in view_groups:
group = utils.group_from_id(group)
resource.view_groups.add(group)
if keywords:
AssignedKeyword.objects.filter(object_pk=resource.id).delete()
ks = [Keyword.objects.get_or_create(title=k) for k in keywords]
ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse
for k in ks:
AssignedKeyword.objects.create(content_object=resource, keyword=k)
# for creating metadata elements based on the new metadata implementation
if metadata:
_update_science_metadata(resource, metadata)
return resource
def handle(self, *args, **options):
if not options['resource_id']:
raise CommandError('resource_id argument is required')
res_id = options['resource_id']
try:
res = get_resource_by_shortkey(res_id, or_404=False)
except ObjectDoesNotExist:
raise CommandError("No Resource found for id {}".format(res_id))
if options['new_resource_id']:
try:
UUID(options['new_resource_id'])
new_res_id = options['new_resource_id']
except Exception as e:
raise CommandError('new_resource_id {} must be a valid uuid hex string'
.format(options['new_resource_id']), e)
try:
if BaseResource.objects.get(short_id=new_res_id):
raise CommandError('resource with id {} already exists'.format(new_res_id))
except ObjectDoesNotExist:
pass
else:
new_res_id = short_id()
storage = res.get_irods_storage()
if storage.exists(res.bag_path):
try:
storage.delete(res.bag_path)
print("{} deleted".format(res.bag_path))
except SessionException as ex:
print("{} delete failed: {}".format(res.bag_path, ex.stderr))
raise EnvironmentError()
try:
with transaction.atomic():
print("Deleting existing bag")
res.setAVU("bag_modified", True)
res.setAVU('metadata_dirty', 'true')
print("Updating BaseResource short_id from {} to {}".format(res_id, new_res_id))
res.short_id = new_res_id
res.save()
print("Updating resource slug")
res.set_slug('resource/{}'.format(new_res_id))
print("Updating Resource files short_path")
for file in res.files.all():
file_name = file.short_path.split('data/contents/')[1]
file.set_short_path(file_name)
print("Updating metadata identifiers")
for i in res.metadata.identifiers.all():
i.url = i.url.replace(res_id, new_res_id)
i.save()
print("Updating logical_files metadata")
for aggregation in res.logical_files:
aggregation.metadata.is_dirty = True
aggregation.metadata.save()
except IntegrityError:
raise EnvironmentError("Error occurred while updating")
print("Moving Resource files")
storage.moveFile(res_id, new_res_id)
print("Creating Bag")
create_bag(res)
print(("Resource id successfully update from {} to {}".format(res_id, new_res_id)))
def delete_resource_file(pk, filename_or_id, user):
"""
Deletes an individual file from a HydroShare resource. If the file does not exist, the Exceptions.NotFound exception
is raised.
REST URL: DELETE /resource/{pid}/files/{filename}
Parameters:
pid - The unique HydroShare identifier for the resource from which the file will be deleted
filename - Name of the file to be deleted from the resource
Returns: The pid of the resource from which the file was deleted
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist or the file identified by file does not exist
Exception.ServiceFailure - The service is unable to process the request
Note: For mutable resources (resources that have not been formally published), this method modifies the resource by
deleting the file. For immutable resources (formally published resources), this method creates a new resource that
is a new version of the formally published resource. HydroShare will record the update by storing the
SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata
HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they
accurately represent the relationship between the new and old objects. HydroShare MUST also set
SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in
HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on
SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer
version. Once a resource is obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
res_cls = resource.__class__
try:
file_id = int(filename_or_id)
filter_condition = lambda fl: fl.id == file_id
except ValueError:
filter_condition = lambda fl: os.path.basename(fl.resource_file.name) == filename_or_id
for f in ResourceFile.objects.filter(object_id=resource.id):
if filter_condition(f):
# send signal
signals.pre_delete_file_from_resource.send(sender=res_cls, file=f, resource=resource)
file_name = f.resource_file.name
f.resource_file.delete()
f.delete()
delete_file_mime_type = utils.get_file_mime_type(file_name)
delete_file_extension = os.path.splitext(file_name)[1]
# if there is no other resource file with the same extension as the
# file just deleted then delete the matching format metadata element for the resource
resource_file_extensions = [os.path.splitext(f.resource_file.name)[1] for f in resource.files.all()]
if delete_file_extension not in resource_file_extensions:
format_element = resource.metadata.formats.filter(value=delete_file_mime_type).first()
if format_element:
resource.metadata.delete_element(format_element.term, format_element.id)
break
else:
raise ObjectDoesNotExist(filename_or_id)
if resource.public:
if not resource.can_be_public:
resource.public = False
resource.save()
# generate bag
utils.resource_modified(resource, user)
return filename_or_id
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in HydroShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique HydroShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
For mutable resources (resources not formally published), the update adds the file that is
passed to this method to the resource. For immutable resources (formally published resources),
this method creates a new resource that is a new version of the formally published resource.
HydroShare will record the update by storing the SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy fields for the respective resources in their system metadata
HydroShare MUST check or set the values of SystemMetadata.obsoletes and
SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new
and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified
system metadata entries must then be available in HydroShare.listObjects() to ensure that any
cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified.
A formally published resource can only be obsoleted by one newer version. Once a resource is
obsoleted, no other resources can obsolete it.
"""
resource = utils.get_resource_by_shortkey(pk)
fed_res_file_names = kwargs.pop('fed_res_file_names', '')
ret = []
fed_zone_home_path = kwargs.pop('fed_zone_home_path', '')
# for adding files to existing resources, the default action is copy
fed_copy_or_move = kwargs.pop('fed_copy_or_move', 'copy')
folder = kwargs.pop('folder', None)
for f in files:
if fed_zone_home_path:
# user has selected files from a federated iRODS zone, so files uploaded from local disk
# need to be stored to the federated iRODS zone rather than HydroShare zone as well
# TODO: why do we allow the federation prefix to vary in this statement?
ret.append(utils.add_file_to_resource(resource, f, folder=folder,
fed_res_file_name_or_path=fed_zone_home_path))
elif resource.resource_federation_path:
# file needs to be added to a resource in a federated zone
# TODO: why do we allow the federation prefix to vary in this statement?
ret.append(utils.add_file_to_resource(
resource, f, folder=folder,
fed_res_file_name_or_path=resource.resource_federation_path))
else:
ret.append(utils.add_file_to_resource(resource, f, folder=folder))
if fed_res_file_names:
if isinstance(fed_res_file_names, basestring):
ifnames = string.split(fed_res_file_names, ',')
elif isinstance(fed_res_file_names, list):
ifnames = fed_res_file_names
else:
return ret
for ifname in ifnames:
ret.append(utils.add_file_to_resource(resource, None, folder=folder,
fed_res_file_name_or_path=ifname,
fed_copy_or_move=fed_copy_or_move))
if not ret:
# no file has been added, make sure data/contents directory exists if no file is added
utils.create_empty_contents_directory(resource)
return ret
def create_bag_by_irods(resource_id):
"""Create a resource bag on iRODS side by running the bagit rule and ibun zip.
This function runs as a celery task, invoked asynchronously so that it does not
block the main web thread when it creates bags for very large files which will take some time.
:param
resource_id: the resource uuid that is used to look for the resource to create the bag for.
:return: True if bag creation operation succeeds;
False if there is an exception raised or resource does not exist.
"""
res = utils.get_resource_by_shortkey(resource_id)
istorage = res.get_irods_storage()
bag_path = res.bag_path
metadata_dirty = istorage.getAVU(res.root_path, 'metadata_dirty')
# if metadata has been changed, then regenerate metadata xml files
if metadata_dirty is None or metadata_dirty.lower() == "true":
try:
create_bag_files(res)
except Exception as ex:
logger.error('Failed to create bag files. Error:{}'.format(
ex.message))
# release the lock before returning bag creation failure
return False
irods_bagit_input_path = res.get_irods_path(resource_id,
prepend_short_id=False)
# check to see if bagit readme.txt file exists or not
bagit_readme_file = res.get_irods_path('readme.txt')
is_bagit_readme_exist = istorage.exists(bagit_readme_file)
if irods_bagit_input_path.startswith(resource_id):
# resource is in data zone, need to append the full path for iRODS bagit rule execution
irods_dest_prefix = "/" + settings.IRODS_ZONE + "/home/" + settings.IRODS_USERNAME
irods_bagit_input_path = os.path.join(irods_dest_prefix, resource_id)
bagit_input_resource = "*DESTRESC='{def_res}'".format(
def_res=settings.IRODS_DEFAULT_RESOURCE)
else:
# this will need to be changed with the default resource in whatever federated zone the
# resource is stored in when we have such use cases to support
bagit_input_resource = "*DESTRESC='{def_res}'".format(
def_res=settings.HS_IRODS_USER_ZONE_DEF_RES)
bagit_input_path = "*BAGITDATA='{path}'".format(
path=irods_bagit_input_path)
bagit_files = [
res.get_irods_path('bagit.txt'),
res.get_irods_path('manifest-md5.txt'),
res.get_irods_path('tagmanifest-md5.txt'), bag_path
]
# only proceed when the resource is not deleted potentially by another request
# when being downloaded
is_exist = istorage.exists(irods_bagit_input_path)
if is_exist:
# if bagit readme.txt does not exist, add it.
if not is_bagit_readme_exist:
from_file_name = getattr(settings,
'HS_BAGIT_README_FILE_WITH_PATH',
'docs/bagit/readme.txt')
istorage.saveFile(from_file_name, bagit_readme_file, True)
# call iRODS bagit rule here
bagit_rule_file = getattr(settings, 'IRODS_BAGIT_RULE',
'hydroshare/irods/ruleGenerateBagIt_HS.r')
try:
# call iRODS run and ibun command to create and zip the bag, ignore SessionException
# for now as a workaround which could be raised from potential race conditions when
# multiple ibun commands try to create the same zip file or the very same resource
# gets deleted by another request when being downloaded
istorage.runBagitRule(bagit_rule_file, bagit_input_path,
bagit_input_resource)
istorage.zipup(irods_bagit_input_path, bag_path)
if res.raccess.published:
# compute checksum to meet DataONE distribution requirement
chksum = istorage.checksum(bag_path)
res.bag_checksum = chksum
istorage.setAVU(irods_bagit_input_path, 'bag_modified', "false")
return True
except SessionException as ex:
# if an exception occurs, delete incomplete files potentially being generated by
# iRODS bagit rule and zipping operations
for fname in bagit_files:
if istorage.exists(fname):
istorage.delete(fname)
logger.error(ex.stderr)
return False
else:
logger.error('Resource does not exist.')
return False
def add_resource_files(pk, *files, **kwargs):
"""
Called by clients to update a resource in CommonsShare by adding one or more files.
REST URL: PUT /resource/{pid}/files/{file}
Parameters:
pk - Unique CommonsShare identifier for the resource that is to be updated.
files - A list of file-like objects representing files that will be added
to the existing resource identified by pid
Returns: A list of ResourceFile objects added to the resource
Return Type: list
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.InvalidContent - The content of the file is invalid
Exception.ServiceFailure - The service is unable to process the request
Notes:
This does **not** handle mutability; changes to immutable resources should be denied elsewhere.
"""
resource = utils.get_resource_by_shortkey(pk)
ret = []
source_names = kwargs.pop('source_names', [])
source_sizes = kwargs.pop('source_sizes', [])
is_file_reference = kwargs.pop('is_file_reference', False)
if __debug__:
assert (isinstance(source_names, list))
move = kwargs.pop('move', False)
folder = kwargs.pop('folder', None)
if __debug__: # assure that there are no spurious kwargs left.
for k in kwargs:
print("kwargs[{}]".format(k))
assert len(kwargs) == 0
for f in files:
ret.append(utils.add_file_to_resource(resource, f, folder=folder))
if len(source_names) > 0:
if len(source_names) != len(source_sizes):
# if length is not equal, there is an issue with source_sizes input parameter, so it will not be
# used by setting it to be empty
source_sizes = []
idx = 0
for ifname in source_names:
s_size = source_sizes[idx] if source_sizes else 0
idx += 1
ret.append(
utils.add_file_to_resource(
resource,
None,
folder=folder,
source_name=ifname,
source_size=s_size,
move=move,
is_file_reference=is_file_reference))
# make sure data/contents directory exists if not exist already
utils.create_empty_contents_directory(resource)
return ret
def update_collection(request, shortkey, *args, **kwargs):
"""
Update collection. The POST request should contain a
list of resource ids and a 'update_type' parameter with value of 'set', 'add' or 'remove',
which are three differnt mode to update the collection.If no 'update_type' parameter is
provided, the 'set' will be used by default.
To add a resource to collection, user should have certain premission on both collection
and resources being added.
For collection: user should have at least Edit permission
For resources being added, one the following criteria should be met:
1) user has at lest View permission and the resource is Shareable
2) user is resource owner
:param shortkey: id of the collection resource to which resources are to be added.
"""
status = "success"
msg = ""
metadata_status = "Insufficient to make public"
new_coverage_list = []
hasPart = "hasPart"
try:
with transaction.atomic():
collection_res_obj, is_authorized, user \
= authorize(request, shortkey,
needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE)
if collection_res_obj.resource_type.lower(
) != "collectionresource":
raise Exception(
"Resource {0} is not a collection resource.".format(
shortkey))
# get 'resource_id_list' list from POST
updated_contained_res_id_list = request.POST.getlist(
"resource_id_list")
# get optional 'update_type' parameter:
# 1) "set" (default): set collection content to the list,
# following code will find out which resources are newly added, removed and unchanged
# 2) 'add': add resources in the list to collection
# adding a resource that is already in the collection will raise error
# 3) 'remove': remove resources in the list from collection
# removing a resource that is not in collection will raise error
update_type = request.POST.get("update_type", 'set').lower()
if update_type not in ["set", "add", "remove"]:
raise Exception("Invalid value of 'update_type' parameter")
if len(updated_contained_res_id_list) > len(
set(updated_contained_res_id_list)):
raise Exception(
"Duplicate resources exist in list 'resource_id_list'")
for updated_contained_res_id in updated_contained_res_id_list:
# avoid adding collection itself
if updated_contained_res_id == shortkey:
raise Exception("Can not contain collection itself.")
# current contained res
res_id_list_current_collection = \
[res.short_id for res in collection_res_obj.resources.all()]
# res to remove
res_id_list_remove = []
if update_type == "remove":
res_id_list_remove = updated_contained_res_id_list
for res_id_remove in res_id_list_remove:
if res_id_remove not in res_id_list_current_collection:
raise Exception('Cannot remove resource {0} as it '
'is not currently contained '
'in collection'.format(res_id_remove))
elif update_type == "set":
for res_id_remove in res_id_list_current_collection:
if res_id_remove not in updated_contained_res_id_list:
res_id_list_remove.append(res_id_remove)
for res_id_remove in res_id_list_remove:
# user with Edit permission over this collection can remove any resource from it
res_obj_remove = get_resource_by_shortkey(res_id_remove)
collection_res_obj.resources.remove(res_obj_remove)
# change "Relation" metadata in collection
value = RES_LANDING_PAGE_URL_TEMPLATE.format(res_id_remove)
add_or_remove_relation_metadata(
add=False,
target_res_obj=collection_res_obj,
relation_type=hasPart,
relation_value=value,
set_res_modified=False)
# res to add
res_id_list_add = []
if update_type == "add":
res_id_list_add = updated_contained_res_id_list
for res_id_add in res_id_list_add:
if res_id_add in res_id_list_current_collection:
raise Exception(
'Cannot add resource {0} as it '
'is already contained in collection'.format(
res_id_add))
elif update_type == "set":
for res_id_add in updated_contained_res_id_list:
if res_id_add not in res_id_list_current_collection:
res_id_list_add.append(res_id_add)
for res_id_add in res_id_list_add:
# check authorization for all new resources being added to the collection
# the requesting user should at least have metadata view permission for each of
# the new resources to be added to the collection
res_to_add, _, _ \
= authorize(request, res_id_add,
needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA)
# the resources being added should be 'Shareable'
# or is owned by current user
is_shareable = res_to_add.raccess.shareable
is_owner = res_to_add.raccess.owners.filter(
pk=user.pk).exists()
if not is_shareable and not is_owner:
raise Exception(
'Only resource owner can add a non-shareable '
'resource to a collection ')
# add this new res to collection
res_obj_add = get_resource_by_shortkey(res_id_add)
collection_res_obj.resources.add(res_obj_add)
# change "Relation" metadata in collection
value = RES_LANDING_PAGE_URL_TEMPLATE.format(res_id_add)
add_or_remove_relation_metadata(
add=True,
target_res_obj=collection_res_obj,
relation_type=hasPart,
relation_value=value,
set_res_modified=False)
if collection_res_obj.can_be_public_or_discoverable:
metadata_status = "Sufficient to make public"
new_coverage_list = _update_collection_coverages(
collection_res_obj)
update_collection_list_csv(collection_res_obj)
resource_modified(collection_res_obj, user, overwrite_bag=False)
except Exception as ex:
err_msg = "update_collection: {0} ; username: {1}; collection_id: {2} ."
logger.error(
err_msg.format(
str(ex), request.user.username
if request.user.is_authenticated() else "anonymous", shortkey))
status = "error"
msg = str(ex)
finally:
ajax_response_data = \
{'status': status, 'msg': msg,
'metadata_status': metadata_status,
'new_coverage_list': new_coverage_list}
return JsonResponse(ajax_response_data)
def test_resource_create_with_core_metadata(self):
"""
The followings are the core metadata elements that can be passed as part of the
'metadata' parameter when creating a resource:
coverage
creator
contributor
source,
relation,
identifier,
fundingagency
"""
rtype = 'GenericResource'
title = 'My Test resource'
metadata = []
metadata.append({'coverage': {'type': 'period', 'value': {'start': '01/01/2000',
'end': '12/12/2010'}}})
statement = 'This resource is shared under the Creative Commons Attribution CC BY.'
url = 'http://creativecommons.org/licenses/by/4.0/'
metadata.append({'rights': {'statement': statement, 'url': url}})
metadata.append({'language': {'code': 'fre'}})
# contributor
con_name = 'Mike Sundar'
con_org = "USU"
con_email = '*****@*****.**'
con_address = "11 River Drive, Logan UT-84321, USA"
con_phone = '435-567-0989'
con_homepage = 'http://usu.edu/homepage/001'
con_identifiers = {'ORCID': 'https://orcid.org/mike_s',
'ResearchGateID': 'https://www.researchgate.net/mike_s'}
metadata.append({'contributor': {'name': con_name,
'organization': con_org, 'email': con_email,
'address': con_address, 'phone': con_phone,
'homepage': con_homepage,
'identifiers': con_identifiers}})
# creator
cr_name = 'John Smith'
cr_org = "USU"
cr_email = '*****@*****.**'
cr_address = "101 Clarson Ave, Provo UT-84321, USA"
cr_phone = '801-567=9090'
cr_homepage = 'http://byu.edu/homepage/002'
cr_identifiers = {'ORCID': 'https://orcid.org/john_smith',
'ResearchGateID': 'https://www.researchgate.net/john_smith'}
metadata.append({'creator': {'name': cr_name, 'organization': cr_org,
'email': cr_email, 'address': cr_address,
'phone': cr_phone, 'homepage': cr_homepage,
'identifiers': cr_identifiers}})
# relation
metadata.append({'relation': {'type': 'isPartOf',
'value': 'http://hydroshare.org/resource/001'}})
# source
metadata.append({'source': {'derived_from': 'http://hydroshare.org/resource/0001'}})
# identifier
metadata.append({'identifier': {'name': 'someIdentifier', 'url': 'http://some.org/001'}})
# fundingagency
agency_name = 'NSF'
award_title = "Cyber Infrastructure"
award_number = "NSF-101-20-6789"
agency_url = "http://www.nsf.gov"
metadata.append({'fundingagency': {'agency_name': agency_name, 'award_title': award_title,
'award_number': award_number, 'agency_url': agency_url}})
params = {'resource_type': rtype,
'title': title,
'metadata': json.dumps(metadata),
'file': ('cea.tif',
open('hs_core/tests/data/cea.tif', 'rb'),
'image/tiff')}
rest_url = '/hsapi/resource/'
response = self.client.post(rest_url, params)
self.assertEqual(response.status_code, status.HTTP_201_CREATED)
content = json.loads(response.content.decode())
res_id = content['resource_id']
resource = get_resource_by_shortkey(res_id)
self.assertEqual(resource.metadata.coverages.all().count(), 1)
self.assertEqual(resource.metadata.coverages.filter(type='period').count(), 1)
coverage = resource.metadata.coverages.all().first()
self.assertEqual(parser.parse(coverage.value['start']).date(),
parser.parse('01/01/2000').date())
self.assertEqual(parser.parse(coverage.value['end']).date(),
parser.parse('12/12/2010').date())
self.assertEqual(resource.metadata.rights.statement, statement)
self.assertEqual(resource.metadata.rights.url, url)
self.assertEqual(resource.metadata.language.code, 'fre')
# there should be 1 contributor
self.assertEqual(resource.metadata.contributors.all().count(), 1)
contributor = resource.metadata.contributors.all().first()
self.assertEqual(contributor.name, con_name)
self.assertEqual(contributor.organization, con_org)
self.assertEqual(contributor.email, con_email)
self.assertEqual(contributor.address, con_address)
self.assertEqual(contributor.phone, con_phone)
self.assertEqual(contributor.homepage, con_homepage)
# there should be 1 creator (based on the metadata we provided for one creator)
# system automatically adds the user who creates the resource as the creator
# only in the case where not metadata for creator is provided.
self.assertEqual(resource.metadata.creators.all().count(), 1)
creator = resource.metadata.creators.filter(name=cr_name).first()
self.assertEqual(creator.name, cr_name)
self.assertEqual(creator.organization, cr_org)
self.assertEqual(creator.email, cr_email)
self.assertEqual(creator.address, cr_address)
self.assertEqual(creator.phone, cr_phone)
self.assertEqual(creator.homepage, cr_homepage)
# there should be 1 relation element
self.assertEqual(resource.metadata.relations.all().count(), 1)
relation = resource.metadata.relations.all().first()
self.assertEqual(relation.type, 'isPartOf')
self.assertEqual(relation.value, 'http://hydroshare.org/resource/001')
# there should be 1 source element
self.assertEqual(resource.metadata.sources.all().count(), 1)
source = resource.metadata.sources.all().first()
self.assertEqual(source.derived_from, 'http://hydroshare.org/resource/0001')
# there should be 2 identifiers
self.assertEqual(resource.metadata.identifiers.all().count(), 2)
identifier = resource.metadata.identifiers.filter(name='someIdentifier').first()
self.assertEqual(identifier.url, 'http://some.org/001')
# there should be 1 fundingagency
self.assertEqual(resource.metadata.funding_agencies.all().count(), 1)
agency = resource.metadata.funding_agencies.all().first()
self.assertEqual(agency.agency_name, agency_name)
self.assertEqual(agency.award_title, award_title)
self.assertEqual(agency.award_number, award_number)
self.assertEqual(agency.agency_url, agency_url)
self.resources_to_delete.append(res_id)
def resource_from_uuid(id):
return get_resource_by_shortkey(id)
def publish_resource(user, pk):
"""
Formally publishes a resource in HydroShare. Triggers the creation of a DOI for the resource,
and triggers the exposure of the resource to the HydroShare DataONE Member Node. The user must
be an owner of a resource or an administrator to perform this action.
Parameters:
user - requesting user to publish the resource who must be one of the owners of the resource
pk - Unique HydroShare identifier for the resource to be formally published.
Returns: The id of the resource that was published
Return Type: string
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist
Exception.ServiceFailure - The service is unable to process the request
and other general exceptions
Note: This is different than just giving public access to a resource via access control rule
"""
resource = utils.get_resource_by_shortkey(pk)
# TODO: whether a resource can be published is not considered in can_be_published
# TODO: can_be_published is currently an alias for can_be_public_or_discoverable
if not resource.can_be_published:
raise ValidationError("This resource cannot be published since it does not have required "
"metadata or content files, or this resource contains referenced "
"content, or this resource type is not allowed for publication.")
# append pending to the doi field to indicate DOI is not activated yet. Upon successful
# activation, "pending" will be removed from DOI field
resource.doi = get_resource_doi(pk, 'pending')
resource.save()
response = deposit_res_metadata_with_crossref(resource)
if not response.status_code == status.HTTP_200_OK:
# resource metadata deposition failed from CrossRef - set failure flag to be retried in a
# crontab celery task
resource.doi = get_resource_doi(pk, 'failure')
resource.save()
resource.set_public(True) # also sets discoverable to True
resource.raccess.immutable = True
resource.raccess.shareable = False
resource.raccess.published = True
resource.raccess.save()
# change "Publisher" element of science metadata to CUAHSI
md_args = {'name': 'Consortium of Universities for the Advancement of Hydrologic Science, '
'Inc. (CUAHSI)',
'url': 'https://www.cuahsi.org'}
resource.metadata.create_element('Publisher', **md_args)
# create published date
resource.metadata.create_element('date', type='published', start_date=resource.updated)
# add doi to "Identifier" element of science metadata
md_args = {'name': 'doi',
'url': get_activated_doi(resource.doi)}
resource.metadata.create_element('Identifier', **md_args)
utils.resource_modified(resource, user, overwrite_bag=False)
return pk
def resource_from_uuid(id):
if not id:
return None
return get_resource_by_shortkey(id)
def delete_resource_file(pk, filename_or_id, user, delete_logical_file=True):
"""
Deletes an individual file from a HydroShare resource. If the file does not exist,
the Exceptions.NotFound exception is raised.
REST URL: DELETE /resource/{pid}/files/{filename}
Parameters:
:param pk: The unique HydroShare identifier for the resource from which the file will be deleted
:param filename_or_id: Name of the file or id of the file to be deleted from the resource
:param user: requesting user
:param delete_logical_file: If True then if the ResourceFile object to be deleted is part of a
LogicalFile object then the LogicalFile object will be deleted which deletes all associated
ResourceFile objects and file type metadata objects.
:returns: The name or id of the file which was deleted
Return Type: string or integer
Raises:
Exceptions.NotAuthorized - The user is not authorized
Exceptions.NotFound - The resource identified by pid does not exist or the file identified by
file does not exist
Exception.ServiceFailure - The service is unable to process the request
Note: This does not handle immutability as previously intended.
"""
resource = utils.get_resource_by_shortkey(pk)
res_cls = resource.__class__
for f in ResourceFile.objects.filter(object_id=resource.id):
if filter_condition(filename_or_id, f):
if delete_logical_file:
if f.has_logical_file and not f.logical_file.is_fileset:
# delete logical file if any resource file that belongs to logical file
# gets deleted for any logical file other than fileset logical file
# logical_delete() calls this function (delete_resource_file())
# to delete each of its contained ResourceFile objects
f.logical_file.logical_delete(user)
return filename_or_id
signals.pre_delete_file_from_resource.send(sender=res_cls, file=f,
resource=resource, user=user)
file_name = delete_resource_file_only(resource, f)
# This presumes that the file is no longer in django
delete_format_metadata_after_delete_file(resource, file_name)
signals.post_delete_file_from_resource.send(sender=res_cls, resource=resource)
# set to private if necessary -- AFTER post_delete_file handling
resource.update_public_and_discoverable() # set to False if necessary
# generate bag
utils.resource_modified(resource, user, overwrite_bag=False)
return filename_or_id
# if execution gets here, file was not found
raise ObjectDoesNotExist(str.format("resource {}, file {} not found",
resource.short_id, filename_or_id))
def handle(self, *args, **options):
ind = BaseResourceIndex()
if len(options['resource_ids']) > 0: # an array of resource short_id to check.
for rid in options['resource_ids']:
print("updating resource {}".format(rid))
try:
r = BaseResource.objects.get(short_id=rid)
if r.show_in_discover:
ind.update_object(r)
except BaseResource.DoesNotExist:
print("resource {} does not exist in Django".format(rid))
except Exception as e:
print("resource {} generated exception {}".format(rid, str(e)))
else:
sqs = SearchQuerySet().all()
print("SOLR count = {}".format(sqs.count()))
dqs = BaseResource.objects.filter(Q(raccess__discoverable=True) |
Q(raccess__public=True))
print("Django count = {}".format(dqs.count()))
# what is in Django that isn't in SOLR
print("checking for resources in Django that aren't in SOLR...")
found_in_solr = set()
for r in list(sqs):
found_in_solr.add(r.short_id) # enable fast matching
django_indexed = 0
django_replaced = 0
django_refreshed = 0
for r in dqs:
try:
resource = get_resource_by_shortkey(r.short_id, or_404=False)
if resource.show_in_discover:
django_indexed += 1
else:
django_replaced += 1
except BaseResource.DoesNotExist:
# race condition in processing while in production
print("resource {} no longer found in Django.".format(r.short_id))
continue
except Exception as e:
print("resource {} generated exception {}".format(r.short_id, str(e)))
if resource.show_in_discover and r.short_id not in found_in_solr:
print("{} {} NOT FOUND in SOLR: adding to index".format(
r.short_id, resource.discovery_content_type))
try:
ind.update_object(r)
django_refreshed += 1
except Exception as e:
print("resource {} generated exception {}".format(r.short_id, str(e)))
# # This always returns True whether or not SOLR needs updating
# # This is likely a Haystack bug.
# elif ind.should_update(r):
# update everything to be safe.
elif options['force']:
if r.show_in_discover:
print("{} {}: refreshing index (forced)".format(
r.short_id, resource.discovery_content_type))
try:
ind.update_object(r)
django_refreshed += 1
except Exception as e:
print("resource {} generated exception {}".format(r.short_id, str(e)))
# what is in SOLR that isn't in Django:
print("checking for resources in SOLR that aren't in Django...")
sqs = SearchQuerySet().all() # refresh for changes from above
solr_indexed = 0
solr_replaced = 0
solr_deleted = 0
for r in sqs:
try:
resource = get_resource_by_shortkey(r.short_id, or_404=False)
if resource.show_in_discover:
solr_indexed += 1
else:
solr_replaced += 1
except BaseResource.DoesNotExist:
print("SOLR resource {} NOT FOUND in Django; cannot remove from SOLR"
.format(r.short_id))
solr_deleted += 1
continue
print("Django contains {} discoverable resources and {} replaced resources"
.format(django_indexed, django_replaced))
print("{} resources in Django refreshed in SOLR".format(django_refreshed))
print("SOLR contains {} discoverable resources and {} replaced resources"
.format(solr_indexed, solr_replaced))
print("{} resources not in Django removed from SOLR".format(solr_deleted))
