def handle(self, *args, **options): if len(options['resource_ids']) > 0: # an array of resource short_id to check. for rid in options['resource_ids']: try: resource = get_resource_by_shortkey(rid) except BaseResource.DoesNotExist: msg = "Resource with id {} not found in Django Resources".format(rid) print(msg) continue print("LOOKING FOR RELATION ERRORS FOR RESOURCE {}".format(rid)) check_relations(resource) else: # check all resources print("LOOKING FOR RELATION ERRORS FOR ALL RESOURCES") for r in BaseResource.objects.all(): try: resource = get_resource_by_shortkey(r.short_id) except BaseResource.DoesNotExist: msg = "Resource with id {} not found in Django Resources".format(rid) print(msg) continue print("LOOKING FOR RELATION ERRORS FOR RESOURCE {}".format(rid)) check_relations(resource)
def handle(self, *args, **options): logger = logging.getLogger(__name__) log_errors = options['log'] echo_errors = not options['log'] if len(options['resource_ids']) > 0: # an array of resource short_id to check. for rid in options['resource_ids']: try: resource = get_resource_by_shortkey(rid) except BaseResource.NotFoundException: msg = "resource {} not found".format(rid) print(msg) continue _, count = repair_resource(resource, echo_errors=echo_errors, log_errors=log_errors, return_errors=False) if count: msg = "... affected resource {} has type {}, title '{}'"\ .format(resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace')) if log_errors: logger.info(msg) if echo_errors: print(msg) else: # check all resources print("REPAIRING ALL RESOURCES") for r in BaseResource.objects.all(): try: resource = get_resource_by_shortkey(r.short_id) except BaseResource.NotFoundException: msg = "resource {} not found".format(rid) print(msg) continue _, count = repair_resource(resource, echo_errors=echo_errors, log_errors=log_errors, return_errors=False) if count: msg = "... affected resource {} has type {}, title '{}'"\ .format(resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace')) if log_errors: logger.info(msg) if echo_errors: print(msg)
def test_resource_create_with_core_and_extra_metadata(self): rtype = 'GenericResource' title = 'My Test resource' metadata = [] metadata.append({'coverage': {'type': 'period', 'value': {'start': '01/01/2000', 'end': '12/12/2010'}}}) extra_metadata = {'latitude': '40', 'longitude': '-110'} params = {'resource_type': rtype, 'title': title, 'metadata': json.dumps(metadata), 'extra_metadata': json.dumps(extra_metadata)} rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) # test core metadata content = json.loads(response.content) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) self.assertEqual(resource.metadata.coverages.all().count(), 1) self.assertEqual(resource.metadata.coverages.filter(type='period').count(), 1) coverage = resource.metadata.coverages.all().first() self.assertEqual(parser.parse(coverage.value['start']).date(), parser.parse('01/01/2000').date()) self.assertEqual(parser.parse(coverage.value['end']).date(), parser.parse('12/12/2010').date()) # test extra metadata self.assertEquals(resource.extra_metadata.get('latitude'), '40') self.assertEquals(resource.extra_metadata.get('longitude'), '-110') self.resources_to_delete.append(res_id)
def write_metadata_to_resource(self, resource): """ Write metadata to resource :param resource: ModelInstanceResource instance """ super(ModelInstanceResourceMeta, self).write_metadata_to_resource(resource) resource.metadata._model_output.update(includes_output=self.has_model_output) if self.executed_by_uri: uri_stripped = self.executed_by_uri.strip('/') short_id = os.path.basename(uri_stripped) if short_id == '': msg = "ExecutedBy URL {0} does not contain a model program resource ID, " msg += "for resource {1}" msg = msg.format(self.executed_by_uri, self.root_uri) raise GenericResourceMeta.ResourceMetaException(msg) # Make sure the resource specified by ExecutedBy exists try: executed_by_resource = get_resource_by_shortkey(short_id, or_404=False) except BaseResource.DoesNotExist: msg = "ExecutedBy resource {0} does not exist.".format(short_id) raise HsDeserializationDependencyException(short_id, msg) executed_by = resource.metadata.executed_by if not executed_by: # Create ExecutedBy.create(content_object=resource.metadata, model_name=short_id) else: # Update ExecutedBy.update(executed_by.element_id, model_name=short_id)
def delete_resource(pk): """ Deletes a resource managed by HydroShare. The caller must be an owner of the resource or an administrator to perform this function. The operation removes the resource from further interaction with HydroShare services and interfaces. The implementation may delete the resource bytes, and should do so since a delete operation may be in response to a problem with the resource (e.g., it contains malicious content, is inappropriate, or is subject to a legal request). If the resource does not exist, the Exceptions.NotFound exception is raised. REST URL: DELETE /resource/{pid} Parameters: pid - The unique HydroShare identifier of the resource to be deleted Returns: The pid of the resource that was deleted Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request Note: Only HydroShare administrators will be able to delete formally published resour """ #utils.get_resource_by_shortkey(pk).delete() res = utils.get_resource_by_shortkey(pk) res.delete() return pk
def publish_resource(pk): """ Formally publishes a resource in HydroShare. Triggers the creation of a DOI for the resource, and triggers the exposure of the resource to the HydroShare DataONE Member Node. The user must be an owner of a resource or an adminstrator to perform this action. REST URL: PUT /publishResource/{pid} Parameters: pid - Unique HydroShare identifier for the resource to be formally published. Returns: The pid of the resource that was published Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request Note: This is different than just giving public access to a resource via access control rul """ resource = utils.get_resource_by_shortkey(pk) resource.published_and_frozen = True resource.frozen = True resource.edit_users = [] resource.edit_groups = [] resource.save()
def update_resource_file(pk, filename, f): """ Called by clients to update an individual file within a HydroShare resource. REST URL: PUT /resource/{pid}/files/{filename} Parameters: pid - Unique HydroShare identifier for the resource from which the file will be extracted. filename - The data bytes of the file that will be extracted from the resource identified by pid file - the data bytes of the file to update Returns: The bytes of the file extracted from the resource Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified does not exist or the file identified by filename does not exist Exception.ServiceFailure - The service is unable to process the request """ resource = utils.get_resource_by_shortkey(pk) for rf in ResourceFile.objects.filter(object_id=resource.id): if os.path.basename(rf.resource_file.name) == filename: rf.resource_file.delete() rf.resource_file = File(f) if not isinstance(f, UploadedFile) else f rf.save() return rf else: raise ObjectDoesNotExist(filename)
def short_url(request, *args, **kwargs): try: shortkey = kwargs['shortkey'] except KeyError: raise TypeError('shortkey must be specified...') m = get_resource_by_shortkey(shortkey) return HttpResponseRedirect(m.get_absolute_url())
def get(self, request, pk): """ Get resource system metadata, as well as URLs to the bag and science metadata """ view_utils.authorize(request, pk, view=True, full=True) res = get_resource_by_shortkey(pk) ser = self.get_serializer_class()(self.resourceToResourceListItem(res)) return Response(data=ser.data, status=status.HTTP_200_OK)
def test_refts_creation_via_rest_api(self): rtype = 'RefTimeSeriesResource' title = 'My Test RefTS res' ref_url = "http://data.iutahepscor.org/LoganRiverWOF/REST/waterml_1_1.svc/datavalues?" \ "location=iutah:LR_WaterLab_AA&variable=iutah:WaterTemp_EXO&" \ "startDate=2014-12-02T19:45:00Z&endDate=2014-12-05T19:45:00Z" ref_type = "rest" # test the "ref_url" rest endpoint is up response = requests.get(ref_url) self.assertEqual(response.status_code, status.HTTP_200_OK) metadata = [] # add core metadata metadata.append({'relation': {'type': 'isPartOf', 'value': 'http://hydroshare.org/resource/001'}}) # add refts-specific metadata metadata.append({"referenceurl": {"value": ref_url, "type": ref_type}}) # post to rest api params = {'resource_type': rtype, 'title': title, 'metadata': json.dumps(metadata), } rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) content = json.loads(response.content) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) # test core metadata self.assertEqual(resource.metadata.relations.all().count(), 1) relation = resource.metadata.relations.all().first() self.assertEqual(relation.type, 'isPartOf') self.assertEqual(relation.value, 'http://hydroshare.org/resource/001') # test resource-specific metadata self.assertEqual(resource.resource_type.lower(), "reftimeseriesresource") self.assertEqual(resource.metadata.referenceURLs.all().count(), 1) referenceURLs = resource.metadata.referenceURLs.all().first() self.assertEquals(referenceURLs.value, ref_url) self.assertEquals(referenceURLs.type, ref_type) self.assertEqual(resource.metadata.sites.all().count(), 1) sites = resource.metadata.sites.all().first() self.assertEquals(sites.code, "LR_WaterLab_AA") self.assertEqual(resource.metadata.variables.all().count(), 1) variables = resource.metadata.variables.all().first() self.assertEquals(variables.code, "WaterTemp_EXO") self.resources_to_delete.append(res_id)
def get_science_metadata_xml(resource_short_id): """ Gets science metadata as an xml string for a specified resource :param resource_short_id: id of the resource :return: science metadata as an xml string """ res = utils.get_resource_by_shortkey(resource_short_id) return res.metadata.get_xml()
def handle(self, *args, **options): if len(options['resource_ids']) > 0: # an array of resource short_id to check. for rid in options['resource_ids']: resource = get_resource_by_shortkey(rid) self.repair_filtered_solr(resource, options) else: for resource in BaseResource.objects.all(): self.repair_filtered_solr(resource, options)
def copy_res_to_specified_federated_irods_server_as_needed(app_shortkey, res_shortkey, user): """ When app resource has iRODS federation target path and target resource defined as extended metadata, the resource needs to be pushed to the specified iRODS federation path in the specified iRODS storage resource. :param app_shortkey: shortkey of the app tool resource to be launched :param res_shortkey: shortkey of the resource launching the app tool resource :param user: requesting user :return: """ # check whether irods_path_key and irods_resc_key are added as extended metadata of the # app tool resource, and if they are, push resource to specified iRODS target accordingly filterd_app_obj = BaseResource.objects.filter(short_id=app_shortkey).filter( extra_metadata__has_key=irods_path_key).filter( extra_metadata__has_key=irods_resc_key).first() if filterd_app_obj: try: res = get_resource_by_shortkey(res_shortkey) app_res = get_resource_by_shortkey(app_shortkey) user_auth_key_exist = BaseResource.objects.filter(short_id=app_shortkey).filter( extra_metadata__has_key=user_auth_flag_key).first() if user_auth_key_exist: req_user_auth = True \ if app_res.extra_metadata[user_auth_flag_key].lower() == 'true' \ else False if req_user_auth and not user.is_authenticated(): err_msg = "Only authorized users can launch the web app tool - " \ "Please sign in first." raise WebAppLaunchException(err_msg) irods_path = app_res.extra_metadata[irods_path_key] irods_resc = app_res.extra_metadata[irods_resc_key] istorage = res.get_irods_storage() src_path = res.root_path # delete all temporary resources copied to this user's space before pushing resource dest_path = os.path.join(irods_path, user.username) if istorage.exists(dest_path): istorage.delete(dest_path) dest_path = os.path.join(dest_path, res_shortkey) istorage.copyFiles(src_path, dest_path, irods_resc) except SessionException as ex: raise WebAppLaunchException(ex.stderr) except Exception as ex: raise WebAppLaunchException(ex.message)
def delete_metadata_element(resource_short_id, element_model_name, element_id): """ Deletes a specific type of metadata element for a specified resource :param resource_short_id: id of the resource for which metadata element to be deleted :param element_model_name: metadata element name (e.g., creator) :param element_id: id of the metadata element to be deleted :return: """ res = utils.get_resource_by_shortkey(resource_short_id) res.metadata.delete_element(element_model_name, element_id)
def create_metadata_element(resource_short_id, element_model_name, **kwargs): """ Creates a specific type of metadata element for a given resource :param resource_short_id: id of the resource for which a metadata element needs to be created :param element_model_name: metadata element name (e.g., creator) :param kwargs: metadata element attribute name/value pairs for all those attributes that require a value :return: """ res = utils.get_resource_by_shortkey(resource_short_id) res.metadata.create_element(element_model_name, **kwargs)
def handle(self, *args, **options): if len(options['resource_ids']) > 0: # an array of resource short_id to check. for rid in options['resource_ids']: resource = get_resource_by_shortkey(rid) if (options['type'] is None or resource.resource_type == options['type']) and \ (options['storage'] is None or resource.storage_type == options['storage']): CheckResource(rid).test() else: for resource in BaseResource.objects.all(): if (options['type'] is None or resource.resource_type == options['type']) and \ (options['storage'] is None or resource.storage_type == options['storage']): CheckResource(resource.short_id).test()
def update_metadata_element(resource_short_id, element_model_name, element_id, **kwargs): """ Updates the data associated with a metadata element for a specified resource :param resource_short_id: id of the resource for which a metadata element needs to be updated :param element_model_name: metadata element name (e.g., creator) :param element_id: id of the metadata element to be updated :param kwargs: metadata element attribute name/value pairs for all those attributes that need update :return: """ res = utils.get_resource_by_shortkey(resource_short_id) res.metadata.update_element(element_model_name, element_id, **kwargs)
def update_science_metadata(pk, metadata=None, keywords=None, **kwargs): """ Called by clients to update the science metadata for a resource in HydroShare. REST URL: PUT /scimeta/{pid} Parameters: pid - Unique HydroShare identifier for the resource that is to be updated. ScienceMetadata - The data bytes of the ScienceMetadata that will update the existing Science Metadata for the resource identified by pid Returns: The pid assigned to the resource whose Science Metadata was updated Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the resource is incomplete Exception.ServiceFailure - The service is unable to process the request Notes: For mutable resources (resources that have not been formally published), the update overwrites existing Science Metadata using the ScienceMetadata that is passed to this method. For immutable resources (formally published resources), this method creates a new resource that is a new version of the formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata. HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. hydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) if keywords: AssignedKeyword.objects.filter(object_pk=resource.id).delete() ks = [Keyword.objects.get_or_create(title=k) for k in keywords] ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse for k in ks: AssignedKeyword.objects.create(content_object=resource.id, keyword=k) # for creating metadata elements based on the new metadata implementation if metadata: _update_science_metadata(resource, metadata) if kwargs: for field, value in kwargs.items(): setattr(resource, field, value) resource.save()
def add_zip_file_contents_to_resource(pk, zip_file_path): """Add zip file to existing resource and remove tmp zip file.""" zfile = None resource = None try: resource = utils.get_resource_by_shortkey(pk, or_404=False) zfile = zipfile.ZipFile(zip_file_path) num_files = len(zfile.infolist()) zcontents = utils.ZipContents(zfile) files = zcontents.get_files() resource.file_unpack_status = 'Running' resource.save() for i, f in enumerate(files): logger.debug("Adding file {0} to resource {1}".format(f.name, pk)) utils.add_file_to_resource(resource, f) resource.file_unpack_message = "Imported {0} of about {1} file(s) ...".format( i, num_files) resource.save() # This might make the resource unsuitable for public consumption resource.update_public_and_discoverable() # TODO: this is a bit of a lie because a different user requested the bag overwrite utils.resource_modified(resource, resource.creator, overwrite_bag=False) # Call success callback resource.file_unpack_message = None resource.file_unpack_status = 'Done' resource.save() except BaseResource.DoesNotExist: msg = "Unable to add zip file contents to non-existent resource {pk}." msg = msg.format(pk=pk) logger.error(msg) except: exc_info = "".join(traceback.format_exception(*sys.exc_info())) if resource: resource.file_unpack_status = 'Error' resource.file_unpack_message = exc_info resource.save() if zfile: zfile.close() logger.error(exc_info) finally: # Delete upload file os.unlink(zip_file_path)
def put(self, request, pk): """ Update access rules """ view_utils.authorize(request, pk, edit=True, full=True) access_rules_validator = serializers.AccessRulesRequestValidator(data=request.data) if not access_rules_validator.is_valid(): raise ValidationError(detail=access_rules_validator.errors) validated_request_data = access_rules_validator.validated_data res = get_resource_by_shortkey(pk) res.public = validated_request_data["public"] res.save() return Response(data={"resource_id": pk}, status=status.HTTP_200_OK)
def update_web_services(services_url, api_token, timeout, publish_urls, res_id): """Update web services hosted by GeoServer and HydroServer. This function sends a resource id to the HydroShare web services manager application, which will check the current status of the resource and register or unregister services hosted by GeoServer and HydroServer. The HydroShare web services manager will return a list of endpoint URLs for both the resource and individual aggregations. If publish_urls is set to True, these endpoints will be added to the extra metadata fields of the resource and aggregations. """ session = requests.Session() session.headers.update( {"Authorization": " ".join(("Token", str(api_token)))} ) rest_url = str(services_url) + "/" + str(res_id) + "/" try: response = session.post(rest_url, timeout=timeout) if publish_urls and response.status_code == status.HTTP_201_CREATED: try: resource = utils.get_resource_by_shortkey(res_id) response_content = json.loads(response.content) for key, value in response_content["resource"].iteritems(): resource.extra_metadata[key] = value resource.save() for url in response_content["content"]: lf = resource.logical_files[[i.aggregation_name for i in resource.logical_files].index( url["layer_name"].encode("utf-8") )] lf.metadata.extra_metadata["Web Services URL"] = url["message"] lf.metadata.save() except Exception as e: logger.error(e) return e return response except (requests.exceptions.RequestException, ValueError) as e: logger.error(e) return e
def add_resource_files(pk, *files): """ Called by clients to update a resource in HydroShare by adding a single file. REST URL: PUT /resource/{pid}/files/{file} Parameters: pid - Unique HydroShare identifier for the resource that is to be updated. file - The data bytes of the file that will be added to the existing resource identified by pid Returns: The pid assigned to the updated resource Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the file is invalid Exception.ServiceFailure - The service is unable to process the request Notes: For mutable resources (resources not formally published), the update adds the file that is passed to this method to the resource. For immutable resources (formally published resources), this method creates a new resource that is a new version of the formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) ret = [] for file in files: ret.append(ResourceFile.objects.create( content_object=resource, resource_file=File(file) if not isinstance(file, UploadedFile) else file )) # add format metadata element if necessary file_format_type = utils.get_file_mime_type(file.name) if file_format_type not in [mime.value for mime in resource.metadata.formats.all()]: resource.metadata.create_element('format', value=file_format_type) return ret
def get_capabilities(pk): """ Describes API services exposed for a resource. If there are extra capabilites for a particular resource type over and above the standard Hydroshare API, then this API call will list these REST URL: GET /capabilites/{pid} Parameters: Unique HydroShare identifier for the resource whose capabilites are to be retrieved. Return Type: Capabilites Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request """ res = utils.get_resource_by_shortkey(pk) return getattr(res, 'extra_capabilities', lambda: None)()
def put(self, request, pk): """ Update access rules """ # only resource owners are allowed to change resource flags (e.g., public) view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.SET_RESOURCE_FLAG) access_rules_validator = serializers.AccessRulesRequestValidator(data=request.data) if not access_rules_validator.is_valid(): raise ValidationError(detail=access_rules_validator.errors) validated_request_data = access_rules_validator.validated_data res = get_resource_by_shortkey(pk) try: res.set_public(validated_request_data['public'], request.user) except CoreValidationError: return Response(data={'resource_id': pk}, status=status.HTTP_403_FORBIDDEN) return Response(data={'resource_id': pk}, status=status.HTTP_200_OK)
def get_system_metadata(pk): """ Describes the resource identified by the pid by returning the associated system metadata object. If the resource does not exist, Exceptions.NotFound must be raised. REST URL: GET /sysmeta/{pid} Parameters: pk - Unique HydroShare identifier for the resource whose system metadata is to be retrieved. Returns: System metadata document describing the resource. Return Type: SystemMetadata Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request """ return utils.get_resource_by_shortkey(pk)
def get_revisions(pk): """ Returns a list of pids for resources that are revisions of the resource identified by the specified pid. REST URL: GET /revisions/{pid} Parameters: pid - Unique HydroShare identifier for the resource whose revisions are to be retrieved. Returns: List of pids for resources that are revisions of the specified resource. Return Type: List of pids Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The Resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request """ return utils.get_resource_by_shortkey(pk).bags.all()
def create_temp_zip(resource_id, input_path, output_path, sf_aggregation, sf_zip=False): """ Create temporary zip file from input_path and store in output_path :param input_path: full irods path of input starting with federation path :param output_path: full irods path of output starting with federation path :param sf_aggregation: if True, include logical metadata files """ from hs_core.hydroshare.utils import get_resource_by_shortkey res = get_resource_by_shortkey(resource_id) istorage = res.get_irods_storage() # invoke federated storage as necessary if res.resource_type == "CompositeResource": if '/data/contents/' in input_path: short_path = input_path.split('/data/contents/')[1] # strip /data/contents/ res.create_aggregation_xml_documents(aggregation_name=short_path) else: # all metadata included, e.g., /data/* res.create_aggregation_xml_documents() try: if sf_zip: # input path points to single file aggregation # ensure that foo.zip contains aggregation metadata # by copying these into a temp subdirectory foo/foo parallel to where foo.zip is stored temp_folder_name, ext = os.path.splitext(output_path) # strip zip to get scratch dir head, tail = os.path.split(temp_folder_name) # tail is unqualified folder name "foo" out_with_folder = os.path.join(temp_folder_name, tail) # foo/foo is subdir to zip istorage.copyFiles(input_path, out_with_folder) if sf_aggregation: try: istorage.copyFiles(input_path + '_resmap.xml', out_with_folder + '_resmap.xml') except SessionException: logger.error("cannot copy {}".format(input_path + '_resmap.xml')) try: istorage.copyFiles(input_path + '_meta.xml', out_with_folder + '_meta.xml') except SessionException: logger.error("cannot copy {}".format(input_path + '_meta.xml')) istorage.zipup(temp_folder_name, output_path) istorage.delete(temp_folder_name) # delete working directory; this isn't the zipfile else: # regular folder to zip istorage.zipup(input_path, output_path) except SessionException as ex: logger.error(ex.stderr) return False return True
def get_resource_map(pk): """ Describes the resource identified by the pid by returning the associated resource map document. If the resource does not exist, Exceptions.NotFound must be raised. REST URL: GET /resourcemap/{pid} Parameters: pid - Unique HydroShare identifier for the resource whose resource map is to be retrieved. Returns: Resource map document describing the resource. Return Type: ResourceMap Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request """ return utils.get_resource_by_shortkey(pk)
def handle(self, *args, **options): if options['username'] is None or options['resource_id'] is None: usage() exit(1) username = options['username'] resource_id = options['resource_id'] user = user_from_name(username) if user is None: usage() exit(1) try: resource = get_resource_by_shortkey(resource_id, or_404=False) except BaseResource.DoesNotExist: print("No such resource {}.".format(resource_id)) usage() exit(1) print(access_provenance(user, resource))
def test_resource_create_with_extra_metadata(self): rtype = 'GenericResource' title = 'My Test resource' extra_metadata = {'latitude': '40', 'longitude': '-110'} params = {'resource_type': rtype, 'title': title, 'extra_metadata': json.dumps(extra_metadata)} rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) content = json.loads(response.content) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) # test extra metadata self.assertEquals(resource.extra_metadata.get('latitude'), '40') self.assertEquals(resource.extra_metadata.get('longitude'), '-110') self.resources_to_delete.append(res_id)
def write_metadata_to_resource(self, resource): """ Write metadata to resource :param resource: ModelInstanceResource instance """ super(ModelInstanceResourceMeta, self).write_metadata_to_resource(resource) resource.metadata._model_output.update( includes_output=self.has_model_output) if self.executed_by_uri: uri_stripped = self.executed_by_uri.strip('/') short_id = os.path.basename(uri_stripped) if short_id == '': msg = "ExecutedBy URL {0} does not contain a model program resource ID, " msg += "for resource {1}" msg = msg.format(self.executed_by_uri, self.root_uri) raise GenericResourceMeta.ResourceMetaException(msg) # Make sure the resource specified by ExecutedBy exists try: executed_by_resource = get_resource_by_shortkey(short_id, or_404=False) except BaseResource.DoesNotExist: msg = "ExecutedBy resource {0} does not exist.".format( short_id) raise HsDeserializationDependencyException(short_id, msg) executed_by = resource.metadata.executed_by if not executed_by: # Create ExecutedBy.create(content_object=resource.metadata, model_name=short_id) else: # Update ExecutedBy.update(executed_by.element_id, model_name=short_id)
def create_temp_zip(resource_id, input_path, output_path, aggregation_name=None, sf_zip=False): """ Create temporary zip file from input_path and store in output_path :param resource_id: the short_id of a resource :param input_path: full irods path of input starting with federation path :param output_path: full irods path of output starting with federation path :param aggregation_name: The name of the aggregation to zip :param sf_zip: signals a single file to zip """ from hs_core.hydroshare.utils import get_resource_by_shortkey res = get_resource_by_shortkey(resource_id) aggregation = None if aggregation_name: aggregation = res.get_aggregation_by_aggregation_name(aggregation_name) istorage = res.get_irods_storage() # invoke federated storage as necessary if res.resource_type == "CompositeResource": if '/data/contents/' in input_path: short_path = input_path.split('/data/contents/')[ 1] # strip /data/contents/ res.create_aggregation_xml_documents(path=short_path) else: # all metadata included, e.g., /data/* res.create_aggregation_xml_documents() try: if aggregation or sf_zip: # input path points to single file aggregation # ensure that foo.zip contains aggregation metadata # by copying these into a temp subdirectory foo/foo parallel to where foo.zip is stored temp_folder_name, ext = os.path.splitext( output_path) # strip zip to get scratch dir head, tail = os.path.split( temp_folder_name) # tail is unqualified folder name "foo" out_with_folder = os.path.join(temp_folder_name, tail) # foo/foo is subdir to zip istorage.copyFiles(input_path, out_with_folder) if aggregation: try: istorage.copyFiles(aggregation.map_file_path, temp_folder_name) except SessionException: logger.error("cannot copy {}".format( aggregation.map_file_path)) try: istorage.copyFiles(aggregation.metadata_file_path, temp_folder_name) except SessionException: logger.error("cannot copy {}".format( aggregation.metadata_file_path)) for file in aggregation.files.all(): try: istorage.copyFiles(file.storage_path, temp_folder_name) except SessionException: logger.error("cannot copy {}".format( file.storage_path)) istorage.zipup(temp_folder_name, output_path) istorage.delete( temp_folder_name ) # delete working directory; this isn't the zipfile else: # regular folder to zip istorage.zipup(input_path, output_path) except SessionException as ex: logger.error(ex.stderr) return False return True
def create_bag_by_irods(resource_id): """Create a resource bag on iRODS side by running the bagit rule and ibun zip. This function runs as a celery task, invoked asynchronously so that it does not block the main web thread when it creates bags for very large files which will take some time. :param resource_id: the resource uuid that is used to look for the resource to create the bag for. :return: True if bag creation operation succeeds; False if there is an exception raised or resource does not exist. """ from hs_core.hydroshare.utils import get_resource_by_shortkey res = get_resource_by_shortkey(resource_id) istorage = res.get_irods_storage() metadata_dirty = istorage.getAVU(res.root_path, 'metadata_dirty') # if metadata has been changed, then regenerate metadata xml files if metadata_dirty is None or metadata_dirty.lower() == "true": try: create_bag_files(res) except Exception as ex: logger.error('Failed to create bag files. Error:{}'.format( ex.message)) return False bag_full_name = 'bags/{res_id}.zip'.format(res_id=resource_id) if res.resource_federation_path: irods_bagit_input_path = os.path.join(res.resource_federation_path, resource_id) is_exist = istorage.exists(irods_bagit_input_path) # check to see if bagit readme.txt file exists or not bagit_readme_file = '{fed_path}/{res_id}/readme.txt'.format( fed_path=res.resource_federation_path, res_id=resource_id) is_bagit_readme_exist = istorage.exists(bagit_readme_file) bagit_input_path = "*BAGITDATA='{path}'".format( path=irods_bagit_input_path) bagit_input_resource = "*DESTRESC='{def_res}'".format( def_res=settings.HS_IRODS_USER_ZONE_DEF_RES) bag_full_name = os.path.join(res.resource_federation_path, bag_full_name) bagit_files = [ '{fed_path}/{res_id}/bagit.txt'.format( fed_path=res.resource_federation_path, res_id=resource_id), '{fed_path}/{res_id}/manifest-md5.txt'.format( fed_path=res.resource_federation_path, res_id=resource_id), '{fed_path}/{res_id}/tagmanifest-md5.txt'.format( fed_path=res.resource_federation_path, res_id=resource_id), '{fed_path}/bags/{res_id}.zip'.format( fed_path=res.resource_federation_path, res_id=resource_id) ] else: is_exist = istorage.exists(resource_id) # check to see if bagit readme.txt file exists or not bagit_readme_file = '{res_id}/readme.txt'.format(res_id=resource_id) is_bagit_readme_exist = istorage.exists(bagit_readme_file) irods_dest_prefix = "/" + settings.IRODS_ZONE + "/home/" + settings.IRODS_USERNAME irods_bagit_input_path = os.path.join(irods_dest_prefix, resource_id) bagit_input_path = "*BAGITDATA='{path}'".format( path=irods_bagit_input_path) bagit_input_resource = "*DESTRESC='{def_res}'".format( def_res=settings.IRODS_DEFAULT_RESOURCE) bagit_files = [ '{res_id}/bagit.txt'.format(res_id=resource_id), '{res_id}/manifest-md5.txt'.format(res_id=resource_id), '{res_id}/tagmanifest-md5.txt'.format(res_id=resource_id), 'bags/{res_id}.zip'.format(res_id=resource_id) ] # only proceed when the resource is not deleted potentially by another request # when being downloaded if is_exist: # if bagit readme.txt does not exist, add it. if not is_bagit_readme_exist: from_file_name = getattr(settings, 'HS_BAGIT_README_FILE_WITH_PATH', 'docs/bagit/readme.txt') istorage.saveFile(from_file_name, bagit_readme_file, True) # call iRODS bagit rule here bagit_rule_file = getattr(settings, 'IRODS_BAGIT_RULE', 'hydroshare/irods/ruleGenerateBagIt_HS.r') try: # call iRODS run and ibun command to create and zip the bag, ignore SessionException # for now as a workaround which could be raised from potential race conditions when # multiple ibun commands try to create the same zip file or the very same resource # gets deleted by another request when being downloaded istorage.runBagitRule(bagit_rule_file, bagit_input_path, bagit_input_resource) istorage.zipup(irods_bagit_input_path, bag_full_name) istorage.setAVU(irods_bagit_input_path, 'bag_modified', "false") return True except SessionException as ex: # if an exception occurs, delete incomplete files potentially being generated by # iRODS bagit rule and zipping operations for fname in bagit_files: if istorage.exists(fname): istorage.delete(fname) logger.error(ex.stderr) return False else: logger.error('Resource does not exist.') return False
def test_get_resource_by_shortkey(self): self.assertEqual( utils.get_resource_by_shortkey(self.res.short_id), self.res )
def add_resource_files(pk, *files, **kwargs): """ Called by clients to update a resource in HydroShare by adding one or more files. REST URL: PUT /resource/{pid}/files/{file} Parameters: pk - Unique HydroShare identifier for the resource that is to be updated. files - A list of file-like objects representing files that will be added to the existing resource identified by pid Returns: A list of ResourceFile objects added to the resource Return Type: list Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the file is invalid Exception.ServiceFailure - The service is unable to process the request Notes: This does **not** handle mutability; changes to immutable resources should be denied elsewhere. """ resource = utils.get_resource_by_shortkey(pk) ret = [] source_names = kwargs.pop('source_names', []) full_paths = kwargs.pop('full_paths', {}) auto_aggregate = kwargs.pop('auto_aggregate', True) if __debug__: assert(isinstance(source_names, list)) folder = kwargs.pop('folder', None) if __debug__: # assure that there are no spurious kwargs left. for k in kwargs: print("kwargs[{}]".format(k)) assert len(kwargs) == 0 prefix_path = 'data/contents' if folder is None or folder == prefix_path: base_dir = "" elif folder.startswith(prefix_path): base_dir = folder[len(prefix_path) + 1:] else: base_dir = folder new_folders = set() for f in files: full_dir = base_dir if f in full_paths: # TODO, put this in it's own method? full_path = full_paths[f] dir_name = os.path.dirname(full_path) # Only do join if dir_name is not empty, otherwise, it'd result in a trailing slash full_dir = os.path.join(base_dir, dir_name) if dir_name else base_dir if full_dir: new_folders.add(os.path.join(resource.file_path, full_dir)) ret.append(utils.add_file_to_resource(resource, f, folder=full_dir)) else: ret.append(utils.add_file_to_resource(resource, f, folder=None)) if len(source_names) > 0: for ifname in source_names: ret.append(utils.add_file_to_resource(resource, None, folder=folder, source_name=ifname)) if not ret: # no file has been added, make sure data/contents directory exists if no file is added utils.create_empty_contents_directory(resource) else: if resource.resource_type == "CompositeResource" and auto_aggregate: utils.check_aggregations(resource, new_folders, ret) # some file(s) added, need to update quota usage update_quota_usage(res=resource) return ret
def test_resource_create_with_extended_metadata(self): """ The followings are the extended metadata elements for the NetCDF resource that can be passed as part of the 'metadata' parameter when creating a resource: originalcoverage variable """ rtype = 'NetcdfResource' title = 'My Test resource' metadata = [] # originalcover value = {"northlimit": 12, "projection": "transverse_mercator", "units": "meter", "southlimit": 10, "eastlimit": 23, "westlimit": 2} metadata.append({'originalcoverage': {'value': value, 'projection_string_text': '+proj=tmerc +lon_0=-111.0 ' '+lat_0=0.0 +x_0=500000.0 ' '+y_0=0.0 +k_0=0.9996', 'projection_string_type': 'Proj4 String'}}) # variable (this element is defined in multiple resource types including # NetcdfResource type) var_name = 'SWE' var_type = 'Float' var_shape = 'y,x,time' var_unit = 'm' var_missing_value = '-9999' var_des_name = 'Snow water equivalent' var_method = 'model simulation of UEB' metadata.append({'variable': {'name': var_name, 'type': var_type, 'shape': var_shape, 'unit': var_unit, 'missing_value': var_missing_value, 'descriptive_name': var_des_name, 'method': var_method}}) params = {'resource_type': rtype, 'title': title, 'metadata': json.dumps(metadata), } rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) content = json.loads(response.content.decode()) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) # there should be 1 originalcoverage element self.assertEqual(resource.metadata.ori_coverage.all().count(), 1) ori_coverage = resource.metadata.ori_coverage.all().first() self.assertEqual(ori_coverage.value, value) self.assertEqual(ori_coverage.projection_string_text, '+proj=tmerc +lon_0=-111.0 ' '+lat_0=0.0 +x_0=500000.0 ' '+y_0=0.0 +k_0=0.9996') self.assertEqual(ori_coverage.projection_string_type, 'Proj4 String') # there should be 1 variable element self.assertEqual(resource.metadata.variables.all().count(), 1) variable = resource.metadata.variables.all().first() self.assertEqual(variable.name, var_name) self.assertEqual(variable.type, var_type) self.assertEqual(variable.shape, var_shape) self.assertEqual(variable.unit, var_unit) self.assertEqual(variable.missing_value, var_missing_value) self.assertEqual(variable.descriptive_name, var_des_name) self.assertEqual(variable.method, var_method) self.resources_to_delete.append(res_id)
def delete_resource_file(pk, filename_or_id, user, delete_logical_file=True): """ Deletes an individual file from a HydroShare resource. If the file does not exist, the Exceptions.NotFound exception is raised. REST URL: DELETE /resource/{pid}/files/{filename} Parameters: pk - The unique HydroShare identifier for the resource from which the file will be deleted filename_or_id - Name of the file or id of the file to be deleted from the resource user - requesting user delete_logical_file - If True then the ResourceFile object to be deleted if it is part of a LogicalFile object then the LogicalFile object will be deleted which deletes all associated ResourceFile objects and file type metadata objects. Returns: The name or id of the file which was deleted Return Type: string or integer Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist or the file identified by file does not exist Exception.ServiceFailure - The service is unable to process the request Note: For mutable resources (resources that have not been formally published), this method modifies the resource by deleting the file. For immutable resources (formally published resources), this method creates a new resource that is a new version of the formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) res_cls = resource.__class__ fed_path = resource.resource_federation_path for f in ResourceFile.objects.filter(object_id=resource.id): if filter_condition(filename_or_id, fed_path, f): if delete_logical_file: if f.logical_file is not None: # logical_delete() calls this function (delete_resource_file()) # to delete each of its contained ResourceFile objects f.logical_file.logical_delete(user) return filename_or_id # send signal signals.pre_delete_file_from_resource.send(sender=res_cls, file=f, resource=resource, user=user) file_name = delete_resource_file_only(resource, f) delete_format_metadata_after_delete_file(resource, file_name) break else: raise ObjectDoesNotExist(filename_or_id) if resource.raccess.public or resource.raccess.discoverable: if not resource.can_be_public_or_discoverable: resource.raccess.public = False resource.raccess.discoverable = False resource.raccess.save() signals.post_delete_file_from_resource.send(sender=res_cls, resource=resource) # generate bag utils.resource_modified(resource, user, overwrite_bag=False) return filename_or_id
def create_new_version_resource_task(ori_res_id, username, new_res_id=None): """ Task for creating a new version of a resource Args: ori_res_id: the original resource id that is to be versioned. new_res_id: the new versioned resource id from the original resource. If None, a new resource will be created. username: the requesting user's username Returns: the new versioned resource url as the payload """ try: new_res = None if not new_res_id: new_res = create_empty_resource(ori_res_id, username) new_res_id = new_res.short_id utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id) # copy metadata from source resource to target new-versioned resource except three elements ori_res = utils.get_resource_by_shortkey(ori_res_id) if not new_res: new_res = utils.get_resource_by_shortkey(new_res_id) utils.copy_and_create_metadata(ori_res, new_res) # add or update Relation element to link source and target resources hs_identifier = new_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] ori_res.metadata.create_element('relation', type='isReplacedBy', value=hs_identifier.url) if new_res.metadata.relations.all().filter( type='isVersionOf').exists(): # the original resource is already a versioned resource, and its isVersionOf relation # element is copied over to this new version resource, needs to delete this element so # it can be created to link to its original resource correctly eid = new_res.metadata.relations.all().filter( type='isVersionOf').first().id new_res.metadata.delete_element('relation', eid) hs_identifier = ori_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] new_res.metadata.create_element('relation', type='isVersionOf', value=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new version collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource create_bag(new_res) # since an isReplaceBy relation element is added to original resource, needs to call # resource_modified() for original resource utils.resource_modified(ori_res, by_user=username, overwrite_bag=False) # if everything goes well up to this point, set original resource to be immutable so that # obsoleted resources cannot be modified from REST API ori_res.raccess.immutable = True ori_res.raccess.save() ori_res.save() return new_res.get_absolute_url() except Exception as ex: if new_res: new_res.delete() raise utils.ResourceVersioningException(str(ex)) finally: # release the lock regardless ori_res.locked_time = None ori_res.save()
def test_refts_creation_via_rest_api(self): rtype = 'RefTimeSeriesResource' title = 'My Test RefTS res' ref_url = "http://data.iutahepscor.org/LoganRiverWOF/REST/waterml_1_1.svc/datavalues?" \ "location=iutah:LR_WaterLab_AA&variable=iutah:WaterTemp_EXO&" \ "startDate=2014-12-02T19:45:00Z&endDate=2014-12-05T19:45:00Z" ref_type = "rest" # test the "ref_url" rest endpoint is up response = requests.get(ref_url) self.assertEqual(response.status_code, status.HTTP_200_OK) metadata = [] # add core metadata metadata.append({ 'relation': { 'type': 'isPartOf', 'value': 'http://hydroshare.org/resource/001' } }) # add refts-specific metadata metadata.append({"referenceurl": {"value": ref_url, "type": ref_type}}) # post to rest api params = { 'resource_type': rtype, 'title': title, 'metadata': json.dumps(metadata), } rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) content = json.loads(response.content) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) # test core metadata self.assertEqual(resource.metadata.relations.all().count(), 1) relation = resource.metadata.relations.all().first() self.assertEqual(relation.type, 'isPartOf') self.assertEqual(relation.value, 'http://hydroshare.org/resource/001') # test resource-specific metadata self.assertEqual(resource.resource_type.lower(), "reftimeseriesresource") self.assertEqual(resource.metadata.referenceURLs.all().count(), 1) referenceURLs = resource.metadata.referenceURLs.all().first() self.assertEquals(referenceURLs.value, ref_url) self.assertEquals(referenceURLs.type, ref_type) self.assertEqual(resource.metadata.sites.all().count(), 1) sites = resource.metadata.sites.all().first() self.assertEquals(sites.code, "LR_WaterLab_AA") self.assertEqual(resource.metadata.variables.all().count(), 1) variables = resource.metadata.variables.all().first() self.assertEquals(variables.code, "WaterTemp_EXO") self.resources_to_delete.append(res_id)
def update_resource( pk, edit_users=None, view_users=None, edit_groups=None, view_groups=None, keywords=None, metadata=None, *files, **kwargs): """ Called by clients to update a resource in HydroShare. REST URL: PUT /resource/{pid} Parameters: pid - Unique HydroShare identifier for the resource that is to be updated. resource - The data bytes of the resource that will update the existing resource identified by pid Returns: The pid assigned to the updated resource Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the resource is incomplete Exception.ServiceFailure - The service is unable to process the request Notes: For mutable resources (resources that have not been formally published), the update overwrites existing data and metadata using the resource that is passed to this method. If a user wants to create a copy or modified version of a mutable resource this should be done using HydroShare.createResource(). For immutable resources (formally published resources), this method creates a new resource that is a new version of formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata.HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) if files: ResourceFile.objects.filter(object_id=resource.id).delete() for file in files: ResourceFile.objects.create( content_object=resource, resource_file=File(file) if not isinstance(file, UploadedFile) else file ) if 'owner' in kwargs: owner = utils.user_from_id(kwargs['owner']) resource.owners.add(owner) if edit_users: resource.edit_users.clear() for user in edit_users: user = utils.user_from_id(user) resource.edit_users.add(user) resource.view_users.add(user) if view_users: resource.view_users.clear() for user in view_users: user = utils.user_from_id(user) resource.view_users.add(user) if edit_groups: resource.edit_groups.clear() for group in edit_groups: group = utils.group_from_id(group) resource.edit_groups.add(group) resource.view_groups.add(group) if view_groups: resource.edit_groups.clear() for group in view_groups: group = utils.group_from_id(group) resource.view_groups.add(group) if keywords: AssignedKeyword.objects.filter(object_pk=resource.id).delete() ks = [Keyword.objects.get_or_create(title=k) for k in keywords] ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse for k in ks: AssignedKeyword.objects.create(content_object=resource, keyword=k) # for creating metadata elements based on the new metadata implementation if metadata: _update_science_metadata(resource, metadata) return resource
def handle(self, *args, **options): if not options['resource_id']: raise CommandError('resource_id argument is required') res_id = options['resource_id'] try: res = get_resource_by_shortkey(res_id, or_404=False) except ObjectDoesNotExist: raise CommandError("No Resource found for id {}".format(res_id)) if options['new_resource_id']: try: UUID(options['new_resource_id']) new_res_id = options['new_resource_id'] except Exception as e: raise CommandError('new_resource_id {} must be a valid uuid hex string' .format(options['new_resource_id']), e) try: if BaseResource.objects.get(short_id=new_res_id): raise CommandError('resource with id {} already exists'.format(new_res_id)) except ObjectDoesNotExist: pass else: new_res_id = short_id() storage = res.get_irods_storage() if storage.exists(res.bag_path): try: storage.delete(res.bag_path) print("{} deleted".format(res.bag_path)) except SessionException as ex: print("{} delete failed: {}".format(res.bag_path, ex.stderr)) raise EnvironmentError() try: with transaction.atomic(): print("Deleting existing bag") res.setAVU("bag_modified", True) res.setAVU('metadata_dirty', 'true') print("Updating BaseResource short_id from {} to {}".format(res_id, new_res_id)) res.short_id = new_res_id res.save() print("Updating resource slug") res.set_slug('resource/{}'.format(new_res_id)) print("Updating Resource files short_path") for file in res.files.all(): file_name = file.short_path.split('data/contents/')[1] file.set_short_path(file_name) print("Updating metadata identifiers") for i in res.metadata.identifiers.all(): i.url = i.url.replace(res_id, new_res_id) i.save() print("Updating logical_files metadata") for aggregation in res.logical_files: aggregation.metadata.is_dirty = True aggregation.metadata.save() except IntegrityError: raise EnvironmentError("Error occurred while updating") print("Moving Resource files") storage.moveFile(res_id, new_res_id) print("Creating Bag") create_bag(res) print(("Resource id successfully update from {} to {}".format(res_id, new_res_id)))
def delete_resource_file(pk, filename_or_id, user): """ Deletes an individual file from a HydroShare resource. If the file does not exist, the Exceptions.NotFound exception is raised. REST URL: DELETE /resource/{pid}/files/{filename} Parameters: pid - The unique HydroShare identifier for the resource from which the file will be deleted filename - Name of the file to be deleted from the resource Returns: The pid of the resource from which the file was deleted Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist or the file identified by file does not exist Exception.ServiceFailure - The service is unable to process the request Note: For mutable resources (resources that have not been formally published), this method modifies the resource by deleting the file. For immutable resources (formally published resources), this method creates a new resource that is a new version of the formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) res_cls = resource.__class__ try: file_id = int(filename_or_id) filter_condition = lambda fl: fl.id == file_id except ValueError: filter_condition = lambda fl: os.path.basename(fl.resource_file.name) == filename_or_id for f in ResourceFile.objects.filter(object_id=resource.id): if filter_condition(f): # send signal signals.pre_delete_file_from_resource.send(sender=res_cls, file=f, resource=resource) file_name = f.resource_file.name f.resource_file.delete() f.delete() delete_file_mime_type = utils.get_file_mime_type(file_name) delete_file_extension = os.path.splitext(file_name)[1] # if there is no other resource file with the same extension as the # file just deleted then delete the matching format metadata element for the resource resource_file_extensions = [os.path.splitext(f.resource_file.name)[1] for f in resource.files.all()] if delete_file_extension not in resource_file_extensions: format_element = resource.metadata.formats.filter(value=delete_file_mime_type).first() if format_element: resource.metadata.delete_element(format_element.term, format_element.id) break else: raise ObjectDoesNotExist(filename_or_id) if resource.public: if not resource.can_be_public: resource.public = False resource.save() # generate bag utils.resource_modified(resource, user) return filename_or_id
def add_resource_files(pk, *files, **kwargs): """ Called by clients to update a resource in HydroShare by adding one or more files. REST URL: PUT /resource/{pid}/files/{file} Parameters: pk - Unique HydroShare identifier for the resource that is to be updated. files - A list of file-like objects representing files that will be added to the existing resource identified by pid Returns: A list of ResourceFile objects Return Type: list Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the file is invalid Exception.ServiceFailure - The service is unable to process the request Notes: For mutable resources (resources not formally published), the update adds the file that is passed to this method to the resource. For immutable resources (formally published resources), this method creates a new resource that is a new version of the formally published resource. HydroShare will record the update by storing the SystemMetadata.obsoletes and SystemMetadata.obsoletedBy fields for the respective resources in their system metadata HydroShare MUST check or set the values of SystemMetadata.obsoletes and SystemMetadata.obsoletedBy so that they accurately represent the relationship between the new and old objects. HydroShare MUST also set SystemMetadata.dateSysMetadataModified. The modified system metadata entries must then be available in HydroShare.listObjects() to ensure that any cataloging systems pick up the changes when filtering on SystmeMetadata.dateSysMetadataModified. A formally published resource can only be obsoleted by one newer version. Once a resource is obsoleted, no other resources can obsolete it. """ resource = utils.get_resource_by_shortkey(pk) fed_res_file_names = kwargs.pop('fed_res_file_names', '') ret = [] fed_zone_home_path = kwargs.pop('fed_zone_home_path', '') # for adding files to existing resources, the default action is copy fed_copy_or_move = kwargs.pop('fed_copy_or_move', 'copy') folder = kwargs.pop('folder', None) for f in files: if fed_zone_home_path: # user has selected files from a federated iRODS zone, so files uploaded from local disk # need to be stored to the federated iRODS zone rather than HydroShare zone as well # TODO: why do we allow the federation prefix to vary in this statement? ret.append(utils.add_file_to_resource(resource, f, folder=folder, fed_res_file_name_or_path=fed_zone_home_path)) elif resource.resource_federation_path: # file needs to be added to a resource in a federated zone # TODO: why do we allow the federation prefix to vary in this statement? ret.append(utils.add_file_to_resource( resource, f, folder=folder, fed_res_file_name_or_path=resource.resource_federation_path)) else: ret.append(utils.add_file_to_resource(resource, f, folder=folder)) if fed_res_file_names: if isinstance(fed_res_file_names, basestring): ifnames = string.split(fed_res_file_names, ',') elif isinstance(fed_res_file_names, list): ifnames = fed_res_file_names else: return ret for ifname in ifnames: ret.append(utils.add_file_to_resource(resource, None, folder=folder, fed_res_file_name_or_path=ifname, fed_copy_or_move=fed_copy_or_move)) if not ret: # no file has been added, make sure data/contents directory exists if no file is added utils.create_empty_contents_directory(resource) return ret
def create_bag_by_irods(resource_id): """Create a resource bag on iRODS side by running the bagit rule and ibun zip. This function runs as a celery task, invoked asynchronously so that it does not block the main web thread when it creates bags for very large files which will take some time. :param resource_id: the resource uuid that is used to look for the resource to create the bag for. :return: True if bag creation operation succeeds; False if there is an exception raised or resource does not exist. """ res = utils.get_resource_by_shortkey(resource_id) istorage = res.get_irods_storage() bag_path = res.bag_path metadata_dirty = istorage.getAVU(res.root_path, 'metadata_dirty') # if metadata has been changed, then regenerate metadata xml files if metadata_dirty is None or metadata_dirty.lower() == "true": try: create_bag_files(res) except Exception as ex: logger.error('Failed to create bag files. Error:{}'.format( ex.message)) # release the lock before returning bag creation failure return False irods_bagit_input_path = res.get_irods_path(resource_id, prepend_short_id=False) # check to see if bagit readme.txt file exists or not bagit_readme_file = res.get_irods_path('readme.txt') is_bagit_readme_exist = istorage.exists(bagit_readme_file) if irods_bagit_input_path.startswith(resource_id): # resource is in data zone, need to append the full path for iRODS bagit rule execution irods_dest_prefix = "/" + settings.IRODS_ZONE + "/home/" + settings.IRODS_USERNAME irods_bagit_input_path = os.path.join(irods_dest_prefix, resource_id) bagit_input_resource = "*DESTRESC='{def_res}'".format( def_res=settings.IRODS_DEFAULT_RESOURCE) else: # this will need to be changed with the default resource in whatever federated zone the # resource is stored in when we have such use cases to support bagit_input_resource = "*DESTRESC='{def_res}'".format( def_res=settings.HS_IRODS_USER_ZONE_DEF_RES) bagit_input_path = "*BAGITDATA='{path}'".format( path=irods_bagit_input_path) bagit_files = [ res.get_irods_path('bagit.txt'), res.get_irods_path('manifest-md5.txt'), res.get_irods_path('tagmanifest-md5.txt'), bag_path ] # only proceed when the resource is not deleted potentially by another request # when being downloaded is_exist = istorage.exists(irods_bagit_input_path) if is_exist: # if bagit readme.txt does not exist, add it. if not is_bagit_readme_exist: from_file_name = getattr(settings, 'HS_BAGIT_README_FILE_WITH_PATH', 'docs/bagit/readme.txt') istorage.saveFile(from_file_name, bagit_readme_file, True) # call iRODS bagit rule here bagit_rule_file = getattr(settings, 'IRODS_BAGIT_RULE', 'hydroshare/irods/ruleGenerateBagIt_HS.r') try: # call iRODS run and ibun command to create and zip the bag, ignore SessionException # for now as a workaround which could be raised from potential race conditions when # multiple ibun commands try to create the same zip file or the very same resource # gets deleted by another request when being downloaded istorage.runBagitRule(bagit_rule_file, bagit_input_path, bagit_input_resource) istorage.zipup(irods_bagit_input_path, bag_path) if res.raccess.published: # compute checksum to meet DataONE distribution requirement chksum = istorage.checksum(bag_path) res.bag_checksum = chksum istorage.setAVU(irods_bagit_input_path, 'bag_modified', "false") return True except SessionException as ex: # if an exception occurs, delete incomplete files potentially being generated by # iRODS bagit rule and zipping operations for fname in bagit_files: if istorage.exists(fname): istorage.delete(fname) logger.error(ex.stderr) return False else: logger.error('Resource does not exist.') return False
def add_resource_files(pk, *files, **kwargs): """ Called by clients to update a resource in CommonsShare by adding one or more files. REST URL: PUT /resource/{pid}/files/{file} Parameters: pk - Unique CommonsShare identifier for the resource that is to be updated. files - A list of file-like objects representing files that will be added to the existing resource identified by pid Returns: A list of ResourceFile objects added to the resource Return Type: list Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.InvalidContent - The content of the file is invalid Exception.ServiceFailure - The service is unable to process the request Notes: This does **not** handle mutability; changes to immutable resources should be denied elsewhere. """ resource = utils.get_resource_by_shortkey(pk) ret = [] source_names = kwargs.pop('source_names', []) source_sizes = kwargs.pop('source_sizes', []) is_file_reference = kwargs.pop('is_file_reference', False) if __debug__: assert (isinstance(source_names, list)) move = kwargs.pop('move', False) folder = kwargs.pop('folder', None) if __debug__: # assure that there are no spurious kwargs left. for k in kwargs: print("kwargs[{}]".format(k)) assert len(kwargs) == 0 for f in files: ret.append(utils.add_file_to_resource(resource, f, folder=folder)) if len(source_names) > 0: if len(source_names) != len(source_sizes): # if length is not equal, there is an issue with source_sizes input parameter, so it will not be # used by setting it to be empty source_sizes = [] idx = 0 for ifname in source_names: s_size = source_sizes[idx] if source_sizes else 0 idx += 1 ret.append( utils.add_file_to_resource( resource, None, folder=folder, source_name=ifname, source_size=s_size, move=move, is_file_reference=is_file_reference)) # make sure data/contents directory exists if not exist already utils.create_empty_contents_directory(resource) return ret
def update_collection(request, shortkey, *args, **kwargs): """ Update collection. The POST request should contain a list of resource ids and a 'update_type' parameter with value of 'set', 'add' or 'remove', which are three differnt mode to update the collection.If no 'update_type' parameter is provided, the 'set' will be used by default. To add a resource to collection, user should have certain premission on both collection and resources being added. For collection: user should have at least Edit permission For resources being added, one the following criteria should be met: 1) user has at lest View permission and the resource is Shareable 2) user is resource owner :param shortkey: id of the collection resource to which resources are to be added. """ status = "success" msg = "" metadata_status = "Insufficient to make public" new_coverage_list = [] hasPart = "hasPart" try: with transaction.atomic(): collection_res_obj, is_authorized, user \ = authorize(request, shortkey, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) if collection_res_obj.resource_type.lower( ) != "collectionresource": raise Exception( "Resource {0} is not a collection resource.".format( shortkey)) # get 'resource_id_list' list from POST updated_contained_res_id_list = request.POST.getlist( "resource_id_list") # get optional 'update_type' parameter: # 1) "set" (default): set collection content to the list, # following code will find out which resources are newly added, removed and unchanged # 2) 'add': add resources in the list to collection # adding a resource that is already in the collection will raise error # 3) 'remove': remove resources in the list from collection # removing a resource that is not in collection will raise error update_type = request.POST.get("update_type", 'set').lower() if update_type not in ["set", "add", "remove"]: raise Exception("Invalid value of 'update_type' parameter") if len(updated_contained_res_id_list) > len( set(updated_contained_res_id_list)): raise Exception( "Duplicate resources exist in list 'resource_id_list'") for updated_contained_res_id in updated_contained_res_id_list: # avoid adding collection itself if updated_contained_res_id == shortkey: raise Exception("Can not contain collection itself.") # current contained res res_id_list_current_collection = \ [res.short_id for res in collection_res_obj.resources.all()] # res to remove res_id_list_remove = [] if update_type == "remove": res_id_list_remove = updated_contained_res_id_list for res_id_remove in res_id_list_remove: if res_id_remove not in res_id_list_current_collection: raise Exception('Cannot remove resource {0} as it ' 'is not currently contained ' 'in collection'.format(res_id_remove)) elif update_type == "set": for res_id_remove in res_id_list_current_collection: if res_id_remove not in updated_contained_res_id_list: res_id_list_remove.append(res_id_remove) for res_id_remove in res_id_list_remove: # user with Edit permission over this collection can remove any resource from it res_obj_remove = get_resource_by_shortkey(res_id_remove) collection_res_obj.resources.remove(res_obj_remove) # change "Relation" metadata in collection value = RES_LANDING_PAGE_URL_TEMPLATE.format(res_id_remove) add_or_remove_relation_metadata( add=False, target_res_obj=collection_res_obj, relation_type=hasPart, relation_value=value, set_res_modified=False) # res to add res_id_list_add = [] if update_type == "add": res_id_list_add = updated_contained_res_id_list for res_id_add in res_id_list_add: if res_id_add in res_id_list_current_collection: raise Exception( 'Cannot add resource {0} as it ' 'is already contained in collection'.format( res_id_add)) elif update_type == "set": for res_id_add in updated_contained_res_id_list: if res_id_add not in res_id_list_current_collection: res_id_list_add.append(res_id_add) for res_id_add in res_id_list_add: # check authorization for all new resources being added to the collection # the requesting user should at least have metadata view permission for each of # the new resources to be added to the collection res_to_add, _, _ \ = authorize(request, res_id_add, needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA) # the resources being added should be 'Shareable' # or is owned by current user is_shareable = res_to_add.raccess.shareable is_owner = res_to_add.raccess.owners.filter( pk=user.pk).exists() if not is_shareable and not is_owner: raise Exception( 'Only resource owner can add a non-shareable ' 'resource to a collection ') # add this new res to collection res_obj_add = get_resource_by_shortkey(res_id_add) collection_res_obj.resources.add(res_obj_add) # change "Relation" metadata in collection value = RES_LANDING_PAGE_URL_TEMPLATE.format(res_id_add) add_or_remove_relation_metadata( add=True, target_res_obj=collection_res_obj, relation_type=hasPart, relation_value=value, set_res_modified=False) if collection_res_obj.can_be_public_or_discoverable: metadata_status = "Sufficient to make public" new_coverage_list = _update_collection_coverages( collection_res_obj) update_collection_list_csv(collection_res_obj) resource_modified(collection_res_obj, user, overwrite_bag=False) except Exception as ex: err_msg = "update_collection: {0} ; username: {1}; collection_id: {2} ." logger.error( err_msg.format( str(ex), request.user.username if request.user.is_authenticated() else "anonymous", shortkey)) status = "error" msg = str(ex) finally: ajax_response_data = \ {'status': status, 'msg': msg, 'metadata_status': metadata_status, 'new_coverage_list': new_coverage_list} return JsonResponse(ajax_response_data)
def test_resource_create_with_core_metadata(self): """ The followings are the core metadata elements that can be passed as part of the 'metadata' parameter when creating a resource: coverage creator contributor source, relation, identifier, fundingagency """ rtype = 'GenericResource' title = 'My Test resource' metadata = [] metadata.append({'coverage': {'type': 'period', 'value': {'start': '01/01/2000', 'end': '12/12/2010'}}}) statement = 'This resource is shared under the Creative Commons Attribution CC BY.' url = 'http://creativecommons.org/licenses/by/4.0/' metadata.append({'rights': {'statement': statement, 'url': url}}) metadata.append({'language': {'code': 'fre'}}) # contributor con_name = 'Mike Sundar' con_org = "USU" con_email = '*****@*****.**' con_address = "11 River Drive, Logan UT-84321, USA" con_phone = '435-567-0989' con_homepage = 'http://usu.edu/homepage/001' con_identifiers = {'ORCID': 'https://orcid.org/mike_s', 'ResearchGateID': 'https://www.researchgate.net/mike_s'} metadata.append({'contributor': {'name': con_name, 'organization': con_org, 'email': con_email, 'address': con_address, 'phone': con_phone, 'homepage': con_homepage, 'identifiers': con_identifiers}}) # creator cr_name = 'John Smith' cr_org = "USU" cr_email = '*****@*****.**' cr_address = "101 Clarson Ave, Provo UT-84321, USA" cr_phone = '801-567=9090' cr_homepage = 'http://byu.edu/homepage/002' cr_identifiers = {'ORCID': 'https://orcid.org/john_smith', 'ResearchGateID': 'https://www.researchgate.net/john_smith'} metadata.append({'creator': {'name': cr_name, 'organization': cr_org, 'email': cr_email, 'address': cr_address, 'phone': cr_phone, 'homepage': cr_homepage, 'identifiers': cr_identifiers}}) # relation metadata.append({'relation': {'type': 'isPartOf', 'value': 'http://hydroshare.org/resource/001'}}) # source metadata.append({'source': {'derived_from': 'http://hydroshare.org/resource/0001'}}) # identifier metadata.append({'identifier': {'name': 'someIdentifier', 'url': 'http://some.org/001'}}) # fundingagency agency_name = 'NSF' award_title = "Cyber Infrastructure" award_number = "NSF-101-20-6789" agency_url = "http://www.nsf.gov" metadata.append({'fundingagency': {'agency_name': agency_name, 'award_title': award_title, 'award_number': award_number, 'agency_url': agency_url}}) params = {'resource_type': rtype, 'title': title, 'metadata': json.dumps(metadata), 'file': ('cea.tif', open('hs_core/tests/data/cea.tif', 'rb'), 'image/tiff')} rest_url = '/hsapi/resource/' response = self.client.post(rest_url, params) self.assertEqual(response.status_code, status.HTTP_201_CREATED) content = json.loads(response.content.decode()) res_id = content['resource_id'] resource = get_resource_by_shortkey(res_id) self.assertEqual(resource.metadata.coverages.all().count(), 1) self.assertEqual(resource.metadata.coverages.filter(type='period').count(), 1) coverage = resource.metadata.coverages.all().first() self.assertEqual(parser.parse(coverage.value['start']).date(), parser.parse('01/01/2000').date()) self.assertEqual(parser.parse(coverage.value['end']).date(), parser.parse('12/12/2010').date()) self.assertEqual(resource.metadata.rights.statement, statement) self.assertEqual(resource.metadata.rights.url, url) self.assertEqual(resource.metadata.language.code, 'fre') # there should be 1 contributor self.assertEqual(resource.metadata.contributors.all().count(), 1) contributor = resource.metadata.contributors.all().first() self.assertEqual(contributor.name, con_name) self.assertEqual(contributor.organization, con_org) self.assertEqual(contributor.email, con_email) self.assertEqual(contributor.address, con_address) self.assertEqual(contributor.phone, con_phone) self.assertEqual(contributor.homepage, con_homepage) # there should be 1 creator (based on the metadata we provided for one creator) # system automatically adds the user who creates the resource as the creator # only in the case where not metadata for creator is provided. self.assertEqual(resource.metadata.creators.all().count(), 1) creator = resource.metadata.creators.filter(name=cr_name).first() self.assertEqual(creator.name, cr_name) self.assertEqual(creator.organization, cr_org) self.assertEqual(creator.email, cr_email) self.assertEqual(creator.address, cr_address) self.assertEqual(creator.phone, cr_phone) self.assertEqual(creator.homepage, cr_homepage) # there should be 1 relation element self.assertEqual(resource.metadata.relations.all().count(), 1) relation = resource.metadata.relations.all().first() self.assertEqual(relation.type, 'isPartOf') self.assertEqual(relation.value, 'http://hydroshare.org/resource/001') # there should be 1 source element self.assertEqual(resource.metadata.sources.all().count(), 1) source = resource.metadata.sources.all().first() self.assertEqual(source.derived_from, 'http://hydroshare.org/resource/0001') # there should be 2 identifiers self.assertEqual(resource.metadata.identifiers.all().count(), 2) identifier = resource.metadata.identifiers.filter(name='someIdentifier').first() self.assertEqual(identifier.url, 'http://some.org/001') # there should be 1 fundingagency self.assertEqual(resource.metadata.funding_agencies.all().count(), 1) agency = resource.metadata.funding_agencies.all().first() self.assertEqual(agency.agency_name, agency_name) self.assertEqual(agency.award_title, award_title) self.assertEqual(agency.award_number, award_number) self.assertEqual(agency.agency_url, agency_url) self.resources_to_delete.append(res_id)
def resource_from_uuid(id): return get_resource_by_shortkey(id)
def publish_resource(user, pk): """ Formally publishes a resource in HydroShare. Triggers the creation of a DOI for the resource, and triggers the exposure of the resource to the HydroShare DataONE Member Node. The user must be an owner of a resource or an administrator to perform this action. Parameters: user - requesting user to publish the resource who must be one of the owners of the resource pk - Unique HydroShare identifier for the resource to be formally published. Returns: The id of the resource that was published Return Type: string Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist Exception.ServiceFailure - The service is unable to process the request and other general exceptions Note: This is different than just giving public access to a resource via access control rule """ resource = utils.get_resource_by_shortkey(pk) # TODO: whether a resource can be published is not considered in can_be_published # TODO: can_be_published is currently an alias for can_be_public_or_discoverable if not resource.can_be_published: raise ValidationError("This resource cannot be published since it does not have required " "metadata or content files, or this resource contains referenced " "content, or this resource type is not allowed for publication.") # append pending to the doi field to indicate DOI is not activated yet. Upon successful # activation, "pending" will be removed from DOI field resource.doi = get_resource_doi(pk, 'pending') resource.save() response = deposit_res_metadata_with_crossref(resource) if not response.status_code == status.HTTP_200_OK: # resource metadata deposition failed from CrossRef - set failure flag to be retried in a # crontab celery task resource.doi = get_resource_doi(pk, 'failure') resource.save() resource.set_public(True) # also sets discoverable to True resource.raccess.immutable = True resource.raccess.shareable = False resource.raccess.published = True resource.raccess.save() # change "Publisher" element of science metadata to CUAHSI md_args = {'name': 'Consortium of Universities for the Advancement of Hydrologic Science, ' 'Inc. (CUAHSI)', 'url': 'https://www.cuahsi.org'} resource.metadata.create_element('Publisher', **md_args) # create published date resource.metadata.create_element('date', type='published', start_date=resource.updated) # add doi to "Identifier" element of science metadata md_args = {'name': 'doi', 'url': get_activated_doi(resource.doi)} resource.metadata.create_element('Identifier', **md_args) utils.resource_modified(resource, user, overwrite_bag=False) return pk
def resource_from_uuid(id): if not id: return None return get_resource_by_shortkey(id)
def delete_resource_file(pk, filename_or_id, user, delete_logical_file=True): """ Deletes an individual file from a HydroShare resource. If the file does not exist, the Exceptions.NotFound exception is raised. REST URL: DELETE /resource/{pid}/files/{filename} Parameters: :param pk: The unique HydroShare identifier for the resource from which the file will be deleted :param filename_or_id: Name of the file or id of the file to be deleted from the resource :param user: requesting user :param delete_logical_file: If True then if the ResourceFile object to be deleted is part of a LogicalFile object then the LogicalFile object will be deleted which deletes all associated ResourceFile objects and file type metadata objects. :returns: The name or id of the file which was deleted Return Type: string or integer Raises: Exceptions.NotAuthorized - The user is not authorized Exceptions.NotFound - The resource identified by pid does not exist or the file identified by file does not exist Exception.ServiceFailure - The service is unable to process the request Note: This does not handle immutability as previously intended. """ resource = utils.get_resource_by_shortkey(pk) res_cls = resource.__class__ for f in ResourceFile.objects.filter(object_id=resource.id): if filter_condition(filename_or_id, f): if delete_logical_file: if f.has_logical_file and not f.logical_file.is_fileset: # delete logical file if any resource file that belongs to logical file # gets deleted for any logical file other than fileset logical file # logical_delete() calls this function (delete_resource_file()) # to delete each of its contained ResourceFile objects f.logical_file.logical_delete(user) return filename_or_id signals.pre_delete_file_from_resource.send(sender=res_cls, file=f, resource=resource, user=user) file_name = delete_resource_file_only(resource, f) # This presumes that the file is no longer in django delete_format_metadata_after_delete_file(resource, file_name) signals.post_delete_file_from_resource.send(sender=res_cls, resource=resource) # set to private if necessary -- AFTER post_delete_file handling resource.update_public_and_discoverable() # set to False if necessary # generate bag utils.resource_modified(resource, user, overwrite_bag=False) return filename_or_id # if execution gets here, file was not found raise ObjectDoesNotExist(str.format("resource {}, file {} not found", resource.short_id, filename_or_id))
def handle(self, *args, **options): ind = BaseResourceIndex() if len(options['resource_ids']) > 0: # an array of resource short_id to check. for rid in options['resource_ids']: print("updating resource {}".format(rid)) try: r = BaseResource.objects.get(short_id=rid) if r.show_in_discover: ind.update_object(r) except BaseResource.DoesNotExist: print("resource {} does not exist in Django".format(rid)) except Exception as e: print("resource {} generated exception {}".format(rid, str(e))) else: sqs = SearchQuerySet().all() print("SOLR count = {}".format(sqs.count())) dqs = BaseResource.objects.filter(Q(raccess__discoverable=True) | Q(raccess__public=True)) print("Django count = {}".format(dqs.count())) # what is in Django that isn't in SOLR print("checking for resources in Django that aren't in SOLR...") found_in_solr = set() for r in list(sqs): found_in_solr.add(r.short_id) # enable fast matching django_indexed = 0 django_replaced = 0 django_refreshed = 0 for r in dqs: try: resource = get_resource_by_shortkey(r.short_id, or_404=False) if resource.show_in_discover: django_indexed += 1 else: django_replaced += 1 except BaseResource.DoesNotExist: # race condition in processing while in production print("resource {} no longer found in Django.".format(r.short_id)) continue except Exception as e: print("resource {} generated exception {}".format(r.short_id, str(e))) if resource.show_in_discover and r.short_id not in found_in_solr: print("{} {} NOT FOUND in SOLR: adding to index".format( r.short_id, resource.discovery_content_type)) try: ind.update_object(r) django_refreshed += 1 except Exception as e: print("resource {} generated exception {}".format(r.short_id, str(e))) # # This always returns True whether or not SOLR needs updating # # This is likely a Haystack bug. # elif ind.should_update(r): # update everything to be safe. elif options['force']: if r.show_in_discover: print("{} {}: refreshing index (forced)".format( r.short_id, resource.discovery_content_type)) try: ind.update_object(r) django_refreshed += 1 except Exception as e: print("resource {} generated exception {}".format(r.short_id, str(e))) # what is in SOLR that isn't in Django: print("checking for resources in SOLR that aren't in Django...") sqs = SearchQuerySet().all() # refresh for changes from above solr_indexed = 0 solr_replaced = 0 solr_deleted = 0 for r in sqs: try: resource = get_resource_by_shortkey(r.short_id, or_404=False) if resource.show_in_discover: solr_indexed += 1 else: solr_replaced += 1 except BaseResource.DoesNotExist: print("SOLR resource {} NOT FOUND in Django; cannot remove from SOLR" .format(r.short_id)) solr_deleted += 1 continue print("Django contains {} discoverable resources and {} replaced resources" .format(django_indexed, django_replaced)) print("{} resources in Django refreshed in SOLR".format(django_refreshed)) print("SOLR contains {} discoverable resources and {} replaced resources" .format(solr_indexed, solr_replaced)) print("{} resources not in Django removed from SOLR".format(solr_deleted))