def copy_resource(ori_res, new_res, user=None): """ Populate metadata and contents from ori_res object to new_res object to make new_res object as a copy of the ori_res object Args: ori_res: the original resource that is to be copied. new_res: the new_res to be populated with metadata and content from the original resource as a copy of the original resource. user: requesting user for the copy action. It is optional, if being passed in, quota is counted toward the user; otherwise, quota is not counted toward that user Returns: the new resource copied from the original resource """ # add files directly via irods backend file operation utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id) utils.copy_and_create_metadata(ori_res, new_res) hs_identifier = ori_res.metadata.identifiers.all().filter(name="hydroShareIdentifier")[0] if hs_identifier: new_res.metadata.create_element('source', derived_from=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource hs_bagit.create_bag(new_res) # need to update quota usage for new_res quota holder if user: update_quota_usage(user=user) return new_res
def copy_resource(ori_res, new_res): """ Populate metadata and contents from ori_res object to new_res object to make new_res object as a copy of the ori_res object Args: ori_res: the original resource that is to be copied. new_res: the new_res to be populated with metadata and content from the original resource as a copy of the original resource. Returns: the new resource copied from the original resource """ # add files directly via irods backend file operation utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id) utils.copy_and_create_metadata(ori_res, new_res) hs_identifier = ori_res.metadata.identifiers.all().filter(name="hydroShareIdentifier")[0] if hs_identifier: new_res.metadata.create_element('source', derived_from=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource hs_bagit.create_bag(new_res) return new_res
def copy_resource_task(ori_res_id, new_res_id=None, request_username=None): try: new_res = None if not new_res_id: new_res = create_empty_resource(ori_res_id, request_username, action='copy') new_res_id = new_res.short_id utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id) ori_res = utils.get_resource_by_shortkey(ori_res_id) if not new_res: new_res = utils.get_resource_by_shortkey(new_res_id) utils.copy_and_create_metadata(ori_res, new_res) hs_identifier = ori_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] if hs_identifier: new_res.metadata.create_element('source', derived_from=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource create_bag(new_res) return new_res.get_absolute_url() except Exception as ex: if new_res: new_res.delete() raise utils.ResourceCopyException(str(ex))
def create_new_version_resource(ori_res, new_res, user): """ Populate metadata and contents from ori_res object to new_res object to make new_res object as a new version of the ori_res object Args: ori_res: the original resource that is to be versioned. new_res: the new_res to be populated with metadata and content from the original resource to make it a new version user: the requesting user Returns: the new versioned resource for the original resource and thus obsolete the original resource """ # newly created new resource version is private initially # add files directly via irods backend file operation utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id) # copy metadata from source resource to target new-versioned resource except three elements utils.copy_and_create_metadata(ori_res, new_res) # add or update Relation element to link source and target resources hs_identifier = new_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] ori_res.metadata.create_element('relation', type='isReplacedBy', value=hs_identifier.url) if new_res.metadata.relations.all().filter(type='isVersionOf').exists(): # the original resource is already a versioned resource, and its isVersionOf relation # element is copied over to this new version resource, needs to delete this element so # it can be created to link to its original resource correctly eid = new_res.metadata.relations.all().filter( type='isVersionOf').first().id new_res.metadata.delete_element('relation', eid) hs_identifier = ori_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] new_res.metadata.create_element('relation', type='isVersionOf', value=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new version collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource hs_bagit.create_bag(new_res) # since an isReplaceBy relation element is added to original resource, needs to call # resource_modified() for original resource utils.resource_modified(ori_res, user, overwrite_bag=False) # if everything goes well up to this point, set original resource to be immutable so that # obsoleted resources cannot be modified from REST API ori_res.raccess.immutable = True ori_res.raccess.save() return new_res
def test_create_bag(self): # the resource should have only one bags object self.assertEquals(self.test_res.bags.count(), 1) old_bag = self.test_res.bags.all().first() # this is the api call we are testing new_bag = hs_bagit.create_bag(self.test_res) # resource should have one new bags object self.assertEquals(self.test_res.bags.count(), 1) self.assertEquals(new_bag, self.test_res.bags.all().first()) self.assertNotEquals(old_bag, new_bag)
def create_resource( resource_type, owner, title, edit_users=None, view_users=None, edit_groups=None, view_groups=None, keywords=(), metadata=None, extra_metadata=None, files=(), create_metadata=True, create_bag=True, unpack_file=False, full_paths={}, auto_aggregate=True, **kwargs): """ Called by a client to add a new resource to HydroShare. The caller must have authorization to write content to HydroShare. The pid for the resource is assigned by HydroShare upon inserting the resource. The create method returns the newly-assigned pid. REST URL: POST /resource Parameters: Returns: The newly created resource Return Type: BaseResource resource object Note: The calling user will automatically be set as the owner of the created resource. Implementation notes: 1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing. 2. return type is an instance of hs_core.models.BaseResource class. This is for efficiency in the native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated lookups in the database when they are unnecessary. 3. resource_type is a string: see parameter list :param resource_type: string. the type of the resource such as GenericResource :param owner: email address, username, or User instance. The owner of the resource :param title: string. the title of the resource :param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions :param view_users: list of email addresses, usernames, or User instances who will be given view permissions :param edit_groups: list of group names or Group instances who will be given edit permissions :param view_groups: list of group names or Group instances who will be given view permissions :param keywords: string list. list of keywords to add to the resource :param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}. :param extra_metadata: one dict containing keys and corresponding values { 'Outlet Point Latitude': '40', 'Outlet Point Longitude': '-110'}. :param files: list of Django File or UploadedFile objects to be attached to the resource :param create_bag: whether to create a bag for the newly created resource or not. By default, the bag is created. :param unpack_file: boolean. If files contains a single zip file, and unpack_file is True, the unpacked contents of the zip file will be added to the resource instead of the zip file. :param full_paths: Optional. A map of paths keyed by the correlating resource file. When this parameter is provided, a file will be placed at the path specified in the map. :param auto_aggregate: boolean, defaults to True. Find and create aggregations during resource creation. :param kwargs: extra arguments to fill in required values in AbstractResource subclasses :return: a new resource which is an instance of BaseResource with specificed resource_type. """ with transaction.atomic(): cls = check_resource_type(resource_type) owner = utils.user_from_id(owner) # get the metadata class specific to resource type to set resource # content_object (metadata) attribute metadata_class = cls.get_metadata_class() metadata_obj = metadata_class() metadata_obj.save() # create the resource resource = cls.objects.create( resource_type=resource_type, user=owner, creator=owner, title=title, last_changed_by=owner, in_menus=[], content_object=metadata_obj, **kwargs ) resource.resource_type = resource_type # by default make resource private resource.set_slug('resource{0}{1}'.format('/', resource.short_id)) resource.save() if not metadata: metadata = [] if extra_metadata is not None: resource.extra_metadata = extra_metadata resource.save() # by default resource is private resource_access = ResourceAccess(resource=resource) resource_access.save() # use the built-in share routine to set initial provenance. UserResourcePrivilege.share(resource=resource, grantor=owner, user=owner, privilege=PrivilegeCodes.OWNER) resource_labels = ResourceLabels(resource=resource) resource_labels.save() if edit_users: for user in edit_users: user = utils.user_from_id(user) owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.CHANGE) if view_users: for user in view_users: user = utils.user_from_id(user) owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.VIEW) if edit_groups: for group in edit_groups: group = utils.group_from_id(group) owner.uaccess.share_resource_with_group(resource, group, PrivilegeCodes.CHANGE) if view_groups: for group in view_groups: group = utils.group_from_id(group) owner.uaccess.share_resource_with_group(resource, group, PrivilegeCodes.VIEW) # set quota of this resource to this creator # quota holder has to be set before the files are added in order for real time iRODS # quota micro-services to work resource.set_quota_holder(owner, owner) if create_metadata: # prepare default metadata utils.prepare_resource_default_metadata(resource=resource, metadata=metadata, res_title=title) for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] resource.metadata.create_element(k, **v) for keyword in keywords: resource.metadata.create_element('subject', value=keyword) resource.title = resource.metadata.title.value resource.save() if len(files) == 1 and unpack_file and zipfile.is_zipfile(files[0]): # Add contents of zipfile as resource files asynchronously # Note: this is done asynchronously as unzipping may take # a long time (~15 seconds to many minutes). add_zip_file_contents_to_resource_async(resource, files[0]) else: # Add resource file(s) now # Note: this is done synchronously as it should only take a # few seconds. We may want to add the option to do this # asynchronously if the file size is large and would take # more than ~15 seconds to complete. add_resource_files(resource.short_id, *files, full_paths=full_paths, auto_aggregate=auto_aggregate) if create_bag: hs_bagit.create_bag(resource) # set the resource to private resource.setAVU('isPublic', resource.raccess.public) # set the resource type (which is immutable) resource.setAVU("resourceType", resource._meta.object_name) return resource
def generate_files(request, shortkey, *args, **kwargs): res = hydroshare.get_resource_by_shortkey(shortkey) ts, csv_link, csv_size, xml_link, xml_size = {}, '', '', '', '' try: if res.reference_type == 'rest': ts = ts_utils.time_series_from_service(res.url, res.reference_type) else: ts = ts_utils.time_series_from_service(res.url, res.reference_type, site_name_or_code=res.data_site_code, variable_code=res.variable_code) vals = ts['values'] version = ts['wml_version'] d = datetime.date.today() date = '{0}_{1}_{2}'.format(d.month, d.day, d.year) file_base = '{0}-{1}'.format(res.title.replace(" ", ""), date) csv_name = '{0}.{1}'.format(file_base, 'csv') if version == '1': xml_end = 'wml_1' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) elif version == '2.0': xml_end = 'wml_2_0' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) for_csv = [] for k, v in vals.items(): t = (k, v) for_csv.append(t) ResourceFile.objects.filter(object_id=res.pk).delete() with open(csv_name, 'wb') as csv_file: w = csv.writer(csv_file) w.writerow([res.title]) var = '{0}({1})'.format(ts['variable_name'], ts['units']) w.writerow(['time', var]) for r in for_csv: w.writerow(r) with open(xml_name, 'wb') as xml_file: xml_file.write(ts['time_series']) csv_file = open(csv_name, 'r') xml_file = open(xml_name, 'r') files = [csv_file, xml_file] hydroshare.add_resource_files(res.short_id, csv_file, xml_file) create_bag(res) os.remove(csv_name) os.remove(xml_name) files = ResourceFile.objects.filter(object_id=res.pk) for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if xml_end in str(f.resource_file): xml_link = f.resource_file.url xml_size = f.resource_file.size status_code = 200 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_name': csv_name, 'xml_name': xml_name, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # successfully generated new files except Exception: # most likely because the server is unreachable files = ResourceFile.objects.filter(object_id=res.pk) xml_file = None for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if str(f.resource_file).endswith('.xml'): xml_link = f.resource_file.url xml_size = f.resource_file.size xml_file = f.resource_file if xml_file is None: status_code = 404 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, did not find old ones xml_doc = open(str(xml_file), 'r').read() root = etree.XML(xml_doc) os.remove(str(xml_file)) version = ts_utils.get_version(root) if version == '1': ts = ts_utils.parse_1_0_and_1_1(root) status_code = 200 elif version =='2.0': ts = ts_utils.parse_2_0(root) status_code = 200 else: status_code = 503 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, return old ones
def create_resource( resource_type, owner, title, edit_users=None, view_users=None, edit_groups=None, view_groups=None, keywords=None, dublin_metadata=None, metadata=None, files=(), **kwargs): """ Called by a client to add a new resource to HydroShare. The caller must have authorization to write content to HydroShare. The pid for the resource is assigned by HydroShare upon inserting the resource. The create method returns the newly-assigned pid. REST URL: POST /resource Parameters: resource - The data bytes of the resource to be added to HydroShare Returns: The pid assigned to the newly created resource Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized to write to HydroShare Exceptions.InvalidContent - The content of the resource is incomplete Exception.ServiceFailure - The service is unable to process the request Note: The calling user will automatically be set as the owner of the created resource. Implementation notes: 1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing. 2. return type is an instance of a subclass of hs_core.models.AbstractResource. This is for efficiency in the native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated lookups in the database when they are unnecessary. 3. resource_type is a string: see parameter list :param resource_type: string. the classname of the resource type, such as GenericResource :param owner: email address, username, or User instance. The owner of the resource :param title: string. the title of the resource :param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions :param view_users: list of email addresses, usernames, or User instances who will be given view permissions :param edit_groups: list of group names or Group instances who will be given edit permissions :param view_groups: list of group names or Group instances who will be given view permissions :param keywords: string list. list of keywords to add to the resource :param dublin_metadata: list of dicts containing keys { 'term', 'content' } respecting dublin core std. :param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}. :param files: list of Django File or UploadedFile objects to be attached to the resource :param kwargs: extra arguments to fill in required values in AbstractResource subclasses :return: a new resource which is an instance of resource_type. """ for tp in get_resource_types(): if resource_type == tp.__name__: cls = tp break else: raise NotImplementedError("Type {resource_type} does not exist".format(resource_type=resource_type)) # Send pre-create resource signal pre_create_resource.send(sender=cls, dublin_metadata=dublin_metadata, files=files, **kwargs) owner = utils.user_from_id(owner) # create the resource resource = cls.objects.create( user=owner, creator=owner, title=title, last_changed_by=owner, in_menus=[], **kwargs ) for file in files: ResourceFile.objects.create(content_object=resource, resource_file=file) resource.view_users.add(owner) resource.edit_users.add(owner) resource.owners.add(owner) if edit_users: for user in edit_users: user = utils.user_from_id(user) resource.edit_users.add(user) resource.view_users.add(user) if view_users: for user in view_users: user = utils.user_from_id(user) resource.view_users.add(user) if edit_groups: for group in edit_groups: group = utils.group_from_id(group) resource.edit_groups.add(group) resource.view_groups.add(group) if view_groups: for group in view_groups: group = utils.group_from_id(group) resource.view_groups.add(group) if keywords: ks = [Keyword.objects.get_or_create(title=k) for k in keywords] ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse for k in ks: AssignedKeyword.objects.create(content_object=resource, keyword=k) # for creating metadata elements based on the old metadata implementation if dublin_metadata: for d in dublin_metadata: QualifiedDublinCoreElement.objects.create( term=d['term'], content=d['content'], content_object=resource ) # for creating metadata elements based on the new metadata implementation if metadata: for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] resource.metadata.create_element(k, **v) # add the subject elements from the AssignedKeywords (new metadata implementation) for akw in AssignedKeyword.objects.filter(object_pk=resource.id).all(): resource.metadata.create_element('subject', value=akw.keyword.title) hs_bagit.create_bag(resource) # Send post-create resource signal post_create_resource.send(sender=cls, resource=resource) return resource
def create_new_version_resource_task(ori_res_id, username, new_res_id=None): """ Task for creating a new version of a resource Args: ori_res_id: the original resource id that is to be versioned. new_res_id: the new versioned resource id from the original resource. If None, a new resource will be created. username: the requesting user's username Returns: the new versioned resource url as the payload """ try: new_res = None if not new_res_id: new_res = create_empty_resource(ori_res_id, username) new_res_id = new_res.short_id utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id) # copy metadata from source resource to target new-versioned resource except three elements ori_res = utils.get_resource_by_shortkey(ori_res_id) if not new_res: new_res = utils.get_resource_by_shortkey(new_res_id) utils.copy_and_create_metadata(ori_res, new_res) # add or update Relation element to link source and target resources hs_identifier = new_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] ori_res.metadata.create_element('relation', type='isReplacedBy', value=hs_identifier.url) if new_res.metadata.relations.all().filter( type='isVersionOf').exists(): # the original resource is already a versioned resource, and its isVersionOf relation # element is copied over to this new version resource, needs to delete this element so # it can be created to link to its original resource correctly eid = new_res.metadata.relations.all().filter( type='isVersionOf').first().id new_res.metadata.delete_element('relation', eid) hs_identifier = ori_res.metadata.identifiers.all().filter( name="hydroShareIdentifier")[0] new_res.metadata.create_element('relation', type='isVersionOf', value=hs_identifier.url) if ori_res.resource_type.lower() == "collectionresource": # clone contained_res list of original collection and add to new collection # note that new version collection will not contain "deleted resources" new_res.resources = ori_res.resources.all() # create bag for the new resource create_bag(new_res) # since an isReplaceBy relation element is added to original resource, needs to call # resource_modified() for original resource utils.resource_modified(ori_res, by_user=username, overwrite_bag=False) # if everything goes well up to this point, set original resource to be immutable so that # obsoleted resources cannot be modified from REST API ori_res.raccess.immutable = True ori_res.raccess.save() ori_res.save() return new_res.get_absolute_url() except Exception as ex: if new_res: new_res.delete() raise utils.ResourceVersioningException(str(ex)) finally: # release the lock regardless ori_res.locked_time = None ori_res.save()
def handle(self, *args, **options): if not options['resource_id']: raise CommandError('resource_id argument is required') res_id = options['resource_id'] try: res = get_resource_by_shortkey(res_id, or_404=False) except ObjectDoesNotExist: raise CommandError("No Resource found for id {}".format(res_id)) if options['new_resource_id']: try: UUID(options['new_resource_id']) new_res_id = options['new_resource_id'] except Exception as e: raise CommandError('new_resource_id {} must be a valid uuid hex string' .format(options['new_resource_id']), e) try: if BaseResource.objects.get(short_id=new_res_id): raise CommandError('resource with id {} already exists'.format(new_res_id)) except ObjectDoesNotExist: pass else: new_res_id = short_id() storage = res.get_irods_storage() if storage.exists(res.bag_path): try: storage.delete(res.bag_path) print("{} deleted".format(res.bag_path)) except SessionException as ex: print("{} delete failed: {}".format(res.bag_path, ex.stderr)) raise EnvironmentError() try: with transaction.atomic(): print("Deleting existing bag") res.setAVU("bag_modified", True) res.setAVU('metadata_dirty', 'true') print("Updating BaseResource short_id from {} to {}".format(res_id, new_res_id)) res.short_id = new_res_id res.save() print("Updating resource slug") res.set_slug('resource/{}'.format(new_res_id)) print("Updating Resource files short_path") for file in res.files.all(): file_name = file.short_path.split('data/contents/')[1] file.set_short_path(file_name) print("Updating metadata identifiers") for i in res.metadata.identifiers.all(): i.url = i.url.replace(res_id, new_res_id) i.save() print("Updating logical_files metadata") for aggregation in res.logical_files: aggregation.metadata.is_dirty = True aggregation.metadata.save() except IntegrityError: raise EnvironmentError("Error occurred while updating") print("Moving Resource files") storage.moveFile(res_id, new_res_id) print("Creating Bag") create_bag(res) print(("Resource id successfully update from {} to {}".format(res_id, new_res_id)))
def download(request, path, rest_call=False, use_async=True, *args, **kwargs): split_path_strs = path.split('/') is_bag_download = False if split_path_strs[0] == 'bags': res_id = os.path.splitext(split_path_strs[1])[0] is_bag_download = True else: res_id = split_path_strs[0] res, authorized, _ = authorize( request, res_id, needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE, raises_exception=False) if not authorized: response = HttpResponse(status=401) content_msg = "You do not have permission to download this resource!" if rest_call: raise PermissionDenied(content_msg) else: signin_html = '</h1><div class="col-xs-12"><h2 class="page-title">' \ '<a href="/oauth_request/"><span class ="glyphicon glyphicon-log-in"></span>' \ 'Sign In</a></h2>' response.content = '<h1>' + content_msg + signin_html return response if not is_bag_download and "/data" not in path: idx_sep = path.find('/') path = path[idx_sep:] istorage = IrodsStorage() if 'environment' in kwargs: environment = int(kwargs['environment']) environment = m.RodsEnvironment.objects.get(pk=environment) session = Session("/tmp/django_irods", settings.IRODS_ICOMMANDS_PATH, session_id=uuid4()) session.create_environment(environment) session.run('iinit', None, environment.auth) elif getattr(settings, 'IRODS_GLOBAL_SESSION', False): session = GLOBAL_SESSION elif icommands.ACTIVE_SESSION: session = icommands.ACTIVE_SESSION else: raise KeyError('settings must have IRODS_GLOBAL_SESSION set ' 'if there is no environment object') if istorage.exists(res_id) and is_bag_download: bag_modified = istorage.getAVU(res_id, 'bag_modified') # make sure if bag_modified is not set to true, we still recreate the bag if the # bag file does not exist for some reason to resolve the error to download a nonexistent # bag when bag_modified is false due to the flag being out-of-sync with the real bag status if bag_modified is None or bag_modified.lower() == "false": # check whether the bag file exists bag_file_name = res_id + '.zip' bag_full_path = os.path.join('bags', bag_file_name) if not istorage.exists(bag_full_path): bag_modified = 'true' if bag_modified is None or bag_modified.lower() == "true": create_bag(res) resource_cls = check_resource_type(res.resource_type) # send signal for pre download file download_file_name = split_path_strs[-1] pre_download_file.send(sender=resource_cls, resource=res, download_file_name=download_file_name, request=request) # obtain mime_type to set content_type mtype = 'application-x/octet-stream' mime_type = mimetypes.guess_type(path) if mime_type[0] is not None: mtype = mime_type[0] # retrieve file size to set up Content-Length header stdout = session.run("ils", None, "-l", path)[0].split() flen = int(stdout[3]) options = ('-', ) # we're redirecting to stdout. proc = session.run_safe('iget', None, path, *options) response = FileResponse(proc.stdout, content_type=mtype) response['Content-Disposition'] = 'attachment; filename="{name}"'.format( name=path.split('/')[-1]) response['Content-Length'] = flen return response
def create_resource(resource_type, owner, title, edit_users=None, view_users=None, edit_groups=None, view_groups=None, keywords=None, dublin_metadata=None, metadata=None, files=(), **kwargs): """ Called by a client to add a new resource to HydroShare. The caller must have authorization to write content to HydroShare. The pid for the resource is assigned by HydroShare upon inserting the resource. The create method returns the newly-assigned pid. REST URL: POST /resource Parameters: resource - The data bytes of the resource to be added to HydroShare Returns: The pid assigned to the newly created resource Return Type: pid Raises: Exceptions.NotAuthorized - The user is not authorized to write to HydroShare Exceptions.InvalidContent - The content of the resource is incomplete Exception.ServiceFailure - The service is unable to process the request Note: The calling user will automatically be set as the owner of the created resource. Implementation notes: 1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing. 2. return type is an instance of a subclass of hs_core.models.AbstractResource. This is for efficiency in the native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated lookups in the database when they are unnecessary. 3. resource_type is a string: see parameter list :param resource_type: string. the classname of the resource type, such as GenericResource :param owner: email address, username, or User instance. The owner of the resource :param title: string. the title of the resource :param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions :param view_users: list of email addresses, usernames, or User instances who will be given view permissions :param edit_groups: list of group names or Group instances who will be given edit permissions :param view_groups: list of group names or Group instances who will be given view permissions :param keywords: string list. list of keywords to add to the resource :param dublin_metadata: list of dicts containing keys { 'term', 'content' } respecting dublin core std. :param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}. :param files: list of Django File or UploadedFile objects to be attached to the resource :param kwargs: extra arguments to fill in required values in AbstractResource subclasses :return: a new resource which is an instance of resource_type. """ for tp in get_resource_types(): if resource_type == tp.__name__: cls = tp break else: raise NotImplementedError("Type {resource_type} does not exist".format( resource_type=resource_type)) # Send pre-create resource signal pre_create_resource.send(sender=cls, dublin_metadata=dublin_metadata, files=files, **kwargs) owner = utils.user_from_id(owner) # create the resource resource = cls.objects.create(user=owner, creator=owner, title=title, last_changed_by=owner, in_menus=[], **kwargs) for file in files: ResourceFile.objects.create(content_object=resource, resource_file=file) resource.view_users.add(owner) resource.edit_users.add(owner) resource.owners.add(owner) if edit_users: for user in edit_users: user = utils.user_from_id(user) resource.edit_users.add(user) resource.view_users.add(user) if view_users: for user in view_users: user = utils.user_from_id(user) resource.view_users.add(user) if edit_groups: for group in edit_groups: group = utils.group_from_id(group) resource.edit_groups.add(group) resource.view_groups.add(group) if view_groups: for group in view_groups: group = utils.group_from_id(group) resource.view_groups.add(group) if keywords: ks = [Keyword.objects.get_or_create(title=k) for k in keywords] ks = zip( *ks )[0] # ignore whether something was created or not. zip is its own inverse for k in ks: AssignedKeyword.objects.create(content_object=resource, keyword=k) # for creating metadata elements based on the old metadata implementation if dublin_metadata: for d in dublin_metadata: QualifiedDublinCoreElement.objects.create(term=d['term'], content=d['content'], content_object=resource) # for creating metadata elements based on the new metadata implementation if metadata: for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] resource.metadata.create_element(k, **v) # add the subject elements from the AssignedKeywords (new metadata implementation) for akw in AssignedKeyword.objects.filter(object_pk=resource.id).all(): resource.metadata.create_element('subject', value=akw.keyword.title) hs_bagit.create_bag(resource) # Send post-create resource signal post_create_resource.send(sender=cls, resource=resource) return resource
def create_resource(resource_type, owner, title, edit_users=None, view_users=None, edit_groups=None, view_groups=None, keywords=(), metadata=None, extra_metadata=None, files=(), source_names=[], source_sizes=[], fed_res_path='', move=False, is_file_reference=False, create_metadata=True, create_bag=True, unpack_file=False, **kwargs): """ Called by a client to add a new resource to CommonsShare. The caller must have authorization to write content to CommonsShare. The pid for the resource is assigned by CommonsShare upon inserting the resource. The create method returns the newly-assigned pid. REST URL: POST /resource Parameters: Returns: The newly created resource Return Type: BaseResource resource object Note: The calling user will automatically be set as the owner of the created resource. Implementation notes: 1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing. 2. return type is an instance of hs_core.models.BaseResource class. This is for efficiency in the native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated lookups in the database when they are unnecessary. 3. resource_type is a string: see parameter list :param resource_type: string. the type of the resource such as GenericResource :param owner: email address, username, or User instance. The owner of the resource :param title: string. the title of the resource :param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions :param view_users: list of email addresses, usernames, or User instances who will be given view permissions :param edit_groups: list of group names or Group instances who will be given edit permissions :param view_groups: list of group names or Group instances who will be given view permissions :param keywords: string list. list of keywords to add to the resource :param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}. :param extra_metadata: one dict containing keys and corresponding values { 'Outlet Point Latitude': '40', 'Outlet Point Longitude': '-110'}. :param files: list of Django File or UploadedFile objects to be attached to the resource :param source_names: a list of file names from a federated zone to be used to create the resource in the federated zone, default is empty list :param source_sizes: a list of file sizes corresponding to source_names if if_file_reference is True; otherwise, it is not of any use and should be empty. :param fed_res_path: the federated zone path in the format of /federation_zone/home/localHydroProxy that indicate where the resource is stored, default is empty string :param move: a value of False or True indicating whether the content files should be erased from the source directory. default is False. :param is_file_reference: a value of False or True indicating whether the files stored in source_files are references to external files without being physically stored in HydroShare internally. default is False. :param create_bag: whether to create a bag for the newly created resource or not. By default, the bag is created. :param unpack_file: boolean. If files contains a single zip file, and unpack_file is True, the unpacked contents of the zip file will be added to the resource instead of the zip file. :param kwargs: extra arguments to fill in required values in AbstractResource subclasses :return: a new resource which is an instance of BaseResource with specificed resource_type. """ if __debug__: assert (isinstance(source_names, list)) with transaction.atomic(): cls = check_resource_type(resource_type) owner = utils.user_from_id(owner) # create the resource resource = cls.objects.create(resource_type=resource_type, user=owner, creator=owner, title=title, last_changed_by=owner, in_menus=[], **kwargs) resource.resource_type = resource_type # by default make resource private resource.set_slug('resource{0}{1}'.format('/', resource.short_id)) resource.save() if not metadata: metadata = [] if extra_metadata is not None: resource.extra_metadata = extra_metadata resource.save() fed_zone_home_path = '' if fed_res_path: resource.resource_federation_path = fed_res_path fed_zone_home_path = fed_res_path resource.save() # TODO: It would be safer to require an explicit zone path rather than harvesting file path elif len(source_names) > 0 and fed_res_path: fed_zone_home_path = utils.get_federated_zone_home_path( source_names[0]) resource.resource_federation_path = fed_zone_home_path resource.save() if len(files) == 1 and unpack_file and zipfile.is_zipfile(files[0]): # Add contents of zipfile as resource files asynchronously # Note: this is done asynchronously as unzipping may take # a long time (~15 seconds to many minutes). add_zip_file_contents_to_resource_async(resource, files[0]) else: # Add resource file(s) now # Note: this is done synchronously as it should only take a # few seconds. We may want to add the option to do this # asynchronously if the file size is large and would take # more than ~15 seconds to complete. add_resource_files(resource.short_id, *files, source_names=source_names, source_sizes=source_sizes, move=move, is_file_reference=is_file_reference) # by default resource is private resource_access = ResourceAccess(resource=resource) resource_access.save() # use the built-in share routine to set initial provenance. UserResourcePrivilege.share(resource=resource, grantor=owner, user=owner, privilege=PrivilegeCodes.OWNER) resource_labels = ResourceLabels(resource=resource) resource_labels.save() if edit_users: for user in edit_users: user = utils.user_from_id(user) owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.CHANGE) if view_users: for user in view_users: user = utils.user_from_id(user) owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.VIEW) if edit_groups: for group in edit_groups: group = utils.group_from_id(group) owner.uaccess.share_resource_with_group( resource, group, PrivilegeCodes.CHANGE) if view_groups: for group in view_groups: group = utils.group_from_id(group) owner.uaccess.share_resource_with_group( resource, group, PrivilegeCodes.VIEW) if create_metadata: # prepare default metadata utils.prepare_resource_default_metadata(resource=resource, metadata=metadata, res_title=title) for element in metadata: # here k is the name of the element # v is a dict of all element attributes/field names and field values k, v = element.items()[0] resource.metadata.create_element(k, **v) for keyword in keywords: resource.metadata.create_element('subject', value=keyword) resource.title = resource.metadata.title.value resource.save() if create_bag: hs_bagit.create_bag(resource) # set the resource to private resource.setAVU('isPublic', resource.raccess.public) # set the resource type (which is immutable) resource.setAVU("resourceType", resource._meta.object_name) # set quota of this resource to this creator resource.set_quota_holder(owner, owner) return resource