def copy_resource(ori_res, new_res, user=None):
"""
Populate metadata and contents from ori_res object to new_res object to make new_res object
as a copy of the ori_res object
Args:
ori_res: the original resource that is to be copied.
new_res: the new_res to be populated with metadata and content from the original resource
as a copy of the original resource.
user: requesting user for the copy action. It is optional, if being passed in, quota is
counted toward the user; otherwise, quota is not counted toward that user
Returns:
the new resource copied from the original resource
"""
# add files directly via irods backend file operation
utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id)
utils.copy_and_create_metadata(ori_res, new_res)
hs_identifier = ori_res.metadata.identifiers.all().filter(name="hydroShareIdentifier")[0]
if hs_identifier:
new_res.metadata.create_element('source', derived_from=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
hs_bagit.create_bag(new_res)
# need to update quota usage for new_res quota holder
if user:
update_quota_usage(user=user)
return new_res
def copy_resource(ori_res, new_res):
"""
Populate metadata and contents from ori_res object to new_res object to make new_res object
as a copy of the ori_res object
Args:
ori_res: the original resource that is to be copied.
new_res: the new_res to be populated with metadata and content from the original resource
as a copy of the original resource.
Returns:
the new resource copied from the original resource
"""
# add files directly via irods backend file operation
utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id)
utils.copy_and_create_metadata(ori_res, new_res)
hs_identifier = ori_res.metadata.identifiers.all().filter(name="hydroShareIdentifier")[0]
if hs_identifier:
new_res.metadata.create_element('source', derived_from=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
hs_bagit.create_bag(new_res)
return new_res
def copy_resource_task(ori_res_id, new_res_id=None, request_username=None):
try:
new_res = None
if not new_res_id:
new_res = create_empty_resource(ori_res_id,
request_username,
action='copy')
new_res_id = new_res.short_id
utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id)
ori_res = utils.get_resource_by_shortkey(ori_res_id)
if not new_res:
new_res = utils.get_resource_by_shortkey(new_res_id)
utils.copy_and_create_metadata(ori_res, new_res)
hs_identifier = ori_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
if hs_identifier:
new_res.metadata.create_element('source',
derived_from=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
create_bag(new_res)
return new_res.get_absolute_url()
except Exception as ex:
if new_res:
new_res.delete()
raise utils.ResourceCopyException(str(ex))
def create_new_version_resource(ori_res, new_res, user):
"""
Populate metadata and contents from ori_res object to new_res object to make new_res object as
a new version of the ori_res object
Args:
ori_res: the original resource that is to be versioned.
new_res: the new_res to be populated with metadata and content from the original resource
to make it a new version
user: the requesting user
Returns:
the new versioned resource for the original resource and thus obsolete the original resource
"""
# newly created new resource version is private initially
# add files directly via irods backend file operation
utils.copy_resource_files_and_AVUs(ori_res.short_id, new_res.short_id)
# copy metadata from source resource to target new-versioned resource except three elements
utils.copy_and_create_metadata(ori_res, new_res)
# add or update Relation element to link source and target resources
hs_identifier = new_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
ori_res.metadata.create_element('relation',
type='isReplacedBy',
value=hs_identifier.url)
if new_res.metadata.relations.all().filter(type='isVersionOf').exists():
# the original resource is already a versioned resource, and its isVersionOf relation
# element is copied over to this new version resource, needs to delete this element so
# it can be created to link to its original resource correctly
eid = new_res.metadata.relations.all().filter(
type='isVersionOf').first().id
new_res.metadata.delete_element('relation', eid)
hs_identifier = ori_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
new_res.metadata.create_element('relation',
type='isVersionOf',
value=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new version collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
hs_bagit.create_bag(new_res)
# since an isReplaceBy relation element is added to original resource, needs to call
# resource_modified() for original resource
utils.resource_modified(ori_res, user, overwrite_bag=False)
# if everything goes well up to this point, set original resource to be immutable so that
# obsoleted resources cannot be modified from REST API
ori_res.raccess.immutable = True
ori_res.raccess.save()
return new_res
def test_create_bag(self):
# the resource should have only one bags object
self.assertEquals(self.test_res.bags.count(), 1)
old_bag = self.test_res.bags.all().first()
# this is the api call we are testing
new_bag = hs_bagit.create_bag(self.test_res)
# resource should have one new bags object
self.assertEquals(self.test_res.bags.count(), 1)
self.assertEquals(new_bag, self.test_res.bags.all().first())
self.assertNotEquals(old_bag, new_bag)
def test_create_bag(self):
# the resource should have only one bags object
self.assertEquals(self.test_res.bags.count(), 1)
old_bag = self.test_res.bags.all().first()
# this is the api call we are testing
new_bag = hs_bagit.create_bag(self.test_res)
# resource should have one new bags object
self.assertEquals(self.test_res.bags.count(), 1)
self.assertEquals(new_bag, self.test_res.bags.all().first())
self.assertNotEquals(old_bag, new_bag)
def create_resource(
resource_type, owner, title,
edit_users=None, view_users=None, edit_groups=None, view_groups=None,
keywords=(), metadata=None, extra_metadata=None,
files=(), create_metadata=True, create_bag=True, unpack_file=False, full_paths={},
auto_aggregate=True, **kwargs):
"""
Called by a client to add a new resource to HydroShare. The caller must have authorization to
write content to HydroShare. The pid for the resource is assigned by HydroShare upon inserting
the resource. The create method returns the newly-assigned pid.
REST URL: POST /resource
Parameters:
Returns: The newly created resource
Return Type: BaseResource resource object
Note: The calling user will automatically be set as the owner of the created resource.
Implementation notes:
1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be
confusing.
2. return type is an instance of hs_core.models.BaseResource class. This is for efficiency in
the native API. The native API should return actual instance rather than IDs wherever possible
to avoid repeated lookups in the database when they are unnecessary.
3. resource_type is a string: see parameter list
:param resource_type: string. the type of the resource such as GenericResource
:param owner: email address, username, or User instance. The owner of the resource
:param title: string. the title of the resource
:param edit_users: list of email addresses, usernames, or User instances who will be given edit
permissions
:param view_users: list of email addresses, usernames, or User instances who will be given view
permissions
:param edit_groups: list of group names or Group instances who will be given edit permissions
:param view_groups: list of group names or Group instances who will be given view permissions
:param keywords: string list. list of keywords to add to the resource
:param metadata: list of dicts containing keys (element names) and corresponding values as
dicts { 'creator': {'name':'John Smith'}}.
:param extra_metadata: one dict containing keys and corresponding values
{ 'Outlet Point Latitude': '40', 'Outlet Point Longitude': '-110'}.
:param files: list of Django File or UploadedFile objects to be attached to the resource
:param create_bag: whether to create a bag for the newly created resource or not.
By default, the bag is created.
:param unpack_file: boolean. If files contains a single zip file, and unpack_file is True,
the unpacked contents of the zip file will be added to the resource instead of the zip file.
:param full_paths: Optional. A map of paths keyed by the correlating resource file. When
this parameter is provided, a file will be placed at the path specified in the map.
:param auto_aggregate: boolean, defaults to True. Find and create aggregations during
resource creation.
:param kwargs: extra arguments to fill in required values in AbstractResource subclasses
:return: a new resource which is an instance of BaseResource with specificed resource_type.
"""
with transaction.atomic():
cls = check_resource_type(resource_type)
owner = utils.user_from_id(owner)
# get the metadata class specific to resource type to set resource
# content_object (metadata) attribute
metadata_class = cls.get_metadata_class()
metadata_obj = metadata_class()
metadata_obj.save()
# create the resource
resource = cls.objects.create(
resource_type=resource_type,
user=owner,
creator=owner,
title=title,
last_changed_by=owner,
in_menus=[],
content_object=metadata_obj,
**kwargs
)
resource.resource_type = resource_type
# by default make resource private
resource.set_slug('resource{0}{1}'.format('/', resource.short_id))
resource.save()
if not metadata:
metadata = []
if extra_metadata is not None:
resource.extra_metadata = extra_metadata
resource.save()
# by default resource is private
resource_access = ResourceAccess(resource=resource)
resource_access.save()
# use the built-in share routine to set initial provenance.
UserResourcePrivilege.share(resource=resource, grantor=owner, user=owner,
privilege=PrivilegeCodes.OWNER)
resource_labels = ResourceLabels(resource=resource)
resource_labels.save()
if edit_users:
for user in edit_users:
user = utils.user_from_id(user)
owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.CHANGE)
if view_users:
for user in view_users:
user = utils.user_from_id(user)
owner.uaccess.share_resource_with_user(resource, user, PrivilegeCodes.VIEW)
if edit_groups:
for group in edit_groups:
group = utils.group_from_id(group)
owner.uaccess.share_resource_with_group(resource, group, PrivilegeCodes.CHANGE)
if view_groups:
for group in view_groups:
group = utils.group_from_id(group)
owner.uaccess.share_resource_with_group(resource, group, PrivilegeCodes.VIEW)
# set quota of this resource to this creator
# quota holder has to be set before the files are added in order for real time iRODS
# quota micro-services to work
resource.set_quota_holder(owner, owner)
if create_metadata:
# prepare default metadata
utils.prepare_resource_default_metadata(resource=resource, metadata=metadata,
res_title=title)
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
for keyword in keywords:
resource.metadata.create_element('subject', value=keyword)
resource.title = resource.metadata.title.value
resource.save()
if len(files) == 1 and unpack_file and zipfile.is_zipfile(files[0]):
# Add contents of zipfile as resource files asynchronously
# Note: this is done asynchronously as unzipping may take
# a long time (~15 seconds to many minutes).
add_zip_file_contents_to_resource_async(resource, files[0])
else:
# Add resource file(s) now
# Note: this is done synchronously as it should only take a
# few seconds. We may want to add the option to do this
# asynchronously if the file size is large and would take
# more than ~15 seconds to complete.
add_resource_files(resource.short_id, *files, full_paths=full_paths,
auto_aggregate=auto_aggregate)
if create_bag:
hs_bagit.create_bag(resource)
# set the resource to private
resource.setAVU('isPublic', resource.raccess.public)
# set the resource type (which is immutable)
resource.setAVU("resourceType", resource._meta.object_name)
return resource
def generate_files(request, shortkey, *args, **kwargs):
res = hydroshare.get_resource_by_shortkey(shortkey)
ts, csv_link, csv_size, xml_link, xml_size = {}, '', '', '', ''
try:
if res.reference_type == 'rest':
ts = ts_utils.time_series_from_service(res.url, res.reference_type)
else:
ts = ts_utils.time_series_from_service(res.url,
res.reference_type,
site_name_or_code=res.data_site_code,
variable_code=res.variable_code)
vals = ts['values']
version = ts['wml_version']
d = datetime.date.today()
date = '{0}_{1}_{2}'.format(d.month, d.day, d.year)
file_base = '{0}-{1}'.format(res.title.replace(" ", ""), date)
csv_name = '{0}.{1}'.format(file_base, 'csv')
if version == '1':
xml_end = 'wml_1'
xml_name = '{0}-{1}.xml'.format(file_base, xml_end)
elif version == '2.0':
xml_end = 'wml_2_0'
xml_name = '{0}-{1}.xml'.format(file_base, xml_end)
for_csv = []
for k, v in vals.items():
t = (k, v)
for_csv.append(t)
ResourceFile.objects.filter(object_id=res.pk).delete()
with open(csv_name, 'wb') as csv_file:
w = csv.writer(csv_file)
w.writerow([res.title])
var = '{0}({1})'.format(ts['variable_name'], ts['units'])
w.writerow(['time', var])
for r in for_csv:
w.writerow(r)
with open(xml_name, 'wb') as xml_file:
xml_file.write(ts['time_series'])
csv_file = open(csv_name, 'r')
xml_file = open(xml_name, 'r')
files = [csv_file, xml_file]
hydroshare.add_resource_files(res.short_id, csv_file, xml_file)
create_bag(res)
os.remove(csv_name)
os.remove(xml_name)
files = ResourceFile.objects.filter(object_id=res.pk)
for f in files:
if str(f.resource_file).endswith('.csv'):
csv_link = f.resource_file.url
csv_size = f.resource_file.size
if xml_end in str(f.resource_file):
xml_link = f.resource_file.url
xml_size = f.resource_file.size
status_code = 200
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_name': csv_name,
'xml_name': xml_name,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # successfully generated new files
except Exception: # most likely because the server is unreachable
files = ResourceFile.objects.filter(object_id=res.pk)
xml_file = None
for f in files:
if str(f.resource_file).endswith('.csv'):
csv_link = f.resource_file.url
csv_size = f.resource_file.size
if str(f.resource_file).endswith('.xml'):
xml_link = f.resource_file.url
xml_size = f.resource_file.size
xml_file = f.resource_file
if xml_file is None:
status_code = 404
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # did not generate new files, did not find old ones
xml_doc = open(str(xml_file), 'r').read()
root = etree.XML(xml_doc)
os.remove(str(xml_file))
version = ts_utils.get_version(root)
if version == '1':
ts = ts_utils.parse_1_0_and_1_1(root)
status_code = 200
elif version =='2.0':
ts = ts_utils.parse_2_0(root)
status_code = 200
else:
status_code = 503
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # did not generate new files, return old ones
def create_resource(
resource_type, owner, title,
edit_users=None, view_users=None, edit_groups=None, view_groups=None,
keywords=None, dublin_metadata=None, metadata=None,
files=(), **kwargs):
"""
Called by a client to add a new resource to HydroShare. The caller must have authorization to write content to
HydroShare. The pid for the resource is assigned by HydroShare upon inserting the resource. The create method
returns the newly-assigned pid.
REST URL: POST /resource
Parameters:
resource - The data bytes of the resource to be added to HydroShare
Returns: The pid assigned to the newly created resource
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized to write to HydroShare
Exceptions.InvalidContent - The content of the resource is incomplete
Exception.ServiceFailure - The service is unable to process the request
Note: The calling user will automatically be set as the owner of the created resource.
Implementation notes:
1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing.
2. return type is an instance of a subclass of hs_core.models.AbstractResource. This is for efficiency in the
native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated
lookups in the database when they are unnecessary.
3. resource_type is a string: see parameter list
:param resource_type: string. the classname of the resource type, such as GenericResource
:param owner: email address, username, or User instance. The owner of the resource
:param title: string. the title of the resource
:param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions
:param view_users: list of email addresses, usernames, or User instances who will be given view permissions
:param edit_groups: list of group names or Group instances who will be given edit permissions
:param view_groups: list of group names or Group instances who will be given view permissions
:param keywords: string list. list of keywords to add to the resource
:param dublin_metadata: list of dicts containing keys { 'term', 'content' } respecting dublin core std.
:param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}.
:param files: list of Django File or UploadedFile objects to be attached to the resource
:param kwargs: extra arguments to fill in required values in AbstractResource subclasses
:return: a new resource which is an instance of resource_type.
"""
for tp in get_resource_types():
if resource_type == tp.__name__:
cls = tp
break
else:
raise NotImplementedError("Type {resource_type} does not exist".format(resource_type=resource_type))
# Send pre-create resource signal
pre_create_resource.send(sender=cls, dublin_metadata=dublin_metadata, files=files,
**kwargs)
owner = utils.user_from_id(owner)
# create the resource
resource = cls.objects.create(
user=owner,
creator=owner,
title=title,
last_changed_by=owner,
in_menus=[],
**kwargs
)
for file in files:
ResourceFile.objects.create(content_object=resource, resource_file=file)
resource.view_users.add(owner)
resource.edit_users.add(owner)
resource.owners.add(owner)
if edit_users:
for user in edit_users:
user = utils.user_from_id(user)
resource.edit_users.add(user)
resource.view_users.add(user)
if view_users:
for user in view_users:
user = utils.user_from_id(user)
resource.view_users.add(user)
if edit_groups:
for group in edit_groups:
group = utils.group_from_id(group)
resource.edit_groups.add(group)
resource.view_groups.add(group)
if view_groups:
for group in view_groups:
group = utils.group_from_id(group)
resource.view_groups.add(group)
if keywords:
ks = [Keyword.objects.get_or_create(title=k) for k in keywords]
ks = zip(*ks)[0] # ignore whether something was created or not. zip is its own inverse
for k in ks:
AssignedKeyword.objects.create(content_object=resource, keyword=k)
# for creating metadata elements based on the old metadata implementation
if dublin_metadata:
for d in dublin_metadata:
QualifiedDublinCoreElement.objects.create(
term=d['term'],
content=d['content'],
content_object=resource
)
# for creating metadata elements based on the new metadata implementation
if metadata:
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
# add the subject elements from the AssignedKeywords (new metadata implementation)
for akw in AssignedKeyword.objects.filter(object_pk=resource.id).all():
resource.metadata.create_element('subject', value=akw.keyword.title)
hs_bagit.create_bag(resource)
# Send post-create resource signal
post_create_resource.send(sender=cls, resource=resource)
return resource
def create_new_version_resource_task(ori_res_id, username, new_res_id=None):
"""
Task for creating a new version of a resource
Args:
ori_res_id: the original resource id that is to be versioned.
new_res_id: the new versioned resource id from the original resource. If None, a
new resource will be created.
username: the requesting user's username
Returns:
the new versioned resource url as the payload
"""
try:
new_res = None
if not new_res_id:
new_res = create_empty_resource(ori_res_id, username)
new_res_id = new_res.short_id
utils.copy_resource_files_and_AVUs(ori_res_id, new_res_id)
# copy metadata from source resource to target new-versioned resource except three elements
ori_res = utils.get_resource_by_shortkey(ori_res_id)
if not new_res:
new_res = utils.get_resource_by_shortkey(new_res_id)
utils.copy_and_create_metadata(ori_res, new_res)
# add or update Relation element to link source and target resources
hs_identifier = new_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
ori_res.metadata.create_element('relation',
type='isReplacedBy',
value=hs_identifier.url)
if new_res.metadata.relations.all().filter(
type='isVersionOf').exists():
# the original resource is already a versioned resource, and its isVersionOf relation
# element is copied over to this new version resource, needs to delete this element so
# it can be created to link to its original resource correctly
eid = new_res.metadata.relations.all().filter(
type='isVersionOf').first().id
new_res.metadata.delete_element('relation', eid)
hs_identifier = ori_res.metadata.identifiers.all().filter(
name="hydroShareIdentifier")[0]
new_res.metadata.create_element('relation',
type='isVersionOf',
value=hs_identifier.url)
if ori_res.resource_type.lower() == "collectionresource":
# clone contained_res list of original collection and add to new collection
# note that new version collection will not contain "deleted resources"
new_res.resources = ori_res.resources.all()
# create bag for the new resource
create_bag(new_res)
# since an isReplaceBy relation element is added to original resource, needs to call
# resource_modified() for original resource
utils.resource_modified(ori_res, by_user=username, overwrite_bag=False)
# if everything goes well up to this point, set original resource to be immutable so that
# obsoleted resources cannot be modified from REST API
ori_res.raccess.immutable = True
ori_res.raccess.save()
ori_res.save()
return new_res.get_absolute_url()
except Exception as ex:
if new_res:
new_res.delete()
raise utils.ResourceVersioningException(str(ex))
finally:
# release the lock regardless
ori_res.locked_time = None
ori_res.save()
def handle(self, *args, **options):
if not options['resource_id']:
raise CommandError('resource_id argument is required')
res_id = options['resource_id']
try:
res = get_resource_by_shortkey(res_id, or_404=False)
except ObjectDoesNotExist:
raise CommandError("No Resource found for id {}".format(res_id))
if options['new_resource_id']:
try:
UUID(options['new_resource_id'])
new_res_id = options['new_resource_id']
except Exception as e:
raise CommandError('new_resource_id {} must be a valid uuid hex string'
.format(options['new_resource_id']), e)
try:
if BaseResource.objects.get(short_id=new_res_id):
raise CommandError('resource with id {} already exists'.format(new_res_id))
except ObjectDoesNotExist:
pass
else:
new_res_id = short_id()
storage = res.get_irods_storage()
if storage.exists(res.bag_path):
try:
storage.delete(res.bag_path)
print("{} deleted".format(res.bag_path))
except SessionException as ex:
print("{} delete failed: {}".format(res.bag_path, ex.stderr))
raise EnvironmentError()
try:
with transaction.atomic():
print("Deleting existing bag")
res.setAVU("bag_modified", True)
res.setAVU('metadata_dirty', 'true')
print("Updating BaseResource short_id from {} to {}".format(res_id, new_res_id))
res.short_id = new_res_id
res.save()
print("Updating resource slug")
res.set_slug('resource/{}'.format(new_res_id))
print("Updating Resource files short_path")
for file in res.files.all():
file_name = file.short_path.split('data/contents/')[1]
file.set_short_path(file_name)
print("Updating metadata identifiers")
for i in res.metadata.identifiers.all():
i.url = i.url.replace(res_id, new_res_id)
i.save()
print("Updating logical_files metadata")
for aggregation in res.logical_files:
aggregation.metadata.is_dirty = True
aggregation.metadata.save()
except IntegrityError:
raise EnvironmentError("Error occurred while updating")
print("Moving Resource files")
storage.moveFile(res_id, new_res_id)
print("Creating Bag")
create_bag(res)
print(("Resource id successfully update from {} to {}".format(res_id, new_res_id)))
def download(request, path, rest_call=False, use_async=True, *args, **kwargs):
split_path_strs = path.split('/')
is_bag_download = False
if split_path_strs[0] == 'bags':
res_id = os.path.splitext(split_path_strs[1])[0]
is_bag_download = True
else:
res_id = split_path_strs[0]
res, authorized, _ = authorize(
request,
res_id,
needed_permission=ACTION_TO_AUTHORIZE.VIEW_RESOURCE,
raises_exception=False)
if not authorized:
response = HttpResponse(status=401)
content_msg = "You do not have permission to download this resource!"
if rest_call:
raise PermissionDenied(content_msg)
else:
signin_html = '</h1><div class="col-xs-12"><h2 class="page-title">' \
'<a href="/oauth_request/"><span class ="glyphicon glyphicon-log-in"></span>' \
'Sign In</a></h2>'
response.content = '<h1>' + content_msg + signin_html
return response
if not is_bag_download and "/data" not in path:
idx_sep = path.find('/')
path = path[idx_sep:]
istorage = IrodsStorage()
if 'environment' in kwargs:
environment = int(kwargs['environment'])
environment = m.RodsEnvironment.objects.get(pk=environment)
session = Session("/tmp/django_irods",
settings.IRODS_ICOMMANDS_PATH,
session_id=uuid4())
session.create_environment(environment)
session.run('iinit', None, environment.auth)
elif getattr(settings, 'IRODS_GLOBAL_SESSION', False):
session = GLOBAL_SESSION
elif icommands.ACTIVE_SESSION:
session = icommands.ACTIVE_SESSION
else:
raise KeyError('settings must have IRODS_GLOBAL_SESSION set '
'if there is no environment object')
if istorage.exists(res_id) and is_bag_download:
bag_modified = istorage.getAVU(res_id, 'bag_modified')
# make sure if bag_modified is not set to true, we still recreate the bag if the
# bag file does not exist for some reason to resolve the error to download a nonexistent
# bag when bag_modified is false due to the flag being out-of-sync with the real bag status
if bag_modified is None or bag_modified.lower() == "false":
# check whether the bag file exists
bag_file_name = res_id + '.zip'
bag_full_path = os.path.join('bags', bag_file_name)
if not istorage.exists(bag_full_path):
bag_modified = 'true'
if bag_modified is None or bag_modified.lower() == "true":
create_bag(res)
resource_cls = check_resource_type(res.resource_type)
# send signal for pre download file
download_file_name = split_path_strs[-1]
pre_download_file.send(sender=resource_cls,
resource=res,
download_file_name=download_file_name,
request=request)
# obtain mime_type to set content_type
mtype = 'application-x/octet-stream'
mime_type = mimetypes.guess_type(path)
if mime_type[0] is not None:
mtype = mime_type[0]
# retrieve file size to set up Content-Length header
stdout = session.run("ils", None, "-l", path)[0].split()
flen = int(stdout[3])
options = ('-', ) # we're redirecting to stdout.
proc = session.run_safe('iget', None, path, *options)
response = FileResponse(proc.stdout, content_type=mtype)
response['Content-Disposition'] = 'attachment; filename="{name}"'.format(
name=path.split('/')[-1])
response['Content-Length'] = flen
return response
def create_resource(resource_type,
owner,
title,
edit_users=None,
view_users=None,
edit_groups=None,
view_groups=None,
keywords=None,
dublin_metadata=None,
metadata=None,
files=(),
**kwargs):
"""
Called by a client to add a new resource to HydroShare. The caller must have authorization to write content to
HydroShare. The pid for the resource is assigned by HydroShare upon inserting the resource. The create method
returns the newly-assigned pid.
REST URL: POST /resource
Parameters:
resource - The data bytes of the resource to be added to HydroShare
Returns: The pid assigned to the newly created resource
Return Type: pid
Raises:
Exceptions.NotAuthorized - The user is not authorized to write to HydroShare
Exceptions.InvalidContent - The content of the resource is incomplete
Exception.ServiceFailure - The service is unable to process the request
Note: The calling user will automatically be set as the owner of the created resource.
Implementation notes:
1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be confusing.
2. return type is an instance of a subclass of hs_core.models.AbstractResource. This is for efficiency in the
native API. The native API should return actual instance rather than IDs wherever possible to avoid repeated
lookups in the database when they are unnecessary.
3. resource_type is a string: see parameter list
:param resource_type: string. the classname of the resource type, such as GenericResource
:param owner: email address, username, or User instance. The owner of the resource
:param title: string. the title of the resource
:param edit_users: list of email addresses, usernames, or User instances who will be given edit permissions
:param view_users: list of email addresses, usernames, or User instances who will be given view permissions
:param edit_groups: list of group names or Group instances who will be given edit permissions
:param view_groups: list of group names or Group instances who will be given view permissions
:param keywords: string list. list of keywords to add to the resource
:param dublin_metadata: list of dicts containing keys { 'term', 'content' } respecting dublin core std.
:param metadata: list of dicts containing keys (element names) and corresponding values as dicts { 'creator': {'name':'John Smith'}}.
:param files: list of Django File or UploadedFile objects to be attached to the resource
:param kwargs: extra arguments to fill in required values in AbstractResource subclasses
:return: a new resource which is an instance of resource_type.
"""
for tp in get_resource_types():
if resource_type == tp.__name__:
cls = tp
break
else:
raise NotImplementedError("Type {resource_type} does not exist".format(
resource_type=resource_type))
# Send pre-create resource signal
pre_create_resource.send(sender=cls,
dublin_metadata=dublin_metadata,
files=files,
**kwargs)
owner = utils.user_from_id(owner)
# create the resource
resource = cls.objects.create(user=owner,
creator=owner,
title=title,
last_changed_by=owner,
in_menus=[],
**kwargs)
for file in files:
ResourceFile.objects.create(content_object=resource,
resource_file=file)
resource.view_users.add(owner)
resource.edit_users.add(owner)
resource.owners.add(owner)
if edit_users:
for user in edit_users:
user = utils.user_from_id(user)
resource.edit_users.add(user)
resource.view_users.add(user)
if view_users:
for user in view_users:
user = utils.user_from_id(user)
resource.view_users.add(user)
if edit_groups:
for group in edit_groups:
group = utils.group_from_id(group)
resource.edit_groups.add(group)
resource.view_groups.add(group)
if view_groups:
for group in view_groups:
group = utils.group_from_id(group)
resource.view_groups.add(group)
if keywords:
ks = [Keyword.objects.get_or_create(title=k) for k in keywords]
ks = zip(
*ks
)[0] # ignore whether something was created or not. zip is its own inverse
for k in ks:
AssignedKeyword.objects.create(content_object=resource, keyword=k)
# for creating metadata elements based on the old metadata implementation
if dublin_metadata:
for d in dublin_metadata:
QualifiedDublinCoreElement.objects.create(term=d['term'],
content=d['content'],
content_object=resource)
# for creating metadata elements based on the new metadata implementation
if metadata:
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
# add the subject elements from the AssignedKeywords (new metadata implementation)
for akw in AssignedKeyword.objects.filter(object_pk=resource.id).all():
resource.metadata.create_element('subject', value=akw.keyword.title)
hs_bagit.create_bag(resource)
# Send post-create resource signal
post_create_resource.send(sender=cls, resource=resource)
return resource
def create_resource(resource_type,
owner,
title,
edit_users=None,
view_users=None,
edit_groups=None,
view_groups=None,
keywords=(),
metadata=None,
extra_metadata=None,
files=(),
source_names=[],
source_sizes=[],
fed_res_path='',
move=False,
is_file_reference=False,
create_metadata=True,
create_bag=True,
unpack_file=False,
**kwargs):
"""
Called by a client to add a new resource to CommonsShare. The caller must have authorization to
write content to CommonsShare. The pid for the resource is assigned by CommonsShare upon inserting
the resource. The create method returns the newly-assigned pid.
REST URL: POST /resource
Parameters:
Returns: The newly created resource
Return Type: BaseResource resource object
Note: The calling user will automatically be set as the owner of the created resource.
Implementation notes:
1. pid is called short_id. This is because pid is a UNIX term for Process ID and could be
confusing.
2. return type is an instance of hs_core.models.BaseResource class. This is for efficiency in
the native API. The native API should return actual instance rather than IDs wherever possible
to avoid repeated lookups in the database when they are unnecessary.
3. resource_type is a string: see parameter list
:param resource_type: string. the type of the resource such as GenericResource
:param owner: email address, username, or User instance. The owner of the resource
:param title: string. the title of the resource
:param edit_users: list of email addresses, usernames, or User instances who will be given edit
permissions
:param view_users: list of email addresses, usernames, or User instances who will be given view
permissions
:param edit_groups: list of group names or Group instances who will be given edit permissions
:param view_groups: list of group names or Group instances who will be given view permissions
:param keywords: string list. list of keywords to add to the resource
:param metadata: list of dicts containing keys (element names) and corresponding values as
dicts { 'creator': {'name':'John Smith'}}.
:param extra_metadata: one dict containing keys and corresponding values
{ 'Outlet Point Latitude': '40', 'Outlet Point Longitude': '-110'}.
:param files: list of Django File or UploadedFile objects to be attached to the resource
:param source_names: a list of file names from a federated zone to be
used to create the resource in the federated zone, default is empty list
:param source_sizes: a list of file sizes corresponding to source_names if if_file_reference is True; otherwise,
it is not of any use and should be empty.
:param fed_res_path: the federated zone path in the format of
/federation_zone/home/localHydroProxy that indicate where the resource
is stored, default is empty string
:param move: a value of False or True indicating whether the content files
should be erased from the source directory. default is False.
:param is_file_reference: a value of False or True indicating whether the files stored in
source_files are references to external files without being physically stored in
HydroShare internally. default is False.
:param create_bag: whether to create a bag for the newly created resource or not.
By default, the bag is created.
:param unpack_file: boolean. If files contains a single zip file, and unpack_file is True,
the unpacked contents of the zip file will be added to the resource instead of the zip file.
:param kwargs: extra arguments to fill in required values in AbstractResource subclasses
:return: a new resource which is an instance of BaseResource with specificed resource_type.
"""
if __debug__:
assert (isinstance(source_names, list))
with transaction.atomic():
cls = check_resource_type(resource_type)
owner = utils.user_from_id(owner)
# create the resource
resource = cls.objects.create(resource_type=resource_type,
user=owner,
creator=owner,
title=title,
last_changed_by=owner,
in_menus=[],
**kwargs)
resource.resource_type = resource_type
# by default make resource private
resource.set_slug('resource{0}{1}'.format('/', resource.short_id))
resource.save()
if not metadata:
metadata = []
if extra_metadata is not None:
resource.extra_metadata = extra_metadata
resource.save()
fed_zone_home_path = ''
if fed_res_path:
resource.resource_federation_path = fed_res_path
fed_zone_home_path = fed_res_path
resource.save()
# TODO: It would be safer to require an explicit zone path rather than harvesting file path
elif len(source_names) > 0 and fed_res_path:
fed_zone_home_path = utils.get_federated_zone_home_path(
source_names[0])
resource.resource_federation_path = fed_zone_home_path
resource.save()
if len(files) == 1 and unpack_file and zipfile.is_zipfile(files[0]):
# Add contents of zipfile as resource files asynchronously
# Note: this is done asynchronously as unzipping may take
# a long time (~15 seconds to many minutes).
add_zip_file_contents_to_resource_async(resource, files[0])
else:
# Add resource file(s) now
# Note: this is done synchronously as it should only take a
# few seconds. We may want to add the option to do this
# asynchronously if the file size is large and would take
# more than ~15 seconds to complete.
add_resource_files(resource.short_id,
*files,
source_names=source_names,
source_sizes=source_sizes,
move=move,
is_file_reference=is_file_reference)
# by default resource is private
resource_access = ResourceAccess(resource=resource)
resource_access.save()
# use the built-in share routine to set initial provenance.
UserResourcePrivilege.share(resource=resource,
grantor=owner,
user=owner,
privilege=PrivilegeCodes.OWNER)
resource_labels = ResourceLabels(resource=resource)
resource_labels.save()
if edit_users:
for user in edit_users:
user = utils.user_from_id(user)
owner.uaccess.share_resource_with_user(resource, user,
PrivilegeCodes.CHANGE)
if view_users:
for user in view_users:
user = utils.user_from_id(user)
owner.uaccess.share_resource_with_user(resource, user,
PrivilegeCodes.VIEW)
if edit_groups:
for group in edit_groups:
group = utils.group_from_id(group)
owner.uaccess.share_resource_with_group(
resource, group, PrivilegeCodes.CHANGE)
if view_groups:
for group in view_groups:
group = utils.group_from_id(group)
owner.uaccess.share_resource_with_group(
resource, group, PrivilegeCodes.VIEW)
if create_metadata:
# prepare default metadata
utils.prepare_resource_default_metadata(resource=resource,
metadata=metadata,
res_title=title)
for element in metadata:
# here k is the name of the element
# v is a dict of all element attributes/field names and field values
k, v = element.items()[0]
resource.metadata.create_element(k, **v)
for keyword in keywords:
resource.metadata.create_element('subject', value=keyword)
resource.title = resource.metadata.title.value
resource.save()
if create_bag:
hs_bagit.create_bag(resource)
# set the resource to private
resource.setAVU('isPublic', resource.raccess.public)
# set the resource type (which is immutable)
resource.setAVU("resourceType", resource._meta.object_name)
# set quota of this resource to this creator
resource.set_quota_holder(owner, owner)
return resource
