def check_for_dangling_irods(echo_errors=True, log_errors=False, return_errors=False): """ This checks for resource trees in iRODS with no correspondence to Django at all :param log_errors: whether to log errors to Django log :param echo_errors: whether to print errors on stdout :param return_errors: whether to collect errors in an array and return them. """ istorage = IrodsStorage() # local only toplevel = istorage.listdir('.') # list the resources themselves logger = logging.getLogger(__name__) errors = [] for id in toplevel[0]: # directories try: get_resource_by_shortkey(id, or_404=False) except BaseResource.DoesNotExist: msg = "resource {} does not exist in Django".format(id) if echo_errors: print(msg) if log_errors: logger.error(msg) if return_errors: errors.append(msg) return errors
def test_09_retract(self): """Retracted resources cannot be accessed""" chewies = self.chewies resource_short_id = chewies.short_id hydroshare.delete_resource(chewies.short_id) with self.assertRaises(Http404): hydroshare.get_resource_by_shortkey(resource_short_id)
def check_relations(resource): """Check for dangling relations due to deleted resource files. :param resource: resource to check """ for r in resource.metadata.relations.all(): if r.value.startswith('http://www.hydroshare.org/resource/'): target = r.value[len('http://www.hydroshare.org/resource/'):].rstrip('/') try: get_resource_by_shortkey(target, or_404=False) except BaseResource.DoesNotExist: print("relation {} {} {} (this does not exist)" .format(resource.short_id, r.type, target))
def generate_files(shortkey, ts): ''' creates the calls the make files fxn and adds the files as resource files called by view: create_ref_time_series, update_files :param shortkey: res shortkey :param ts: parsed time series dict ''' res = hydroshare.get_resource_by_shortkey(shortkey) if ts is None: #called by update_files in view if res.metadata.referenceURLs.all()[0].type == 'rest': ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type) else: ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type, site_name_or_code=res.metadata.sites.all()[0].code, variable_code=res.metadata.variables.all()[0].code) tempdir = None try: tempdir = tempfile.mkdtemp() files = make_files(res, tempdir, ts) hydroshare.add_resource_files(res.short_id, *files) except Exception as e: raise e finally: if tempdir is not None: shutil.rmtree(tempdir)
def generate_resource_files(shortkey, tempdir): res = hydroshare.get_resource_by_shortkey(shortkey) if res.metadata.referenceURLs.all()[0].type == 'rest': ts = QueryHydroServerGetParsedWML( res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type) else: site_code = res.metadata.sites.all()[0].code # net_work = res.metadata.sites.all()[0].net_work variable_code = res.metadata.variables.all()[0].code method_code = res.metadata.methods.all()[0].code source_code = res.metadata.datasources.all()[0].code quality_control_level_code = res.metadata.quality_levels.all()[0].code site_code_query = "%s:%s" % ("network", site_code) variable_code_query = "%s:%s:methodCode=%s:sourceCode=%s:qualityControlLevelCode=%s" % \ ("network", variable_code, method_code, source_code, quality_control_level_code) ts = QueryHydroServerGetParsedWML( service_url=res.metadata.referenceURLs.all()[0].value, soap_or_rest=res.metadata.referenceURLs.all()[0].type, site_code=site_code_query, variable_code=variable_code_query) files = save_ts_to_files(res, tempdir, ts) return files
def test_add_files(self): user = User.objects.create_user('shaun', '*****@*****.**', 'shaun6745') #create user #create files n1 = "test.txt" n2 = "test2.txt" n3 = "test3.txt" open(n1,"w").close() #files are created open(n2,"w").close() open(n3,"w").close() myfile = open(n1,"r") #files are opened as 'read-only' myfile1 = open(n2,"r") myfile2 = open(n3,"r") res1 = create_resource('GenericResource',user,'res1') #create resource #delete all resource files for created resource res1.files.all().delete() #add files add_resource_files(res1.short_id,myfile,myfile1,myfile2) #add each file of resource to list res1 = get_resource_by_shortkey(res1.short_id) l=[] for f in res1.files.all(): l.append(f.resource_file.name.split('/')[-1]) #check if the file name is in the list of files self.assertTrue(n1 in l, "file 1 has not been added") self.assertTrue(n2 in l, "file 2 has not been added") self.assertTrue(n3 in l, "file 3 has not been added")
def delete(self, request, pk): view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE_ACCESS) keys = request.query_params.keys() user_access = UserAccess(user=request.user) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) if "user_id" in keys and "group_id" in keys: message = "Request cannot contain both a 'user_id' and a 'group_id' parameter." return Response( data={'error': message}, status=status.HTTP_400_BAD_REQUEST ) if "user_id" in keys: user_to_remove = utils.user_from_id(request.query_params['user_id']) user_access.unshare_resource_with_user(resource, user_to_remove) return Response( data={'success': "Resource access privileges removed."}, status=status.HTTP_202_ACCEPTED ) if "group_id" in keys: group_to_remove = utils.group_from_id(request.query_params['group_id']) user_access.unshare_resource_with_group(resource, group_to_remove) return Response( data={'success': "Resource access privileges removed."}, status=status.HTTP_202_ACCEPTED ) message = "Request must contain a 'resource' ID as well as a 'user_id' or 'group_id'" return Response( data={'error': message}, status=status.HTTP_400_BAD_REQUEST )
def get(self, request, pk): view_utils.authorize( request, pk, needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) serializer = resource.metadata.serializer self.serializer_class = resource.metadata.serializer return Response(data=serializer.data, status=status.HTTP_200_OK)
def put(self, request, pk): # Update science metadata resource, _, _ = view_utils.authorize( request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) metadata = [] put_data = request.data.copy() # convert the QueryDict to dict if isinstance(put_data, QueryDict): put_data = put_data.dict() try: resource.metadata.parse_for_bulk_update(put_data, metadata) hydroshare.update_science_metadata(pk=pk, metadata=metadata, user=request.user) except Exception as ex: error_msg = { 'resource': "Resource metadata update failed: %s, %s" % (ex.__class__, ex.message) } raise ValidationError(detail=error_msg) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) serializer = resource.metadata.serializer return Response(data=serializer.data, status=status.HTTP_202_ACCEPTED)
def test_publish_resource(self): # check status prior to publishing the resource self.assertFalse(self.res.raccess.published, msg='The resource is published') self.assertFalse(self.res.raccess.immutable, msg='The resource is frozen') self.assertFalse(self.res.doi, msg='doi is assigned') # there should not be published date type metadata element self.assertFalse( self.res.metadata.dates.filter(type='published').exists()) # publish resource - this is the api we are testing hydroshare.publish_resource(self.res.short_id) self.pub_res = hydroshare.get_resource_by_shortkey(self.res.short_id) # test publish state self.assertTrue(self.pub_res.raccess.published, msg='The resource is not published') # test frozen state self.assertTrue(self.pub_res.raccess.immutable, msg='The resource is not frozen') # test if doi is assigned self.assertTrue(self.pub_res.doi, msg='No doi is assigned with the published resource.') # there should now published date type metadata element self.assertTrue( self.pub_res.metadata.dates.filter(type='published').exists())
def transform_file(request, shortkey, *args, **kwargs): res = hydroshare.get_resource_by_shortkey(shortkey) if res.reference_type == 'soap': client = Client(res.url) response = client.service.GetValues(':'+res.data_site_code, ':'+res.variable_code, '', '', '') elif res.reference_type == 'rest': r = requests.get(res.url) response = str(r.text) waterml_1 = etree.XML(response) wml_string = etree.tostring(waterml_1) s = StringIO(wml_string) dom = etree.parse(s) module_dir = os.path.dirname(__file__) xsl_location = os.path.join(module_dir, "static/ref_ts/xslt/WaterML1_1_timeSeries_to_WaterML2.xsl") xslt = etree.parse(xsl_location) transform = etree.XSLT(xslt) newdom = transform(dom) d = datetime.date.today() date = '{0}_{1}_{2}'.format(d.month, d.day, d.year) xml_name = '{0}-{1}-{2}'.format(res.title.replace(" ", ""), date, 'wml_2_0.xml') with open(xml_name, 'wb') as f: f.write(newdom) xml_file = open(xml_name, 'r') ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0').delete() hydroshare.add_resource_files(res.short_id, xml_file) f = ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0')[0].resource_file data = { 'status_code': 200, 'xml_name': xml_name, 'xml_size': f.size, 'xml_link': f.url } os.remove(xml_name) # print(etree.tostring(newdom, pretty_print=True)) return json_or_jsonp(request, data)
def add_resource_view(request, shortkey, *args, **kwargs): agg = hydroshare.get_resource_by_shortkey(shortkey) resources = Resource.objects.filter(object_id=agg.id) agg_resources = {} for agg_res in resources: res = hydroshare.get_resource_by_shortkey(agg_res.resource_short_id) if res: short_id = res.short_id if agg_res.resource_description: description = agg_res.resource_description else: description = "None" agg_resources[res.title] = [short_id, description] context = {'agg': agg, 'agg_resources': agg_resources} return render(request, "pages/add-resource.html", context)
def test_get_resource_by_shortkey(self): # create a user to be used for creating the resource user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) # create a resource resource = GenericResource.objects.create( user=user_creator, title='My resource', creator=user_creator, last_changed_by=user_creator, doi='doi1000100010001' ) # do the test of the api self.assertEqual( resource, hydroshare.get_resource_by_shortkey(resource.short_id) )
def delete(self, request, pk): view_utils.authorize( request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE_ACCESS) keys = request.query_params.keys() user_access = UserAccess(user=request.user) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) if "user_id" in keys and "group_id" in keys: message = "Request cannot contain both a 'user_id' and a 'group_id' parameter." return Response(data={'error': message}, status=status.HTTP_400_BAD_REQUEST) if "user_id" in keys: user_to_remove = utils.user_from_id( request.query_params['user_id']) user_access.unshare_resource_with_user(resource, user_to_remove) return Response( data={'success': "Resource access privileges removed."}, status=status.HTTP_202_ACCEPTED) if "group_id" in keys: group_to_remove = utils.group_from_id( request.query_params['group_id']) user_access.unshare_resource_with_group(resource, group_to_remove) return Response( data={'success': "Resource access privileges removed."}, status=status.HTTP_202_ACCEPTED) message = "Request must contain a 'resource' ID as well as a 'user_id' or 'group_id'" return Response(data={'error': message}, status=status.HTTP_400_BAD_REQUEST)
def test_create_resource_with_two_files(self): raster = MyTemporaryUploadedFile(open(self.raster_file_path, 'rb'), name=self.raster_file_path, content_type='image/tiff', size=os.stat( self.raster_file_path).st_size) text = MyTemporaryUploadedFile(open(self.txt_file_path, 'r'), name=self.txt_file_path, content_type='text/plain', size=os.stat( self.txt_file_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(raster, text)) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEqual(res.resource_type, 'GenericResource') self.assertTrue(isinstance(res, GenericResource), type(res)) self.assertEqual(res.metadata.title.value, 'My Test resource') self.assertEquals(res.files.all().count(), 2) if res: res.delete()
def add_resource(request, shortkey, *args, **kwargs): frm = AddResourceForm(request.POST) if frm.is_valid(): agg = hydroshare.get_resource_by_shortkey(shortkey) if frm.cleaned_data.get('resource_permalink'): hs_res_permalink = frm.cleaned_data.get('resource_permalink') trash, new_short_id = os.path.split(hs_res_permalink[:-1]) hs_res = hydroshare.get_resource_by_shortkey(new_short_id) description = frm.cleaned_data.get('resource_description') or '' for res in Resource.objects.filter(object_id=agg.id): if res.resource_short_id == new_short_id: res.delete() Resource.objects.create(resource_short_id=hs_res.short_id or new_short_id, resource_description=description, content_object=agg) return HttpResponseRedirect('/my-resources/')
def put(self, request, pk): view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE_ACCESS) user_access = UserAccess(user=request.user) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) keys = request.data.keys() if "user_id" in keys and "group_id" in keys: return Response( data={ 'error': "Request cannot contain both a 'user_id' and a 'group_id' parameter." }, status=status.HTTP_400_BAD_REQUEST ) if "user_id" in keys and "privilege" in keys: if int(request.data['privilege']) in (1, 2, 3, 4): try: user_to_add = utils.user_from_id(request.data['user_id']) user_access.share_resource_with_user(resource, user_to_add, request.data['privilege']) return Response( data={'success': "Resource access privileges added."}, status=status.HTTP_202_ACCEPTED ) except Exception: return Response( data={'error': "This resource may not be shared with that user."}, status=status.HTTP_400_BAD_REQUEST ) if "group_id" in keys and "privilege" in keys: if int(request.data['privilege']) in (1, 2, 3, 4): group_to_add = utils.group_from_id(request.data['group_id']) try: user_access.share_resource_with_group(resource, group_to_add, request.data['privilege']) return Response( data={'success': "Resource access privileges added."}, status=status.HTTP_202_ACCEPTED ) except Exception: return Response( data={'error': "This group may not be added to any resources."}, status=status.HTTP_400_BAD_REQUEST ) message = "Request must contain a 'resource' ID as well as a 'user_id' or " \ "'group_id', and 'privilege' must be one of 1, 2, or 3." return Response( data={'error': message}, status=status.HTTP_400_BAD_REQUEST )
def repair_resource(resource, logger, stop_on_error=False, log_errors=True, echo_errors=False, return_errors=False): errors = [] ecount = 0 try: resource = get_resource_by_shortkey(resource.short_id, or_404=False) except BaseResource.DoesNotExist: msg = "Resource with id {} not found in Django Resources".format( resource.short_id) if log_errors: logger.error(msg) if echo_errors: print(msg) if return_errors: errors.append(msg) ecount = ecount + 1 return errors, ecount print("REPAIRING RESOURCE {}".format(resource.short_id)) # ingest any dangling iRODS files that you can # Do this before check because otherwise, errors get printed twice # TODO: This does not currently work properly for composite resources # if resource.resource_type == 'CompositeResource' or \ if resource.resource_type == 'GenericResource' or \ resource.resource_type == 'ModelInstanceResource' or \ resource.resource_type == 'ModelProgramResource': _, count = ingest_irods_files(resource, logger, stop_on_error=False, echo_errors=True, log_errors=False, return_errors=False) if count: print("... affected resource {} has type {}, title '{}'".format( resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace'))) _, count = check_irods_files(resource, stop_on_error=False, echo_errors=True, log_errors=False, return_errors=False, clean_irods=False, clean_django=True, sync_ispublic=True) if count: print("... affected resource {} has type {}, title '{}'".format( resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace')))
def put(self, request, pk): view_utils.authorize( request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE_ACCESS) user_access = UserAccess(user=request.user) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) keys = request.data.keys() if "user_id" in keys and "group_id" in keys: return Response(data={ 'error': "Request cannot contain both a 'user_id' and a 'group_id' parameter." }, status=status.HTTP_400_BAD_REQUEST) if "user_id" in keys and "privilege" in keys: if int(request.data['privilege']) in (1, 2, 3, 4): try: user_to_add = utils.user_from_id(request.data['user_id']) user_access.share_resource_with_user( resource, user_to_add, request.data['privilege']) return Response( data={'success': "Resource access privileges added."}, status=status.HTTP_202_ACCEPTED) except Exception: return Response(data={ 'error': "This resource may not be shared with that user." }, status=status.HTTP_400_BAD_REQUEST) if "group_id" in keys and "privilege" in keys: if int(request.data['privilege']) in (1, 2, 3, 4): group_to_add = utils.group_from_id(request.data['group_id']) try: user_access.share_resource_with_group( resource, group_to_add, request.data['privilege']) return Response( data={'success': "Resource access privileges added."}, status=status.HTTP_202_ACCEPTED) except Exception: return Response(data={ 'error': "This group may not be added to any resources." }, status=status.HTTP_400_BAD_REQUEST) message = "Request must contain a 'resource' ID as well as a 'user_id' or " \ "'group_id', and 'privilege' must be one of 1, 2, or 3." return Response(data={'error': message}, status=status.HTTP_400_BAD_REQUEST)
def test_create_resource_with_zipfile(self): # Make a zip file zip_path = os.path.join(self.tmp_dir, 'test.zip') with zipfile.ZipFile(zip_path, 'w') as zfile: zfile.write(self.raster_file_path) zfile.write(self.txt_file_path) # Create a resource with zipfile, do not un-pack payload = MyTemporaryUploadedFile(open(zip_path, 'rb'), name=zip_path, content_type='application/zip', size=os.stat(zip_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(payload, )) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEquals(res.files.all().count(), 1) # Create a resource with zipfile, un-pack payload2 = MyTemporaryUploadedFile(open(zip_path, 'rb'), name=zip_path, content_type='application/zip', size=os.stat(zip_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(payload2, ), unpack_file=True) pid = res.short_id res = get_resource_by_shortkey(pid) self.assertEquals(res.files.all().count(), 2) if res: res.delete()
def record(cls, session, name, value=None, resource=None, resource_id=None, rest=False, landing=False): if resource is None and resource_id is not None: try: resource = get_resource_by_shortkey(resource_id, or_404=False) except BaseResource.DoesNotExist: resource = None return Variable.objects.create(session=session, name=name, type=cls.encode_type(value), value=cls.encode(value), last_resource_id=resource_id, resource=resource, rest=rest, landing=landing)
def repair_resource(resource, logger, stop_on_error=False, log_errors=True, echo_errors=False, return_errors=False): errors = [] ecount = 0 try: resource = get_resource_by_shortkey(resource.short_id, or_404=False) except BaseResource.DoesNotExist: msg = "Resource with id {} not found in Django Resources".format(resource.short_id) if log_errors: logger.error(msg) if echo_errors: print(msg) if return_errors: errors.append(msg) ecount = ecount + 1 return errors, ecount print("REPAIRING RESOURCE {}".format(resource.short_id)) # ingest any dangling iRODS files that you can # Do this before check because otherwise, errors get printed twice # TODO: This does not currently work properly for composite resources # if resource.resource_type == 'CompositeResource' or \ if resource.resource_type == 'GenericResource' or \ resource.resource_type == 'ModelInstanceResource' or \ resource.resource_type == 'ModelProgramResource': _, count = ingest_irods_files(resource, logger, stop_on_error=False, echo_errors=True, log_errors=False, return_errors=False) if count: print("... affected resource {} has type {}, title '{}'" .format(resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace'))) _, count = check_irods_files(resource, stop_on_error=False, echo_errors=True, log_errors=False, return_errors=False, clean_irods=False, clean_django=True, sync_ispublic=True) if count: print("... affected resource {} has type {}, title '{}'" .format(resource.short_id, resource.resource_type, resource.title.encode('ascii', 'replace')))
def handle(self, *args, **options): resource_id = options['resource_id'] days = options['days'] n_users = options['n_users'] try: resource = get_resource_by_shortkey(resource_id, or_404=False) except BaseResource.DoesNotExist: print("resource '{}' not found".format(resource_id)) exit(1) recent = Variable.recent_users(resource, days=days, n_users=n_users) for v in recent: print("username={} last_access={}".format( v.username, v.last_accessed.strftime("%Y-%m-%d %H:%M:%S")))
def test_create_resource_with_zipfile(self): # Make a zip file zip_path = os.path.join(self.tmp_dir, 'test.zip') with zipfile.ZipFile(zip_path, 'w') as zfile: zfile.write(self.raster_file_path) zfile.write(self.txt_file_path) # Create a resource with zipfile, do not un-pack payload = MyTemporaryUploadedFile(open(zip_path, 'rb'), name=zip_path, content_type='application/zip', size=os.stat(zip_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(payload,)) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEquals(res.files.all().count(), 1) # Create a resource with zipfile, un-pack payload2 = MyTemporaryUploadedFile(open(zip_path, 'rb'), name=zip_path, content_type='application/zip', size=os.stat(zip_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(payload2,), unpack_file=True) pid = res.short_id res = get_resource_by_shortkey(pid) self.assertEquals(res.files.all().count(), 2) if res: res.delete()
def instantiate_timestamp_range(start, end): """ instantiate a range of Variable requests """ events = 0 ids = 0 print("instantiating ids from -{} days to -{} days from now".format( start, end)) for v in Variable.objects.filter( timestamp__gte=datetime.now() - timedelta(start), timestamp__lte=datetime.now() - timedelta(end)): events = events + 1 value = v.get_value() if v.name in RESOURCE_RE: m = RESOURCE_RE[v.name].search(value) if (m and m.group(1)): resource_id = m.group(1) if (resource_id is not None): v.last_resource_id = resource_id try: resource = get_resource_by_shortkey(resource_id, or_404=False) v.resource = resource except BaseResource.DoesNotExist: pass # print("{} for '{}' ".format(resource_id, value)) ids = ids + 1 if ids % 1000 == 0: print("{} of {}".format(ids, events)) if v.name == 'visit': # for visits, classify kind of visit if LANDING_RE.search(value): v.landing = True else: v.landing = False if REST_RE.search(value): if INTERNAL_RE.search(value): v.rest = False else: v.rest = True else: v.rest = False v.save() # else: # print("NONE for '{}'".format(value)) # else: # print("NONE for '{}'".format(value)) print("resource ids found for {} of {} events".format(ids, events))
def test_create_resource_with_file(self): raster = open(self.raster_file_path) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(raster, )) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEqual(res.resource_type, 'GenericResource') self.assertTrue(isinstance(res, GenericResource), type(res)) self.assertEqual(res.metadata.title.value, 'My Test resource') self.assertEquals(res.files.all().count(), 1) if res: res.delete()
def get_queryset(self, pk, user): resource = hydroshare.get_resource_by_shortkey(shortkey=pk) if user in resource.raccess.owners: querysets = (UserResourcePrivilege.objects.filter( resource=resource), GroupResourcePrivilege.objects.filter( resource=resource)) else: user_groups = Group.objects.filter(gaccess__g2ugp__user=user) querysets = (UserResourcePrivilege.objects.filter( resource=resource, user=user), GroupResourcePrivilege.objects.filter( resource=resource, group__in=user_groups)) return querysets
def get_queryset(self, pk, user): resource = hydroshare.get_resource_by_shortkey(shortkey=pk) if user in resource.raccess.owners: querysets = ( UserResourcePrivilege.objects.filter(resource=resource), GroupResourcePrivilege.objects.filter(resource=resource) ) else: user_groups = Group.objects.filter(gaccess__g2ugp__user=user) querysets = ( UserResourcePrivilege.objects.filter(resource=resource, user=user), GroupResourcePrivilege.objects.filter(resource=resource, group__in=user_groups) ) return querysets
def test_create_resource_with_file(self): raster = open(self.raster_file_path) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(raster,)) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEqual(res.resource_type, 'GenericResource') self.assertTrue(isinstance(res, GenericResource), type(res)) self.assertEqual(res.metadata.title.value, 'My Test resource') self.assertEquals(res.files.all().count(), 1) if res: res.delete()
def test_get_resource_by_shortkey(self): # create a user to be used for creating the resource self.user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[]) # create a resource resource = hydroshare.create_resource('GenericResource', self.user_creator, 'My Test Resource') # do the test of the api self.assertEqual( resource, hydroshare.get_resource_by_shortkey(resource.short_id))
def test_publish_resource(self): # check status prior to publishing the resource self.assertFalse( self.res.raccess.published, msg='The resource is published' ) self.assertFalse( self.res.raccess.immutable, msg='The resource is frozen' ) self.assertIsNone( self.res.doi, msg='doi is assigned' ) # there should not be published date type metadata element self.assertFalse(self.res.metadata.dates.filter(type='published').exists()) # publish resource - this is the api we are testing hydroshare.publish_resource(self.res.short_id) self.pub_res = hydroshare.get_resource_by_shortkey(self.res.short_id) # test publish state self.assertTrue( self.pub_res.raccess.published, msg='The resource is not published' ) # test frozen state self.assertTrue( self.pub_res.raccess.immutable, msg='The resource is not frozen' ) # test if doi is assigned self.assertIsNotNone( self.pub_res.doi, msg='No doi is assigned with the published resource.' ) # there should now published date type metadata element self.assertTrue(self.pub_res.metadata.dates.filter(type='published').exists())
def test_delete_resource(self): new_res = resource.create_resource( 'GenericResource', self.user, 'My Test Resource' ) pid = new_res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertTrue(type(res) == GenericResource, type(res)) self.assertTrue(res.title == 'My Test Resource') self.assertTrue(res.created.strftime('%m/%d/%Y %H:%M') == res.updated.strftime('%m/%d/%Y %H:%M') ) self.assertTrue(res.created.strftime('%m/%d/%Y') == dt.datetime.today().strftime('%m/%d/%Y')) self.assertTrue(res.creator == self.user) self.assertTrue(res.short_id is not None, 'Short ID has not been created!') self.assertTrue(res.bags.exists(), 'Bagit has not been created!')
def test_delete_resource(self): new_res = resource.create_resource('GenericResource', self.user, 'My Test Resource') pid = new_res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertTrue(type(res) == GenericResource, type(res)) self.assertTrue(res.title == 'My Test Resource') self.assertTrue( res.created.strftime('%m/%d/%Y %H:%M') == res.updated.strftime( '%m/%d/%Y %H:%M')) self.assertTrue( res.created.strftime('%m/%d/%Y') == dt.datetime.today().strftime( '%m/%d/%Y')) self.assertTrue(res.creator == self.user) self.assertTrue(res.short_id is not None, 'Short ID has not been created!') self.assertTrue(res.bags.exists(), 'Bagit has not been created!')
def test_get_resource_by_shortkey(self): # create a user to be used for creating the resource user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[]) # create a resource resource = GenericResource.objects.create(user=user_creator, title='My resource', creator=user_creator, last_changed_by=user_creator, doi='doi1000100010001') # do the test of the api self.assertEqual( resource, hydroshare.get_resource_by_shortkey(resource.short_id))
def add_dublin_core(request, page): from dublincore import models as dc class DCTerm(forms.ModelForm): class Meta: model = dc.QualifiedDublinCoreElement fields = ['term', 'content'] cm = page.get_content_model() try: abstract = cm.dublin_metadata.filter(term='AB').first().content except: abstract = None resources = Resource.objects.filter(object_id=cm.id) agg_resources = {} for agg_res in resources: res = hydroshare.get_resource_by_shortkey(agg_res.resource_short_id) if res: short_id = res.short_id if agg_res.resource_description: description = agg_res.resource_description else: description = "None" agg_resources[res.title] = [short_id, description] return { 'dublin_core': [t for t in cm.dublin_metadata.all().exclude(term='AB').exclude(term='DM').exclude(term='DC').exclude(term='DTS').exclude(term='T')], 'abstract' : abstract, 'short_id' : cm.short_id, 'agg_resources': agg_resources, 'dcterm_frm': DCTerm(), 'bag': cm.bags.first(), 'users': User.objects.all(), 'groups': Group.objects.all(), 'owners': set(cm.owners.all()), 'view_users': set(cm.view_users.all()), 'view_groups': set(cm.view_groups.all()), 'edit_users': set(cm.edit_users.all()), 'edit_groups': set(cm.edit_groups.all()), }
def test_create_resource_with_two_files(self): raster = MyTemporaryUploadedFile(open(self.raster_file_path, 'rb'), name=self.raster_file_path, content_type='image/tiff', size=os.stat(self.raster_file_path).st_size) text = MyTemporaryUploadedFile(open(self.txt_file_path, 'r'), name=self.txt_file_path, content_type='text/plain', size=os.stat(self.txt_file_path).st_size) res = resource.create_resource('GenericResource', self.user, 'My Test resource', files=(raster, text)) pid = res.short_id # get the resource by pid res = get_resource_by_shortkey(pid) self.assertEqual(res.resource_type, 'GenericResource') self.assertTrue(isinstance(res, GenericResource), type(res)) self.assertEqual(res.metadata.title.value, 'My Test resource') self.assertEquals(res.files.all().count(), 2) if res: res.delete()
def go_for_tools(request, shortkey, user, tooltype, *args, **kwargs): frm = GoForToolsForm(request.POST) if frm.is_valid(): my_string = frm.cleaned_data.get('my_str') url_base = "http://www.example.com" # just in case the resource doesn't exist res = hydroshare.get_resource_by_shortkey(shortkey) if res: if res.files.first(): f_url = str(res.files.first().resource_file) f_name = f_url.split("/")[-1] # choose the last part of the url for the file, which is it's name else: f_name = "none-no-resource-file" url_base = res.metadata.url_base.first() or "http://www.example.com" # if there isn't a url_base else: f_name = "none-no-resource-provided" myParameters = { "res_id" : shortkey, "user" : user, "tool_type" : tooltype, "file_name": f_name } myURL = "%s?%s" % (url_base, urllib.urlencode(myParameters)) return redirect(myURL)
def create_resource_aggregation(request, *args, **kwargs): frm = CreateAggregationForm(request.POST) if frm.is_valid(): dcterms = [ { 'term': 'T', 'content': frm.cleaned_data['title']}, { 'term': 'AB', 'content': frm.cleaned_data['abstract'] or frm.cleaned_data['title']}, { 'term': 'DTS', 'content': now().isoformat()} ] for cn in frm.cleaned_data['contributors'].split(','): cn = cn.strip() dcterms.append({'term' : 'CN', 'content' : cn}) for cr in frm.cleaned_data['creators'].split(','): cr = cr.strip() dcterms.append({'term' : 'CR', 'content' : cr}) agg = hydroshare.create_resource( resource_type='ResourceAggregation', owner=request.user, title=frm.cleaned_data['title'], keywords=[k.strip() for k in frm.cleaned_data['keywords'].split(',')] if frm.cleaned_data['keywords'] else None, dublin_metadata=dcterms, content=frm.cleaned_data['abstract'] or frm.cleaned_data['title'] ) if frm.cleaned_data.get('resource_permalink'): hs_res_permalink = frm.cleaned_data.get('resource_permalink') trash, hs_res_shortkey = os.path.split(hs_res_permalink[:-1]) hs_res = hydroshare.get_resource_by_shortkey(hs_res_shortkey) description = frm.cleaned_data.get('resource_description') or '' Resource.objects.create(resource_short_id=hs_res.short_id or hs_res_shortkey, resource_description=description, content_object=agg) if frm.cleaned_data.get('more'): data = {'agg_short_id': agg.short_id} return json_or_jsonp(request, data) return HttpResponseRedirect('/my-resources/') # FIXME this will eventually need to change
def test_get_resource_by_shortkey(self): # create a user to be used for creating the resource self.user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) # create a resource resource = hydroshare.create_resource( 'GenericResource', self.user_creator, 'My Test Resource' ) # do the test of the api self.assertEqual( resource, hydroshare.get_resource_by_shortkey(resource.short_id) )
def test_publish_resource(self): # publish resource hydroshare.publish_resource(self.res.short_id) self.pub_res = hydroshare.get_resource_by_shortkey(self.res.short_id) # test publish state self.assertTrue( self.pub_res.published_and_frozen, msg='The resoruce is not published and frozen' ) # test frozen state self.assertTrue( self.pub_res.frozen, msg='The resource is not frozen' ) # test if resource has edit users self.assertListEqual( list(self.pub_res.edit_users.all()), [], msg='edit users list is not empty' ) # test if resource has edit groups self.assertListEqual( list(self.pub_res.edit_groups.all()), [], msg='edit groups is not empty' ) # test if doi is assigned self.assertIsNotNone( self.pub_res.doi, msg='No doi is assigned with the published resource.' )
def test_add_files(self): user = User.objects.create_user('shaun', '*****@*****.**', 'shaun6745') #create user #create files n1 = "test.txt" n2 = "test2.txt" n3 = "test3.txt" open(n1, "w").close() #files are created open(n2, "w").close() open(n3, "w").close() myfile = open(n1, "r") #files are opened as 'read-only' myfile1 = open(n2, "r") myfile2 = open(n3, "r") res1 = create_resource('GenericResource', user, 'res1') #create resource #delete all resource files for created resource res1.files.all().delete() #add files add_resource_files(res1.short_id, myfile, myfile1, myfile2) #add each file of resource to list res1 = get_resource_by_shortkey(res1.short_id) l = [] for f in res1.files.all(): l.append(f.resource_file.name.split('/')[-1]) #check if the file name is in the list of files self.assertTrue(n1 in l, "file 1 has not been added") self.assertTrue(n2 in l, "file 2 has not been added") self.assertTrue(n3 in l, "file 3 has not been added")
def test_publish_resource(self): # publish resource hydroshare.publish_resource(self.res.short_id) self.pub_res = hydroshare.get_resource_by_shortkey(self.res.short_id) # test publish state self.assertTrue(self.pub_res.published_and_frozen, msg='The resoruce is not published and frozen') # test frozen state self.assertTrue(self.pub_res.frozen, msg='The resource is not frozen') # test if resource has edit users self.assertListEqual(list(self.pub_res.edit_users.all()), [], msg='edit users list is not empty') # test if resource has edit groups self.assertListEqual(list(self.pub_res.edit_groups.all()), [], msg='edit groups is not empty') # test if doi is assigned self.assertIsNotNone( self.pub_res.doi, msg='No doi is assigned with the published resource.')
def generate_files(request, shortkey, *args, **kwargs): res = hydroshare.get_resource_by_shortkey(shortkey) ts, csv_link, csv_size, xml_link, xml_size = {}, '', '', '', '' try: if res.reference_type == 'rest': ts = ts_utils.time_series_from_service(res.url, res.reference_type) else: ts = ts_utils.time_series_from_service(res.url, res.reference_type, site_name_or_code=res.data_site_code, variable_code=res.variable_code) vals = ts['values'] version = ts['wml_version'] d = datetime.date.today() date = '{0}_{1}_{2}'.format(d.month, d.day, d.year) file_base = '{0}-{1}'.format(res.title.replace(" ", ""), date) csv_name = '{0}.{1}'.format(file_base, 'csv') if version == '1': xml_end = 'wml_1' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) elif version == '2.0': xml_end = 'wml_2_0' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) for_csv = [] for k, v in vals.items(): t = (k, v) for_csv.append(t) ResourceFile.objects.filter(object_id=res.pk).delete() with open(csv_name, 'wb') as csv_file: w = csv.writer(csv_file) w.writerow([res.title]) var = '{0}({1})'.format(ts['variable_name'], ts['units']) w.writerow(['time', var]) for r in for_csv: w.writerow(r) with open(xml_name, 'wb') as xml_file: xml_file.write(ts['time_series']) csv_file = open(csv_name, 'r') xml_file = open(xml_name, 'r') files = [csv_file, xml_file] hydroshare.add_resource_files(res.short_id, csv_file, xml_file) create_bag(res) os.remove(csv_name) os.remove(xml_name) files = ResourceFile.objects.filter(object_id=res.pk) for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if xml_end in str(f.resource_file): xml_link = f.resource_file.url xml_size = f.resource_file.size status_code = 200 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_name': csv_name, 'xml_name': xml_name, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # successfully generated new files except Exception: # most likely because the server is unreachable files = ResourceFile.objects.filter(object_id=res.pk) xml_file = None for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if str(f.resource_file).endswith('.xml'): xml_link = f.resource_file.url xml_size = f.resource_file.size xml_file = f.resource_file if xml_file is None: status_code = 404 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, did not find old ones xml_doc = open(str(xml_file), 'r').read() root = etree.XML(xml_doc) os.remove(str(xml_file)) version = ts_utils.get_version(root) if version == '1': ts = ts_utils.parse_1_0_and_1_1(root) status_code = 200 elif version =='2.0': ts = ts_utils.parse_2_0(root) status_code = 200 else: status_code = 503 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, return old ones
def get(self, request, pk): view_utils.authorize(request, pk, needed_permission=ACTION_TO_AUTHORIZE.VIEW_METADATA) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) serializer = resource.metadata.serializer return Response(data=serializer.data, status=status.HTTP_200_OK)
def test(self): """ Test view for resource depicts output of various integrity checking scripts """ # print("TESTING {}".format(self.short_id)) # leave this for debugging try: self.resource = get_resource_by_shortkey(self.short_id, or_404=False) except BaseResource.DoesNotExist: print("{} does not exist in Django".format(self.short_id)) return # skip federated resources if not configured to handle these if self.resource.is_federated and not settings.REMOTE_USE_IRODS: msg = "check_resource: skipping check of federated resource {} in unfederated mode"\ .format(self.resource.short_id) print(msg) istorage = self.resource.get_irods_storage() if not istorage.exists(self.resource.root_path): self.label() print(" root path {} does not exist in iRODS".format(self.resource.root_path)) print(" ... resource {} has type {} and title {}" .format(self.resource.short_id, self.resource.resource_type, self.resource.title.encode('ascii', 'replace'))) return for a in ('bag_modified', 'isPublic', 'resourceType', 'quotaUserName'): value = self.check_avu(a) if a == 'resourceType' and value is not None and value != self.resource.resource_type: self.label() print(" AVU resourceType is {}, should be {}".format(value.encode('ascii', 'replace'), self.resource.resource_type)) if a == 'isPublic' and value is not None and value != self.resource.raccess.public: self.label() print(" AVU isPublic is {}, but public is {}".format(value.encode('ascii', 'replace'), self.resource.raccess.public)) irods_issues, irods_errors = check_irods_files(self.resource, log_errors=False, echo_errors=False, return_errors=True) if irods_errors: self.label() print(" iRODS errors:") for e in irods_issues: print(" {}".format(e)) if self.resource.resource_type == 'CompositeResource': logical_issues = [] for res_file in self.resource.files.all(): file_type = get_logical_file_type(res=self.resource, user=None, file_id=res_file.pk, fail_feedback=False) if not res_file.has_logical_file and file_type is not None: msg = "check_resource: file {} does not have required logical file {}"\ .format(res_file.storage_path.encode('ascii', 'replace'), file_type.__name__) logical_issues.append(msg) elif res_file.has_logical_file and file_type is None: msg = "check_resource: logical file for {} has type {}, not needed"\ .format(res_file.storage_path.encode('ascii', 'replace'), type(res_file.logical_file).__name__) logical_issues.append(msg) elif res_file.has_logical_file and file_type is not None and \ not isinstance(res_file.logical_file, file_type): msg = "check_resource: logical file for {} has type {}, should be {}"\ .format(res_file.storage_path.encode('ascii', 'replace'), type(res_file.logical_file).__name__, file_type.__name__) logical_issues.append(msg) if logical_issues: self.label() print(" Logical file errors:") for e in logical_issues: print(" {}".format(e))
def test(self): """ Test view for resource depicts output of various integrity checking scripts """ # print("TESTING {}".format(self.short_id)) # leave this for debugging try: self.resource = get_resource_by_shortkey(self.short_id, or_404=False) except BaseResource.DoesNotExist: print("{} does not exist in Django".format(self.short_id)) return # skip federated resources if not configured to handle these if self.resource.is_federated and not settings.REMOTE_USE_IRODS: msg = "check_resource: skipping check of federated resource {} in unfederated mode"\ .format(self.resource.short_id) print(msg) istorage = self.resource.get_irods_storage() if not istorage.exists(self.resource.root_path): self.label() print(" root path {} does not exist in iRODS".format(self.resource.root_path)) print(" ... resource {} has type {} and title {}" .format(self.resource.short_id, self.resource.resource_type, self.resource.title)) return for a in ('bag_modified', 'isPublic', 'resourceType', 'quotaUserName'): value = self.check_avu(a) if a == 'resourceType' and value is not None and value != self.resource.resource_type: self.label() print((" AVU resourceType is {}, should be {}".format(value, self.resource.resource_type))) if a == 'isPublic' and value is not None and value != self.resource.raccess.public: self.label() print((" AVU isPublic is {}, but public is {}".format(value, self.resource.raccess.public))) irods_issues, irods_errors = check_irods_files(self.resource, log_errors=False, echo_errors=False, return_errors=True) if irods_errors: self.label() print(" iRODS errors:") for e in irods_issues: print(" {}".format(e)) if self.resource.resource_type == 'CompositeResource': logical_issues = [] for res_file in self.resource.files.all(): file_type = get_logical_file_type(res=self.resource, user=None, file_id=res_file.pk, fail_feedback=False) if not res_file.has_logical_file and file_type is not None: msg = "check_resource: file {} does not have required logical file {}"\ .format(res_file.storage_path, file_type.__name__) logical_issues.append(msg) elif res_file.has_logical_file and file_type is None: msg = "check_resource: logical file for {} has type {}, not needed"\ .format(res_file.storage_path, type(res_file.logical_file).__name__) logical_issues.append(msg) elif res_file.has_logical_file and file_type is not None and \ not isinstance(res_file.logical_file, file_type): msg = "check_resource: logical file for {} has type {}, should be {}"\ .format(res_file.storage_path, type(res_file.logical_file).__name__, file_type.__name__) logical_issues.append(msg) if logical_issues: self.label() print(" Logical file errors:") for e in logical_issues: print(" {}".format(e))
def put(self, request, pk): # Update science metadata view_utils.authorize( request, pk, needed_permission=ACTION_TO_AUTHORIZE.EDIT_RESOURCE) metadata = [] put_data = request.data.copy() keys_to_update = put_data.keys() try: if 'title' in keys_to_update: metadata.append({"title": {"value": put_data.pop('title')}}) if 'creators' in keys_to_update: for creator in put_data.pop('creators'): metadata.append({"creator": creator}) if 'contributors' in keys_to_update: for contributor in put_data.pop('contributors'): metadata.append({"contributor": contributor}) if 'coverages' in keys_to_update: for coverage in put_data.pop('coverages'): metadata.append({"coverage": coverage}) if 'dates' in keys_to_update: for date in put_data.pop('dates'): metadata.append({"date": date}) if 'description' in keys_to_update: metadata.append( {"description": { "abstract": put_data.pop('description') }}) if 'language' in keys_to_update: metadata.append( {"language": { "code": put_data.pop('language') }}) if 'rights' in keys_to_update: metadata.append( {"rights": { "statement": put_data.pop('rights') }}) if 'sources' in keys_to_update: for source in put_data.pop('sources'): metadata.append({"source": source}) if 'subjects' in keys_to_update: for subject in put_data.pop('subjects'): metadata.append({"subject": {"value": subject['value']}}) hydroshare.update_science_metadata(pk=pk, metadata=metadata, user=request.user) except Exception as ex: error_msg = { 'resource': "Resource metadata update failed: %s, %s" % (ex.__class__, ex.message) } raise ValidationError(detail=error_msg) resource = hydroshare.get_resource_by_shortkey(shortkey=pk) serializer = CoreMetaDataSerializer(resource.metadata) return Response(data=serializer.data, status=status.HTTP_202_ACCEPTED)
def get(self, request, pk, pathname): """ Get a resource file's metadata. ## Parameters * `id` - alphanumeric uuid of the resource, i.e. cde01b3898c94cdab78a2318330cf795 * `pathname` - The pathname of the file to get these ## Returns ``` { "keywords": [ "keyword1", "keyword2" ], "spatial_coverage": { "units": "Decimal degrees", "east": -84.0465, "north": 49.6791, "name": "12232", "projection": "WGS 84 EPSG:4326" }, "extra_metadata": { "extended1": "one" }, "temporal_coverage": { "start": "2018-02-22", "end": "2018-02-24" }, "title": "File Metadata Title", "logical_file": {} } ``` """ try: resource_file = hydroshare.get_resource_file(pk, pathname) logical_file = resource_file.logical_file metadata = resource_file.metadata except ObjectDoesNotExist: # Backwards compatibility for file_id try: resource_file = ResourceFile.objects.get(id=pathname) logical_file = resource_file.logical_file metadata = resource_file.metadata except Exception: # is it a folder? resource = hydroshare.get_resource_by_shortkey(pk) dir_path = pk + os.path.join("/data/contents/", pathname) logical_file = resource.get_folder_aggregation_object(dir_path) metadata = None title = logical_file.dataset_name \ if logical_file else "" keywords = metadata.keywords \ if metadata else [] spatial_coverage = metadata.spatial_coverage.value \ if metadata and metadata.spatial_coverage else {} extra_metadata = metadata.extra_metadata \ if metadata else {} temporal_coverage = metadata.temporal_coverage.value if \ metadata and metadata.temporal_coverage else {} extra_data = logical_file.metadata.dict() \ if logical_file else {} # TODO: How to leverage serializer for this? return Response({ "title": title, "keywords": keywords, "spatial_coverage": spatial_coverage, "extra_metadata": extra_metadata, "temporal_coverage": temporal_coverage, "logical_file": extra_data })