def transform_file(request, shortkey, *args, **kwargs): res = hydroshare.get_resource_by_shortkey(shortkey) if res.reference_type == 'soap': client = Client(res.url) response = client.service.GetValues(':'+res.data_site_code, ':'+res.variable_code, '', '', '') elif res.reference_type == 'rest': r = requests.get(res.url) response = str(r.text) waterml_1 = etree.XML(response) wml_string = etree.tostring(waterml_1) s = StringIO(wml_string) dom = etree.parse(s) module_dir = os.path.dirname(__file__) xsl_location = os.path.join(module_dir, "static/ref_ts/xslt/WaterML1_1_timeSeries_to_WaterML2.xsl") xslt = etree.parse(xsl_location) transform = etree.XSLT(xslt) newdom = transform(dom) d = datetime.date.today() date = '{0}_{1}_{2}'.format(d.month, d.day, d.year) xml_name = '{0}-{1}-{2}'.format(res.title.replace(" ", ""), date, 'wml_2_0.xml') with open(xml_name, 'wb') as f: f.write(newdom) xml_file = open(xml_name, 'r') ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0').delete() hydroshare.add_resource_files(res.short_id, xml_file) f = ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0')[0].resource_file data = { 'status_code': 200, 'xml_name': xml_name, 'xml_size': f.size, 'xml_link': f.url } os.remove(xml_name) # print(etree.tostring(newdom, pretty_print=True)) return json_or_jsonp(request, data)
def test_move_to_folder_basic(self): group, _ = Group.objects.get_or_create(name='Hydroshare Author') user = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) resource = hydroshare.create_resource( 'GenericResource', user, 'test resource', ) resource.save() open('myfile.txt', "w").close() file = open('myfile.txt', 'r') hydroshare.add_resource_files(resource.short_id, file) create_folder(resource.short_id, "data/contents/test_folder") move_to_folder(user, resource.short_id, src_paths=['data/contents/myfile.txt'], tgt_path="data/contents/test_folder", validate_move=True) folder_contents = list_folder(resource.short_id, "data/contents/test_folder") self.assertTrue(['myfile.txt'] in folder_contents) resource.delete()
def test_unfederated_folder_path_setting(self): """ an unfederated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live shortpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath)) # non-existent files should raise error otherpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file2.txt") with self.assertRaises(ValidationError): resfile.path_is_acceptable(otherpath) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt") # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object with self.assertRaises(ValidationError): resfile.set_storage_path(otherpath) # should not change self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # TODO: how to eliminate this particular error. # dumbpath = 'x' + shortpath # dumbpath = self.res.short_id + "file1.txt" # clean up after folder test # ResourceFile.remove_folder(self.res, 'foo', self.user) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_move_to_folder_basic(self): group, _ = Group.objects.get_or_create(name='Hydroshare Author') user = hydroshare.create_account('*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[]) resource = hydroshare.create_resource( 'GenericResource', user, 'test resource', ) resource.save() open('myfile.txt', "w").close() file = open('myfile.txt', 'rb') hydroshare.add_resource_files(resource.short_id, file) create_folder(resource.short_id, "data/contents/test_folder") move_to_folder(user, resource.short_id, src_paths=['data/contents/myfile.txt'], tgt_path="data/contents/test_folder", validate_move=True) folder_contents = list_folder(resource.short_id, "data/contents/test_folder") self.assertTrue(['myfile.txt'] in folder_contents) resource.delete()
def migrate_tif_file(apps, schema_editor): # create a vrt file from tif file for each of the Raster Resources log = logging.getLogger() istorage = IrodsStorage() for res in RasterResource.objects.all(): try: if len(res.files.all()) == 1: res_file = res.files.all().first() vrt_file_path = create_vrt_file(res_file.resource_file) if os.path.isfile(vrt_file_path): files = (UploadedFile(file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path))) hydroshare.add_resource_files(res.short_id, files) bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) print("Deleted bag for resource ID:" + str(res.short_id)) resource_modified(res, res.creator) log.info('Tif file conversion to VRT successful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) else: log.error('Tif file conversion to VRT unsuccessful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) if os.path.exists(vrt_file_path): shutil.rmtree(os.path.dirname(vrt_file_path)) except: pass
def test_resource_file_put(self): myfile = open(self.filename, 'w') myfile.write('hello world!\n') myfile.close() myfile = open(self.filename, 'r') res1 = hydroshare.create_resource('GenericResource', self.user, 'res1') hydroshare.add_resource_files(res1.short_id, myfile) mynewfile = open(self.filename, 'w') mynewfile.write('anyone there?\n') mynewfile.close() mynewfile = open(self.filename, 'r') url = self.url_proto.format(res1.short_id, self.filename) put_data = { 'resource_file': mynewfile } resp = self.api_client.put(url, data=put_data) self.assertHttpAccepted(resp) resp = self.api_client.get(url) self.assertIn(resp.status_code, [201, 200])
def test_resource_file_put(self): myfile = open(self.filename, 'w') myfile.write('hello world!\n') myfile.close() myfile = open(self.filename, 'r') res1 = hydroshare.create_resource('GenericResource', self.user, 'res1') hydroshare.add_resource_files(res1.short_id, myfile) mynewfile = open(self.filename, 'w') mynewfile.write('anyone there?\n') mynewfile.close() mynewfile = open(self.filename, 'r') url = self.url_proto.format(res1.short_id, self.filename) put_data = {'resource_file': mynewfile} resp = self.api_client.put(url, data=put_data) self.assertHttpAccepted(resp) resp = self.api_client.get(url) self.assertIn(resp.status_code, [201, 200])
def generate_files(shortkey, ts): ''' creates the calls the make files fxn and adds the files as resource files called by view: create_ref_time_series, update_files :param shortkey: res shortkey :param ts: parsed time series dict ''' res = hydroshare.get_resource_by_shortkey(shortkey) if ts is None: #called by update_files in view if res.metadata.referenceURLs.all()[0].type == 'rest': ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type) else: ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type, site_name_or_code=res.metadata.sites.all()[0].code, variable_code=res.metadata.variables.all()[0].code) tempdir = None try: tempdir = tempfile.mkdtemp() files = make_files(res, tempdir, ts) hydroshare.add_resource_files(res.short_id, *files) except Exception as e: raise e finally: if tempdir is not None: shutil.rmtree(tempdir)
def setUp(self): super(TestTickets, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) self.res = hydroshare.create_resource( 'GenericResource', self.user, 'test resource', ) # create a file self.test_file_name1 = 'file1.txt' self.file_name_list = [self.test_file_name1, ] # put predictable contents into the file test_file = open(self.test_file_name1, 'w') test_file.write("Test text file in file1.txt") test_file.close() self.test_file_1 = open(self.test_file_name1, 'r') # add one file to the resource: necessary so data/contents is created. hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
def setUp(self): super(TestNewVersionResource, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be versioned self.owner = hydroshare.create_account( '*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[] ) # create a user who is NOT the owner of the resource to be versioned self.nonowner = hydroshare.create_account( '*****@*****.**', username='******', first_name='nonowner_firstname', last_name='nonowner_lastname', superuser=False, groups=[] ) # create a generic resource self.res_generic = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource' ) test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'r') self.test_file2 = open('test2.txt', 'r') hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2) # create a raster resource that represents a specific resource type raster_file = 'hs_core/tests/data/cea.tif' temp_dir = tempfile.mkdtemp() temp_raster_file = os.path.join(temp_dir, 'cea.tif') shutil.copy(raster_file, temp_raster_file) self.raster_obj = open(temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.res_raster = hydroshare.create_resource( resource_type='RasterResource', owner=self.owner, title='Test Raster Resource', files=files, metadata=[] ) # call the post creation process here for the metadata to be # extracted utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_resource_operations_in_user_zone(self): super(TestUserZoneIRODSFederation, self).assert_federated_irods_available() # test adding files from federated user zone to an empty resource # created in hydroshare zone res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in HydroShare Zone') self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) hydroshare.add_resource_files( res.short_id, source_names=[fed_test_file1_full_path, fed_file2_full_path]) # test resource has two files self.assertEqual(res.files.all().count(), 2, msg="Number of content files is not equal to 2") user_path = '/{zone}/home/testuser/'.format( zone=settings.HS_USER_IRODS_ZONE) file_list = [] for f in res.files.all(): file_list.append(os.path.basename(f.storage_path)) self.assertTrue( self.file_one in file_list, msg='file 1 has not been added in the resource in hydroshare zone') self.assertTrue( self.file_two in file_list, msg='file 2 has not been added in the resource in hydroshare zone') # test original file in user test zone still exist after adding it to the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) self.assertTrue(self.irods_storage.exists(user_path + self.file_two)) # test replication of this resource to user zone even if the bag_modified AVU for this # resource is wrongly set to False when the bag for this resource does not exist and # need to be recreated res.setAVU('bag_modified', 'false') hydroshare.resource.replicate_resource_bag_to_user_zone( self.user, res.short_id) self.assertTrue(self.irods_storage.exists(user_path + res.short_id + '.zip'), msg='replicated resource bag is not in the user zone') # test resource deletion hydroshare.resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test to make sure original files still exist after resource deletion self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
def test_bag_ingestion(self): from hs_core.views.utils import unzip_file def normalize_metadata(metadata_str): """Prepares metadata string to match resource id and hydroshare url of original""" return metadata_str\ .replace(current_site_url(), "http://www.hydroshare.org")\ .replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1") # create empty resource res = resource.create_resource( 'CompositeResource', self.user, 'My Test Resource' ) full_paths = {} files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'), name="test_resource_metadata_files.zip")] add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths) unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True, overwrite=True, auto_aggregate=True, ingest_metadata=True) def compare_metadatas(new_metadata_str, original_metadata_file): original_graph = Graph() with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f: original_graph = original_graph.parse(data=f.read()) new_graph = Graph() new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str)) # remove modified date, they'll never match subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified) new_graph.remove((subject, None, None)) subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified) original_graph.remove((subject, None, None)) for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph): self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original") res.refresh_from_db() compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml") compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(), "test_meta.xml") compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(), "asdf/asdf_meta.xml") compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(), "watersheds_meta.xml") compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(), "logan_meta.xml") compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(), "SWE_time_meta.xml") compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(), "msf_version.refts_meta.xml") compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(), "ODM2_Multi_Site_One_Variable_meta.xml")
def add_resource_file(self, pk, filename=None): authorize(self.request, pk, edit=True) if filename: rf = self.request.FILES.values()[0] rf.name = filename f = hydroshare.add_resource_files(pk, rf) return HttpResponse(f.resource_file.url, content_type='text/plain') else: fs = hydroshare.add_resource_files(pk, self.request.FILES.values()) return json_or_jsonp(self.request, { f.name: f.resource_file.url for f in fs})
def add_resource_file(self, pk, filename=None): authorize(self.request, pk, edit=True) if filename: rf = self.request.FILES.values()[0] rf.name = filename f = hydroshare.add_resource_files(pk, rf) return HttpResponse(f.resource_file.url, content_type='text/plain') else: fs = hydroshare.add_resource_files(pk, self.request.FILES.values()) return json_or_jsonp(self.request, {f.name: f.resource_file.url for f in fs})
def test_resource_file_get(self): myfile = open(self.filename, 'w') myfile.write('hello world!\n') myfile.close() myfile = open(self.filename, 'r') res1 = hydroshare.create_resource('GenericResource', self.user, 'res1') hydroshare.add_resource_files(res1.short_id, myfile) url = self.url_proto.format(res1.short_id, self.filename) resp = self.api_client.get(url) self.assertIn(resp.status_code, [201, 200])
def test_post_add_files_to_resource(self): # test: add all files for f in ResourceFile.objects.filter(object_id=self.resGeoFeature.id): f.resource_file.delete() f.delete() # add files first files = [] target = 'hs_geographic_feature_resource/tests/gis.osm_adminareas_v06_with_folder.zip' files.append(UploadedFile(file=open(target, 'r'), name='gis.osm_adminareas_v06_with_folder.zip')) hydroshare.utils.resource_file_add_pre_process(self.resGeoFeature, files, self.user,) hydroshare.add_resource_files(self.resGeoFeature.short_id, *files) self.assertEqual(ResourceFile.objects.filter(object_id=self.resGeoFeature.id).count(), 5) # remove .prj for res_f_obj in ResourceFile.objects.filter(object_id=self.resGeoFeature.id): del_f_fullname = res_f_obj.resource_file.name.lower() del_f_fullname = del_f_fullname[del_f_fullname.rfind('/')+1:] del_f_name, del_f_ext = os.path.splitext(del_f_fullname) if del_f_ext == ".prj": hydroshare.delete_resource_file(self.resGeoFeature.short_id, res_f_obj.id, self.user) self.assertEqual(ResourceFile.objects.filter (object_id=self.resGeoFeature.id).count(), 4) for res_f_obj_2 in ResourceFile.objects.filter(object_id=self.resGeoFeature.id): del_f_fullname = res_f_obj_2.resource_file.name.lower() del_f_fullname = del_f_fullname[del_f_fullname.rfind('/')+1:] del_f_name, del_f_ext = os.path.splitext(del_f_fullname) self.assertNotEqual(del_f_ext, ".prj") originalcoverage_obj = self.resGeoFeature.metadata.\ originalcoverage.all().first() self.assertEqual(originalcoverage_obj.projection_string, UNKNOWN_STR) self.assertEqual(self.resGeoFeature.metadata.coverages.all().count(), 0) # add .prj add_files = [] target = 'hs_geographic_feature_resource/' \ 'tests/gis.osm_adminareas_v06/gis.osm_adminareas_v06.prj' add_files.append(UploadedFile(file=open(target, 'r'), name='gis.osm_adminareas_v06.prj')) hydroshare.add_resource_files(self.resGeoFeature.short_id, *add_files) self.assertEqual(ResourceFile.objects.filter(object_id=self.resGeoFeature.id).count(), 5) geofeature_post_add_files_to_resource_handler(sender=GeographicFeatureResource, resource=self.resGeoFeature, files=add_files, validate_files={'are_files_valid': True, 'message': ''}) originalcoverage_obj = self.resGeoFeature.metadata.originalcoverage.all().first() self.assertNotEqual(originalcoverage_obj.projection_string, UNKNOWN_STR)
def test_resource_file_folder_oprs(self): # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add the three files to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, self.test_file_2, self.test_file_3) # resource should has only three files at this point self.assertEqual(self.res.files.all().count(), 3, msg="resource file count didn't match") super(TestResourceFileFolderOprsAPI, self).resource_file_oprs() # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def add_files_to_resource(self, files_to_add, upload_folder=None): files_to_upload = [] for fl in files_to_add: file_to_upload = UploadedFile(file=open(fl, 'rb'), name=os.path.basename(fl)) files_to_upload.append(file_to_upload) added_resource_files = add_resource_files(self.composite_resource.short_id, *files_to_upload, folder=upload_folder) return added_resource_files
def test_pdf_classification(self): """ files with recognized extensions are classified properly """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add one file to the resource test_file_handle = open(self.filenames[0], 'r') hydroshare.add_resource_files(self.res.short_id, test_file_handle) self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") types = get_content_types(self.res) self.assertTrue(is_equal_to_as_set(types[0], ['Composite', self.content_types[0]])) self.assertTrue(is_equal_to_as_set(types[1], [])) # no left-over extensions # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def setUp(self): super(TestGetResourceFile, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = hydroshare.create_account( '*****@*****.**', username='******', first_name='test', last_name='user', superuser=False) self.res = hydroshare.create_resource( resource_type='CompositeResource', owner=self.user, title='Test File') test_file = open('test1.txt', 'w') test_file.write("Test text file in test1.txt") test_file.close() self.file = open('test1.txt', 'rb') hydroshare.add_resource_files(self.res.short_id, self.file)
def setUp(self): super(TestGetResourceFile, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = hydroshare.create_account('*****@*****.**', username='******', first_name='Tian', last_name='Gan', superuser=False, groups=[]) self.res = hydroshare.create_resource( resource_type='GenericResource', owner=self.user, title='My resource', metadata=[], ) open('myfile.txt', "w").close() self.file = open('myfile.txt', 'r') hydroshare.add_resource_files(self.res.short_id, self.file)
def setUp(self): self.user = hydroshare.create_account( '*****@*****.**', username='******', first_name='Tian', last_name='Gan', superuser=False, groups=[] ) self.res = GenericResource.objects.create( user=self.user, title='My resource', creator=self.user, last_changed_by=self.user, ) self.file = open('myfile.txt', "w") self.file = open('myfile.txt','r') hydroshare.add_resource_files(self.res.short_id, self.file)
def migrate_tif_file(apps, schema_editor): # create a vrt file from tif file for each of the Raster Resources log = logging.getLogger() istorage = IrodsStorage() for res in RasterResource.objects.all(): try: if len(res.files.all()) == 1: res_file = res.files.all().first() vrt_file_path = create_vrt_file(res_file.resource_file) if os.path.isfile(vrt_file_path): files = (UploadedFile( file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path))) hydroshare.add_resource_files(res.short_id, files) bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) print("Deleted bag for resource ID:" + str(res.short_id)) resource_modified(res, res.creator) log.info( 'Tif file conversion to VRT successful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) else: log.error( 'Tif file conversion to VRT unsuccessful for resource:ID:{} ' 'Title:{}'.format(res.short_id, res.metadata.title.value)) if os.path.exists(vrt_file_path): shutil.rmtree(os.path.dirname(vrt_file_path)) except: pass
def test_weird_extensions(self): """ files with unrecognized extensions are classified properly """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add all files to the resource for f in self.weird_filenames: test_file_handle = open(f, 'r') hydroshare.add_resource_files(self.res.short_id, test_file_handle) self.assertEqual(self.res.files.all().count(), 2, msg="resource file count didn't match") types = get_content_types(self.res) # no extensions match content types self.assertTrue(is_equal_to_as_set(types[0], ['Composite', 'Generic Data'])) # extensions are flagged as not matching self.assertTrue(is_equal_to_as_set(types[1], self.weird_extensions)) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def setUp(self): super(TestHideOldVersions, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be versioned self.owner = hydroshare.create_account('*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[]) # create a generic resource self.version0 = hydroshare.create_resource( resource_type='CompositeResource', owner=self.owner, title='Test Composite Resource') test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'rb') self.test_file2 = open('test2.txt', 'rb') hydroshare.add_resource_files(self.version0.short_id, self.test_file1, self.test_file2) # make one version self.version1 = hydroshare.create_empty_resource( self.version0.short_id, self.owner) self.version1 = hydroshare.create_new_version_resource( self.version0, self.version1, self.owner) # and then make a version of that self.version2 = hydroshare.create_empty_resource( self.version1.short_id, self.owner) self.version2 = hydroshare.create_new_version_resource( self.version1, self.version2, self.owner)
def setUp(self): super(TestGetResourceFile, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') self.user = hydroshare.create_account( '*****@*****.**', username='******', first_name='Tian', last_name='Gan', superuser=False, groups=[] ) self.res = hydroshare.create_resource( resource_type='GenericResource', owner=self.user, title='My resource', metadata=[], ) open('myfile.txt', "w").close() self.file = open('myfile.txt', 'r') hydroshare.add_resource_files(self.res.short_id, self.file)
def test_file_extension_classification(self): """ files with recognized extensions are classified properly """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add all files to the resource for f in self.filenames: test_file_handle = open(f, 'r') hydroshare.add_resource_files(self.res.short_id, test_file_handle) self.assertEqual(self.res.files.all().count(), 5, msg="resource file count didn't match") types = get_content_types(self.res) self.assertTrue(is_equal_to_as_set( types[0], ['Composite', 'Document', 'Presentation', 'Spreadsheet', 'Jupyter Notebook'])) self.assertTrue(is_equal_to_as_set(types[1], [])) # no left-over extensions # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_resource_operations_in_user_zone(self): # only do federation testing when REMOTE_USE_IRODS is True and irods docker containers # are set up properly if not super(TestUserZoneIRODSFederation, self).is_federated_irods_available(): return # test resource creation and "move" option in federated user zone fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted) res = resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in User Zone', source_names=[fed_test_file_full_path], move=True) self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) user_path = '/{zone}/home/testuser/'.format( zone=settings.HS_USER_IRODS_ZONE) self.assertEqual(res.resource_federation_path, fed_path) # test original file in user test zone is removed after resource creation # since True is used for move when creating the resource self.assertFalse( self.irods_storage.exists(user_path + self.file_to_be_deleted)) # test resource file deletion res.files.all().delete() self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") # test add multiple files and 'copy' option in federated user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) hydroshare.add_resource_files( res.short_id, source_names=[fed_test_file1_full_path, fed_test_file2_full_path], move=False) # test resource has two files self.assertEqual(res.files.all().count(), 2, msg="Number of content files is not equal to 2") file_list = [] for f in res.files.all(): file_list.append(f.storage_path.split('/')[-1]) self.assertTrue( self.file_one in file_list, msg='file 1 has not been added in the resource in user zone') self.assertTrue( self.file_two in file_list, msg='file 2 has not been added in the resource in user zone') # test original two files in user test zone still exist after adding them to the resource # since False is used for move when creating the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) self.assertTrue(self.irods_storage.exists(user_path + self.file_two)) # test resource deletion resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test create new version resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user) new_res = hydroshare.create_new_version_resource( ori_res, new_res, self.user) # only need to test file-related attributes # ensure new versioned resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new versioned resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up resource.delete_resource(new_res.short_id) resource.delete_resource(ori_res.short_id) # test copy resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user, action='copy') new_res = hydroshare.copy_resource(ori_res, new_res) # only need to test file-related attributes # ensure new copied resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new copied resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up resource.delete_resource(new_res.short_id) resource.delete_resource(ori_res.short_id) # test folder operations in user zone fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three) self.res = resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[ fed_file1_full_path, fed_file2_full_path, fed_file3_full_path ], move=False) # make sure self.res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(self.res.resource_federation_path, fed_path) # resource should has only three files at this point self.assertEqual(self.res.files.all().count(), 3, msg="resource file count didn't match") self.file_name_list = [self.file_one, self.file_two, self.file_three] super(TestUserZoneIRODSFederation, self).resource_file_oprs() # delete resources to clean up resource.delete_resource(self.res.short_id) # test adding files from federated user zone to an empty resource # created in hydroshare zone res = resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in HydroShare Zone') self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) hydroshare.add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) # test resource has one file self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") file_list = [] for f in res.files.all(): file_list.append(os.path.basename(f.storage_path)) self.assertTrue( self.file_one in file_list, msg='file 1 has not been added in the resource in hydroshare zone') # test original file in user test zone still exist after adding it to the resource # since 'copy' is used for fed_copy_or_move when adding the file to the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) # test replication of this resource to user zone hydroshare.replicate_resource_bag_to_user_zone(self.user, res.short_id) self.assertTrue(self.irods_storage.exists(user_path + res.short_id + '.zip'), msg='replicated resource bag is not in the user zone') # test resource deletion resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test to make sure original file still exist after resource deletion self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_resource_operations_in_user_zone(self): super(TestUserZoneIRODSFederation, self).assert_federated_irods_available() # test resource creation and "move" option in federated user zone fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted) res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in User Zone', source_names=[fed_test_file_full_path], move=True ) self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) user_path = '/{zone}/home/testuser/'.format(zone=settings.HS_USER_IRODS_ZONE) self.assertEqual(res.resource_federation_path, fed_path) # test original file in user test zone is removed after resource creation # since True is used for move when creating the resource file_path_name = user_path + self.file_to_be_deleted self.assertFalse(self.irods_storage.exists(file_path_name)) # test django_irods CopyFiles() with an iRODS resource name being passed in # as input parameter to verify the file gets copied to the pass-in iRODS resource istorage = res.get_irods_storage() src_path = os.path.join(res.root_path, 'data', 'contents', self.file_to_be_deleted) dest_path = file_path_name istorage.copyFiles(src_path, dest_path, settings.HS_IRODS_LOCAL_ZONE_DEF_RES) # assert file did get copied over self.assertTrue(self.irods_storage.exists(file_path_name)) stdout = self.irods_storage.session.run("ils", None, "-l", file_path_name)[0].split() # assert copied file gets written to the iRODS resource being passed into copyFiles() call self.assertEqual(stdout[2], settings.HS_IRODS_LOCAL_ZONE_DEF_RES) # test resource file deletion res.files.all().delete() self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") # test add multiple files and 'copy' option in federated user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) hydroshare.add_resource_files( res.short_id, source_names=[fed_test_file1_full_path, fed_test_file2_full_path], move=False) # test resource has two files self.assertEqual(res.files.all().count(), 2, msg="Number of content files is not equal to 2") file_list = [] for f in res.files.all(): file_list.append(f.storage_path.split('/')[-1]) self.assertTrue(self.file_one in file_list, msg='file 1 has not been added in the resource in user zone') self.assertTrue(self.file_two in file_list, msg='file 2 has not been added in the resource in user zone') # test original two files in user test zone still exist after adding them to the resource # since False is used for move when creating the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) self.assertTrue(self.irods_storage.exists(user_path + self.file_two)) # test resource deletion hydroshare.resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test create new version resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False ) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user) new_res = hydroshare.create_new_version_resource(ori_res, new_res, self.user) # only need to test file-related attributes # ensure new versioned resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new versioned resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up hydroshare.resource.delete_resource(new_res.short_id) hydroshare.resource.delete_resource(ori_res.short_id) # test copy resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False ) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user, action='copy') new_res = hydroshare.copy_resource(ori_res, new_res) # only need to test file-related attributes # ensure new copied resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new copied resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up hydroshare.resource.delete_resource(new_res.short_id) hydroshare.resource.delete_resource(ori_res.short_id) # test folder operations in user zone fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three) self.res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_file1_full_path, fed_file2_full_path, fed_file3_full_path], move=False ) # make sure self.res is created in federated user zone fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(self.res.resource_federation_path, fed_path) # resource should has only three files at this point self.assertEqual(self.res.files.all().count(), 3, msg="resource file count didn't match") self.file_name_list = [self.file_one, self.file_two, self.file_three] super(TestUserZoneIRODSFederation, self).resource_file_oprs() # delete resources to clean up hydroshare.resource.delete_resource(self.res.short_id) # test adding files from federated user zone to an empty resource # created in hydroshare zone res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in HydroShare Zone' ) self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) hydroshare.add_resource_files( res.short_id, source_names=[fed_test_file1_full_path], move=False) # test resource has one file self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") file_list = [] for f in res.files.all(): file_list.append(os.path.basename(f.storage_path)) self.assertTrue(self.file_one in file_list, msg='file 1 has not been added in the resource in hydroshare zone') # test original file in user test zone still exist after adding it to the resource # since 'copy' is used for fed_copy_or_move when adding the file to the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) # test replication of this resource to user zone even if the bag_modified AVU for this # resource is wrongly set to False when the bag for this resource does not exist and # need to be recreated res.setAVU('bag_modified', 'false') hydroshare.resource.replicate_resource_bag_to_user_zone(self.user, res.short_id) self.assertTrue(self.irods_storage.exists(user_path + res.short_id + '.zip'), msg='replicated resource bag is not in the user zone') # test resource deletion hydroshare.resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test to make sure original file still exist after resource deletion self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_federated_folder_path_logic(self): """ a federated file in a subfolder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") ResourceFile.create_folder(self.res, 'foo') # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo') # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] self.assertEqual(resfile.resource_file.name, os.path.join(self.res.short_id, "data", "contents", "foo", "file1.txt")) self.assertEqual(resfile.file_folder, "foo") # cheat: set a fake federated path to test path logic fedpath = "/myzone/home/myuser" self.res.resource_federation_path = fedpath self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path('foo/file1.txt', test_exists=False) # determine where that file should live shortpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file1.txt") self.assertEqual(shortpath, resfile.storage_path) # this should result in an exact path resfile.set_storage_path(shortpath, test_exists=False) self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False)) # non-existent files should raise error otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file2.txt") resfile.path_is_acceptable(otherpath, test_exists=False) # This won't work because federation path is fake. # with self.assertRaises(ValidationError): # resfile.path_is_acceptable(otherpath, test_exists=True) # try setting to an unqualified name; should qualify it resfile.set_storage_path("foo/file1.txt", test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath, test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, "foo") self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object resfile.set_storage_path(otherpath, test_exists=False) # conclusion: unfederate the resource self.res.resource_federation_path = "" self.res.save() resfile.content_object.refresh_from_db() resfile.set_storage_path("foo/file1.txt", test_exists=False) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip, # and file1.txt (2) file_cnt = res.files.all().count() self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " + str(file_cnt) + " != 5") # remove all files except the zippped file remove_folder(user, res.short_id, 'data/contents/sub_test_dir') remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1') # Now resource should contain two files: file3_new.txt sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def migrate_tif_file(apps, schema_editor): log = logging.getLogger() istorage = IrodsStorage() copy_res_fail = [] vrt_update_fail = [] vrt_update_success = [] meta_update_fail = [] meta_update_success = [] # start migration for each raster resource that has raster files for res in RasterResource.objects.all(): if res.files.all(): # copy all the resource files to temp dir try: temp_dir = tempfile.mkdtemp() for res_file in res.files.all(): shutil.copy(res_file.resource_file.file.name, os.path.join(temp_dir, os.path.basename(res_file.resource_file.name))) vrt_file_path = [os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.vrt' == f[-4:]].pop() except Exception as e: log.exception(e.message) copy_res_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value)) continue # update vrt file if the raster resource that has a single tif file try: if len(os.listdir(temp_dir)) == 2: # create new vrt file tif_file_path = [os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.tif' == f[-4:]].pop() with open(os.devnull, 'w') as fp: subprocess.Popen(['gdal_translate', '-of', 'VRT', tif_file_path, vrt_file_path], stdout=fp, stderr=fp).wait() # remember to add .wait() # modify the vrt file contents tree = ET.parse(vrt_file_path) root = tree.getroot() for element in root.iter('SourceFilename'): element.attrib['relativeToVRT'] = '1' tree.write(vrt_file_path) # delete vrt res file for f in res.files.all(): if 'vrt' == f.resource_file.name[-3:]: f.resource_file.delete() f.delete() # add new vrt file to resource new_file = UploadedFile(file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path)) hydroshare.add_resource_files(res.short_id, new_file) # update the bag bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) vrt_update_success.append('{}:{}'.format(res.short_id,res.metadata.title.value)) except Exception as e: log.exception(e.message) vrt_update_fail.append('{}:{}'.format(res.short_id,res.metadata.title.value)) # update the metadata for the band information of all the raster resources try: meta_updated = False # extract meta ori_dir = os.getcwd() os.chdir(temp_dir) res_md_dict = raster_meta_extract.get_raster_meta_dict(vrt_file_path) os.chdir(ori_dir) shutil.rmtree(temp_dir) # update band information metadata in django if res_md_dict['band_info']: for i, band_meta in res_md_dict['band_info'].items(): band_obj = res.metadata.bandInformation.filter(name='Band_{}'.format(i)).first() if band_obj: res.metadata.update_element('bandInformation', band_obj.id, maximumValue=band_meta['maximumValue'], minimumValue=band_meta['minimumValue'], noDataValue=band_meta['noDataValue'], ) meta_updated = True # update the bag if meta is updated if meta_updated: bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) meta_update_success.append('{}:{}'.format(res.short_id, res.metadata.title.value)) except Exception as e: log.exception(e.message) meta_update_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value)) # Print migration results print 'Copy resource to temp folder failure: Number: {} List: {}'.format(len(copy_res_fail), copy_res_fail) print 'VRT file update success: Number: {} List{}'.format(len(vrt_update_success), vrt_update_success) print 'VRT file update fail: Number: {} List{}'.format(len(vrt_update_fail), vrt_update_fail) print 'Meta update success: Number: {} List {}'.format(len(meta_update_success), meta_update_success) print 'Meta update fail: Number: {} List {}'.format(len(meta_update_fail), meta_update_fail)
def test_federated_root_path_logic(self): """ a federated file path in the root folder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # cheat: set a fake federated path to test path logic oldfedpath = self.res.resource_federation_path oldpath = resfile.storage_path # intentionally break path logic by setting an unused federation path fedpath = "/myzone/home/myuser" self.res.resource_federation_path = fedpath self.res.save() # must load changes into resfile from self.res before setting storage path resfile.content_object.refresh_from_db() resfile.set_storage_path('file1.txt', test_exists=False) self.assertEqual(self.res.resource_federation_path, fedpath) self.assertEqual(resfile.storage_path, get_path(resfile, 'file1.txt')) # determine where that file should live; THIS IS FAKE shortpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file1.txt") # intentionally break the resource file path resfile.set_storage_path(shortpath, test_exists=False) self.assertEqual(shortpath, resfile.storage_path) self.assertEqual(resfile.file_folder, '') self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False)) otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file2.txt") resfile.path_is_acceptable(otherpath, test_exists=False) # non-existent files should raise error # This won't work because federation path is fake # with self.assertRaises(ValidationError): # resfile.path_is_acceptable(otherpath, test_exists=True) # try setting to an unqualified name; should qualify it resfile.set_storage_path("file1.txt", test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, '') self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath, test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, '') self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object resfile.set_storage_path(otherpath, test_exists=False) # conclusion: strip off federation path self.res.resource_federation_path = oldfedpath self.res.save() # must load changes into resfile from self.res before setting storage path resfile.content_object.refresh_from_db() resfile.set_storage_path(oldpath, test_exists=False) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_get_revisions(self): # create a user to be used for creating the resource user_creator = hydroshare.create_account( "*****@*****.**", username="******", first_name="Creator_FirstName", last_name="Creator_LastName", superuser=False, groups=[], ) resource_changed_by = hydroshare.create_account( "*****@*****.**", username="******", first_name="Pabitra", last_name="Dash", superuser=False, groups=[], ) # create a resource resource = hydroshare.create_resource("GenericResource", user_creator, "My resource") # test that we have only one revision at this point - this is the api call we are testing res_revisions = hydroshare.get_revisions(resource.short_id) self.assertEqual(len(res_revisions), 1) # set the resource last changed by a different user - to a create another revision of the resource hydroshare.utils.resource_modified(resource, resource_changed_by) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 2 revisions self.assertEqual(len(res_revisions), 2) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) # test that each resource revision has the same resource id for bags in res_revisions: self.assertEqual(resource.id, bags.object_id) # add a file to the resource to generate another revision of the resource # create a file original_file_name = "original.txt" original_file = open(original_file_name, "w") original_file.write("original text") original_file.close() original_file = open(original_file_name, "r") # add the file to the resource hydroshare.add_resource_files(resource.short_id, original_file) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 3 revisions self.assertEqual(len(res_revisions), 3) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp) self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp) # delete the file in the resource to create another revision of the resource hydroshare.delete_resource_file(resource.short_id, original_file_name) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 4 revisions self.assertEqual(len(res_revisions), 4)
def test_unfederated_root_path_checks(self): """ an unfederated file in the root folder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") self.res.check_irods_files(stop_on_error=True) # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1) # should succeed without errors self.res.check_irods_files(stop_on_error=True) # cleaning should not change anything self.res.check_irods_files(stop_on_error=True, log_errors=False, return_errors=True, clean_irods=True, clean_django=True, sync_ispublic=True) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live shortpath = os.path.join(self.res.short_id, "data", "contents", "file1.txt") self.assertEqual(resfile.file_folder, None) self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath)) # now try to intentionally corrupt it resfile.set_short_path("fuzz.txt") # should raise exception with self.assertRaises(ValidationError): self.res.check_irods_files(stop_on_error=True) # now don't raise exception and read error errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS')) self.assertTrue(errors[1].endswith( 'data/contents/file1.txt in iRODs does not exist in Django')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # now try to clean it up errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False, clean_irods=True, clean_django=True) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)')) self.assertTrue(errors[1].endswith( 'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # now check should succeed errors, ecount = self.res.check_irods_files(stop_on_error=True, log_errors=False) self.assertEqual(ecount, 0) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_get_revisions(self): url = 'hsapi/revisions/{0}/'.format(self.res.short_id) resp = self.api_client.get(url) res_revisions = self.deserialize(resp) self.assertValidJSONResponse(resp) self.assertEqual(len(res_revisions), 1) self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions) resource_changed_by = hydroshare.create_account( '*****@*****.**', username='******', first_name='User1_FirstName', last_name='User1_LastName' ) hydroshare.utils.resource_modified(self.res, resource_changed_by) resp = self.api_client.get(url) res_revisions = self.deserialize(resp) self.assertValidJSONResponse(resp) self.assertEqual(len(res_revisions), 2) self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) # test that each resource revision has the same resource id for bags in res_revisions: self.assertEqual(self.res.id, bags.object_id) # add a file to the resource to generate another revision of the resource # create a file original_file_name = 'original.txt' original_file = open(original_file_name, 'w') original_file.write("original text") original_file.close() original_file = open(original_file_name, 'r') # add the file to the resource hydroshare.add_resource_files(self.res.short_id, original_file) resp = self.api_client.get(url) res_revisions = self.deserialize(resp) self.assertValidJSONResponse(resp) # test that we now have 3 revisions self.assertEqual(len(res_revisions), 3) self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp) self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp) # delete the file in the resource to create another revision of the resource hydroshare.delete_resource_file(self.res.short_id, original_file_name) resp = self.api_client.get(url) res_revisions = self.deserialize(resp) self.assertValidJSONResponse(resp) self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions) # test that we now have 4 revisions self.assertEqual(len(res_revisions), 4)
def setUp(self): super(TestCopyResource, self).setUp() self.group, _ = Group.objects.get_or_create(name='Hydroshare Author') # create a user who is the owner of the resource to be copied self.owner = hydroshare.create_account( '*****@*****.**', username='******', first_name='owner_firstname', last_name='owner_lastname', superuser=False, groups=[] ) # create a user who is NOT the owner of the resource to be copied self.nonowner = hydroshare.create_account( '*****@*****.**', username='******', first_name='nonowner_firstname', last_name='nonowner_lastname', superuser=False, groups=[] ) # create a generic resource self.res_generic = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource' ) test_file1 = open('test1.txt', 'w') test_file1.write("Test text file in test1.txt") test_file1.close() test_file2 = open('test2.txt', 'w') test_file2.write("Test text file in test2.txt") test_file2.close() self.test_file1 = open('test1.txt', 'r') self.test_file2 = open('test2.txt', 'r') hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2) # create a generic empty resource with one license that prohibits derivation statement = 'This resource is shared under the Creative Commons Attribution-NoDerivs CC ' \ 'BY-ND.' url = 'http://creativecommons.org/licenses/by-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata ) # create a generic empty resource with another license that prohibits derivation statement = 'This resource is shared under the Creative Commons ' \ 'Attribution-NoCommercial-NoDerivs CC BY-NC-ND.' url = 'http://creativecommons.org/licenses/by-nc-nd/4.0/' metadata = [] metadata.append({'rights': {'statement': statement, 'url': url}}) self.res_generic_lic_nc_nd = hydroshare.create_resource( resource_type='GenericResource', owner=self.owner, title='Test Generic Resource', metadata=metadata ) # create a raster resource that represents a specific resource type raster_file = 'hs_core/tests/data/cea.tif' temp_dir = tempfile.mkdtemp() self.temp_raster_file = os.path.join(temp_dir, 'cea.tif') shutil.copy(raster_file, self.temp_raster_file) self.raster_obj = open(self.temp_raster_file, 'r') files = [UploadedFile(file=self.raster_obj, name='cea.tif')] self.res_raster = hydroshare.create_resource( resource_type='RasterResource', owner=self.owner, title='Test Raster Resource', files=files, metadata=[] ) # call the post creation process here for the metadata to be # extracted utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_federated_root_path_logic(self): """ a federated file path in the root folder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # cheat: set a fake federated path to test path logic oldfedpath = self.res.resource_federation_path oldpath = resfile.storage_path # intentionally break path logic by setting an unused federation path fedpath = "/myzone/home/myuser" self.res.resource_federation_path = fedpath self.res.save() # must load changes into resfile from self.res before setting storage path resfile.content_object.refresh_from_db() resfile.set_storage_path('file1.txt', test_exists=False) self.assertEqual(self.res.resource_federation_path, fedpath) self.assertEqual(resfile.storage_path, get_path(resfile, 'file1.txt')) # determine where that file should live; THIS IS FAKE shortpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file1.txt") # intentionally break the resource file path resfile.set_storage_path(shortpath, test_exists=False) self.assertEqual(shortpath, resfile.storage_path) self.assertEqual(resfile.file_folder, None) self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False)) otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file2.txt") resfile.path_is_acceptable(otherpath, test_exists=False) # non-existent files should raise error # This won't work because federation path is fake # with self.assertRaises(ValidationError): # resfile.path_is_acceptable(otherpath, test_exists=True) # try setting to an unqualified name; should qualify it resfile.set_storage_path("file1.txt", test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, None) self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to what it is already resfile.set_storage_path(shortpath, test_exists=False) # should match computed path self.assertEqual(resfile.file_folder, None) self.assertEqual(resfile.storage_path, shortpath) # now try to change that path to a good path to a non-existent object resfile.set_storage_path(otherpath, test_exists=False) # conclusion: strip off federation path self.res.resource_federation_path = oldfedpath self.res.save() # must load changes into resfile from self.res before setting storage path resfile.content_object.refresh_from_db() resfile.set_storage_path(oldpath, test_exists=False) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_update_resource_file(self): # create a user to be used for creating the resource user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) # create a resource without any owner resource = GenericResource.objects.create( user=user_creator, title='My resource', creator=user_creator, last_changed_by=user_creator, doi='doi1000100010001' ) # resource should not have any files at this point self.assertEqual(resource.files.all().count(), 0, msg="resource file count didn't match") # create a file original_file_name = 'original.txt' original_file = open(original_file_name, 'w') original_file.write("original text") original_file.close() original_file = open(original_file_name, 'r') # add the file to the resource added_files = hydroshare.add_resource_files(resource.short_id, original_file) # resource should have only one file at this point self.assertEqual(len(added_files), 1) self.assertEqual(resource.files.all().count(), 1, msg="resource file count didn't match") self.assertIn( original_file_name, [os.path.basename(f.resource_file.name) for f in resource.files.all()], msg= '%s is not one of the resource files.' % original_file_name ) # create a file that will be used to update the original file -1st update new_file_name = 'update.txt' # file has a different name from the file that we will be updating new_file = open(new_file_name, 'w') new_file_data = 'data in new file' new_file.write(new_file_data) new_file.close() new_file = open(new_file_name, 'r') # this is the api call we are testing rf = hydroshare.update_resource_file(resource.short_id, original_file_name, new_file) # test if the file name matches self.assertEqual(os.path.basename(rf.resource_file.name), new_file_name, msg="resource file name didn't match") # since we are updating a file the number of files in the resource needs to be still 1 self.assertEqual(resource.files.all().count(), 1, msg="resource file count didn't match") # test if the content of the file matches resource_file = hydroshare.get_resource_file(resource.short_id, new_file_name) self.assertEqual(resource_file.read(), new_file_data, msg="resource file content didn't match") # reset the original resource file name for 2nd time resource file update original_file_name = new_file_name # create a file that will be used to update the resource file - 2nd update new_file_name = 'update.txt' # file has the same name as the file that we will be updating new_file = open(new_file_name, 'w') new_file_data = 'data in new file' new_file.write(new_file_data) new_file.close() new_file = open(new_file_name, 'r') # this is the api call we are testing rf = hydroshare.update_resource_file(resource.short_id, original_file_name, new_file) # test if the file name matches self.assertEqual(os.path.basename(rf.resource_file.name), new_file_name, msg="{0} != {1}".format(os.path.basename(rf.resource_file.name), new_file_name)) # exception ObjectDoesNotExist should be raised if resource does not have a file # for the given file name (file_not_in_resource.txt) to update self.assertRaises( ObjectDoesNotExist, lambda: hydroshare.update_resource_file(resource.short_id, 'file_not_in_resource.txt', new_file) )
def migrate_tif_file(apps, schema_editor): log = logging.getLogger() istorage = IrodsStorage() copy_res_fail = [] vrt_update_fail = [] vrt_update_success = [] meta_update_fail = [] meta_update_success = [] # start migration for each raster resource that has raster files for res in RasterResource.objects.all(): if res.files.all(): # copy all the resource files to temp dir try: temp_dir = tempfile.mkdtemp() for res_file in res.files.all(): shutil.copy( res_file.resource_file.file.name, os.path.join( temp_dir, os.path.basename(res_file.resource_file.name))) vrt_file_path = [ os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.vrt' == f[-4:] ].pop() except Exception as e: log.exception(e.message) copy_res_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value)) continue # update vrt file if the raster resource that has a single tif file try: if len(os.listdir(temp_dir)) == 2: # create new vrt file tif_file_path = [ os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.tif' == f[-4:] ].pop() with open(os.devnull, 'w') as fp: subprocess.Popen( [ 'gdal_translate', '-of', 'VRT', tif_file_path, vrt_file_path ], stdout=fp, stderr=fp).wait() # remember to add .wait() # modify the vrt file contents tree = ET.parse(vrt_file_path) root = tree.getroot() for element in root.iter('SourceFilename'): element.attrib['relativeToVRT'] = '1' tree.write(vrt_file_path) # delete vrt res file for f in res.files.all(): if 'vrt' == f.resource_file.name[-3:]: f.resource_file.delete() f.delete() # add new vrt file to resource new_file = UploadedFile( file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path)) hydroshare.add_resource_files(res.short_id, new_file) # update the bag bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) vrt_update_success.append('{}:{}'.format( res.short_id, res.metadata.title.value)) except Exception as e: log.exception(e.message) vrt_update_fail.append('{}:{}'.format( res.short_id, res.metadata.title.value)) # update the metadata for the band information of all the raster resources try: meta_updated = False # extract meta ori_dir = os.getcwd() os.chdir(temp_dir) res_md_dict = raster_meta_extract.get_raster_meta_dict( vrt_file_path) os.chdir(ori_dir) shutil.rmtree(temp_dir) # update band information metadata in django if res_md_dict['band_info']: for i, band_meta in res_md_dict['band_info'].items(): band_obj = res.metadata.bandInformation.filter( name='Band_{}'.format(i)).first() if band_obj: res.metadata.update_element( 'bandInformation', band_obj.id, maximumValue=band_meta['maximumValue'], minimumValue=band_meta['minimumValue'], noDataValue=band_meta['noDataValue'], ) meta_updated = True # update the bag if meta is updated if meta_updated: bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id) if istorage.exists(bag_name): # delete the resource bag as the old bag is not valid istorage.delete(bag_name) resource_modified(res, res.creator) meta_update_success.append('{}:{}'.format( res.short_id, res.metadata.title.value)) except Exception as e: log.exception(e.message) meta_update_fail.append('{}:{}'.format( res.short_id, res.metadata.title.value)) # Print migration results print 'Copy resource to temp folder failure: Number: {} List: {}'.format( len(copy_res_fail), copy_res_fail) print 'VRT file update success: Number: {} List{}'.format( len(vrt_update_success), vrt_update_success) print 'VRT file update fail: Number: {} List{}'.format( len(vrt_update_fail), vrt_update_fail) print 'Meta update success: Number: {} List {}'.format( len(meta_update_success), meta_update_success) print 'Meta update fail: Number: {} List {}'.format( len(meta_update_fail), meta_update_fail)
def test_resource_operations_in_user_zone(self): super(TestUserZoneIRODSFederation, self).assert_federated_irods_available() # test resource creation and "move" option in federated user zone fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted) res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in User Zone', source_names=[fed_test_file_full_path], move=True) self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) user_path = '/{zone}/home/testuser/'.format( zone=settings.HS_USER_IRODS_ZONE) self.assertEqual(res.resource_federation_path, fed_path) # test original file in user test zone is removed after resource creation # since True is used for move when creating the resource file_path_name = user_path + self.file_to_be_deleted self.assertFalse(self.irods_storage.exists(file_path_name)) # test django_irods CopyFiles() with an iRODS resource name being passed in # as input parameter to verify the file gets copied to the pass-in iRODS resource istorage = res.get_irods_storage() src_path = os.path.join(res.root_path, 'data', 'contents', self.file_to_be_deleted) dest_path = file_path_name istorage.copyFiles(src_path, dest_path, settings.HS_IRODS_LOCAL_ZONE_DEF_RES) # assert file did get copied over self.assertTrue(self.irods_storage.exists(file_path_name)) stdout = self.irods_storage.session.run("ils", None, "-l", file_path_name)[0].split() # assert copied file gets written to the iRODS resource being passed into copyFiles() call self.assertEqual(stdout[2], settings.HS_IRODS_LOCAL_ZONE_DEF_RES) # test resource file deletion res.files.all().delete() self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") # test add multiple files and 'copy' option in federated user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) hydroshare.add_resource_files( res.short_id, source_names=[fed_test_file1_full_path, fed_test_file2_full_path], move=False) # test resource has two files self.assertEqual(res.files.all().count(), 2, msg="Number of content files is not equal to 2") file_list = [] for f in res.files.all(): file_list.append(f.storage_path.split('/')[-1]) self.assertTrue( self.file_one in file_list, msg='file 1 has not been added in the resource in user zone') self.assertTrue( self.file_two in file_list, msg='file 2 has not been added in the resource in user zone') # test original two files in user test zone still exist after adding them to the resource # since False is used for move when creating the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) self.assertTrue(self.irods_storage.exists(user_path + self.file_two)) # test resource deletion hydroshare.resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test create new version resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user) new_res = hydroshare.create_new_version_resource( ori_res, new_res, self.user) # only need to test file-related attributes # ensure new versioned resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new versioned resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up hydroshare.resource.delete_resource(new_res.short_id) hydroshare.resource.delete_resource(ori_res.short_id) # test copy resource in user zone fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) ori_res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[fed_test_file1_full_path], move=False) # make sure ori_res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(ori_res.resource_federation_path, fed_path) self.assertEqual(ori_res.files.all().count(), 1, msg="Number of content files is not equal to 1") new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user, action='copy') new_res = hydroshare.copy_resource(ori_res, new_res) # only need to test file-related attributes # ensure new copied resource is created in the same federation zone as original resource self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path) # ensure new copied resource has the same number of content files as original resource self.assertEqual(ori_res.files.all().count(), new_res.files.all().count()) # delete resources to clean up hydroshare.resource.delete_resource(new_res.short_id) hydroshare.resource.delete_resource(ori_res.short_id) # test folder operations in user zone fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two) fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three) self.res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Original Generic Resource in User Zone', source_names=[ fed_file1_full_path, fed_file2_full_path, fed_file3_full_path ], move=False) # make sure self.res is created in federated user zone fed_path = '/{zone}/home/{user}'.format( zone=settings.HS_USER_IRODS_ZONE, user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE) self.assertEqual(self.res.resource_federation_path, fed_path) # resource should has only three files at this point self.assertEqual(self.res.files.all().count(), 3, msg="resource file count didn't match") self.file_name_list = [self.file_one, self.file_two, self.file_three] super(TestUserZoneIRODSFederation, self).resource_file_oprs() # delete resources to clean up hydroshare.resource.delete_resource(self.res.short_id) # test adding files from federated user zone to an empty resource # created in hydroshare zone res = hydroshare.resource.create_resource( resource_type='GenericResource', owner=self.user, title='My Test Generic Resource in HydroShare Zone') self.assertEqual(res.files.all().count(), 0, msg="Number of content files is not equal to 0") fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one) hydroshare.add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) # test resource has one file self.assertEqual(res.files.all().count(), 1, msg="Number of content files is not equal to 1") file_list = [] for f in res.files.all(): file_list.append(os.path.basename(f.storage_path)) self.assertTrue( self.file_one in file_list, msg='file 1 has not been added in the resource in hydroshare zone') # test original file in user test zone still exist after adding it to the resource # since 'copy' is used for fed_copy_or_move when adding the file to the resource self.assertTrue(self.irods_storage.exists(user_path + self.file_one)) # test replication of this resource to user zone even if the bag_modified AVU for this # resource is wrongly set to False when the bag for this resource does not exist and # need to be recreated res.setAVU('bag_modified', 'false') hydroshare.resource.replicate_resource_bag_to_user_zone( self.user, res.short_id) self.assertTrue(self.irods_storage.exists(user_path + res.short_id + '.zip'), msg='replicated resource bag is not in the user zone') # test resource deletion hydroshare.resource.delete_resource(res.short_id) self.assertEquals(BaseResource.objects.all().count(), 0, msg='Number of resources not equal to 0') # test to make sure original file still exist after resource deletion self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_get_revisions(self): # create a user to be used for creating the resource user_creator = hydroshare.create_account( '*****@*****.**', username='******', first_name='Creator_FirstName', last_name='Creator_LastName', superuser=False, groups=[] ) resource_changed_by = hydroshare.create_account( '*****@*****.**', username='******', first_name='Pabitra', last_name='Dash', superuser=False, groups=[] ) # create a resource resource = hydroshare.create_resource('GenericResource', user_creator, 'My resource') # test that we have only one revision at this point - this is the api call we are testing res_revisions = hydroshare.get_revisions(resource.short_id) self.assertEqual(len(res_revisions), 1) # set the resource last changed by a different user - to a create another revision of the resource hydroshare.utils.resource_modified(resource, resource_changed_by) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 2 revisions self.assertEqual(len(res_revisions), 2) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) # test that each resource revision has the same resource id for bags in res_revisions: self.assertEqual(resource.id, bags.object_id) # add a file to the resource to generate another revision of the resource # create a file original_file_name = 'original.txt' original_file = open(original_file_name, 'w') original_file.write("original text") original_file.close() original_file = open(original_file_name, 'r') # add the file to the resource hydroshare.add_resource_files(resource.short_id, original_file) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 3 revisions self.assertEqual(len(res_revisions), 3) # test that each revision has a different time stamp self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp) self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp) self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp) # delete the file in the resource to create another revision of the resource hydroshare.delete_resource_file(resource.short_id, original_file_name) res_revisions = hydroshare.get_revisions(resource.short_id) # test that we now have 4 revisions self.assertEqual(len(res_revisions), 4)
def resource_file_oprs(self): """Test common iRODS file operations. This is a common test utility function to be called by both regular folder operation testing and federated zone folder operation testing. Make sure the calling TestCase object has the following attributes defined before calling this method: self.res: resource that has been created that contains files listed in file_name_list self.user: owner of the resource self.file_name_list: a list of three file names that have been added to the res object self.test_file_1 needs to be present for the calling object for doing regular folder operations without involving federated zone so that the same opened file can be re-added to the resource for testing the case where zipping cannot overwrite existing file """ user = self.user res = self.res file_name_list = self.file_name_list # create a folder, if folder is created successfully, no exception is raised, otherwise, # an iRODS exception will be raised which will be caught by the test runner and mark as # a test failure create_folder(res.short_id, 'data/contents/sub_test_dir') istorage = res.get_irods_storage() res_path = res.file_path store = istorage.listdir(res_path) self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder') # rename the third file in file_name_list move_or_rename_file_or_folder(user, res.short_id, 'data/contents/' + file_name_list[2], 'data/contents/new_' + file_name_list[2]) # move the first two files in file_name_list to the new folder move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[1], 'data/contents/sub_test_dir/' + file_name_list[1]) updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg="resource does not contain the updated file new_" + file_name_list[2]) self.assertNotIn(file_name_list[2], updated_res_file_names, msg='resource still contains the old file ' + file_name_list[2] + ' after renaming') self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' moved to a folder') self.assertNotIn(file_name_list[0], updated_res_file_names, msg='resource still contains the old ' + file_name_list[0] + 'after moving to a folder') self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + 'moved to a new folder') self.assertNotIn(file_name_list[1], updated_res_file_names, msg='resource still contains the old ' + file_name_list[1] + ' after moving to a folder') # zip the folder output_zip_fname, size = \ zip_folder(user, res.short_id, 'data/contents/sub_test_dir', 'sub_test_dir.zip', True) self.assertGreater(size, 0, msg='zipped file has a size of 0') # Now resource should contain only two files: new_file3.txt and sub_test_dir.zip # since the folder is zipped into sub_test_dir.zip with the folder deleted self.assertEqual(res.files.all().count(), 2, msg="resource file count didn't match-") # test unzip does not allow override of existing files # add an existing file in the zip to the resource if res.resource_federation_path: fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format( zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0]) # TODO: why isn't this a method of resource? # TODO: Why do we repeat the resource_federation_path? add_resource_files(res.short_id, source_names=[fed_test_file1_full_path], move=False) else: # TODO: Why isn't this a method of resource? add_resource_files(res.short_id, self.test_file_1) # TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix create_folder(res.short_id, 'data/contents/sub_test_dir') # TODO: use ResourceFile.rename, which doesn't require data/contents prefix move_or_rename_file_or_folder( user, res.short_id, 'data/contents/' + file_name_list[0], 'data/contents/sub_test_dir/' + file_name_list[0]) # Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") with self.assertRaises(SessionException): unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False) # Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt file_cnt = res.files.all().count() self.assertEqual(file_cnt, 3, msg="resource file count didn't match - " + str(file_cnt) + " != 3") # test unzipping the file succeeds now after deleting the existing folder # TODO: this causes a multiple delete because the paths are valid now. istorage = res.get_irods_storage() remove_folder(user, res.short_id, 'data/contents/sub_test_dir') # Now resource should contain two files: file3_new.txt and sub_test_dir.zip file_cnt = res.files.all().count() self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " + str(file_cnt) + " != 2") unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True) # Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn( 'sub_test_dir.zip', updated_res_file_names, msg="resource still contains the zip file after unzipping") self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[0]) self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain unzipped file ' + file_name_list[1]) self.assertIn('new_' + file_name_list[2], updated_res_file_names, msg='resource does not contain unzipped file new_' + file_name_list[2]) # rename a folder move_or_rename_file_or_folder(user, res.short_id, 'data/contents/sub_test_dir', 'data/contents/sub_dir') updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertNotIn('sub_test_dir/' + file_name_list[0], updated_res_file_names, msg='resource still contains ' + file_name_list[0] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names, msg='resource does not contain ' + file_name_list[0] + ' in the new folder after renaming') self.assertNotIn('sub_test_dir/' + file_name_list[1], updated_res_file_names, msg='resource still contains ' + file_name_list[1] + ' in the old folder after renaming') self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names, msg='resource does not contain ' + file_name_list[1] + ' in the new folder after renaming') # remove a folder # TODO: utilize ResourceFile.remove_folder instead. Takes a short path. remove_folder(user, res.short_id, 'data/contents/sub_dir') # Now resource only contains one file self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match") updated_res_file_names = [] for rf in ResourceFile.objects.filter(object_id=res.id): updated_res_file_names.append(rf.short_path) self.assertEqual(len(updated_res_file_names), 1) self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def test_metadata(self): # add a type element resource.create_metadata_element(self.resTimeSeries.short_id, 'type', url="http://hydroshare.org/netcdf") # add another creator with all sub_elements cr_name = 'Mike Sundar' cr_des = 'http://hydroshare.org/user/001' cr_org = "USU" cr_email = '*****@*****.**' cr_address = "11 River Drive, Logan UT-84321, USA" cr_phone = '435-567-0989' cr_homepage = 'http://usu.edu/homepage/001' cr_res_id = 'http://research.org/001' cr_res_gate_id = 'http://research-gate.org/001' resource.create_metadata_element(self.resTimeSeries.short_id,'creator', name=cr_name, description=cr_des, organization=cr_org, email=cr_email, address=cr_address, phone=cr_phone, homepage=cr_homepage, researcherID=cr_res_id, researchGateID=cr_res_gate_id) # add another creator with only the name resource.create_metadata_element(self.resTimeSeries.short_id,'creator', name='Lisa Holley') #test adding a contributor with all sub_elements con_name = 'Sujan Peterson' con_des = 'http://hydroshare.org/user/002' con_org = "USU" con_email = '*****@*****.**' con_address = "101 Center St, Logan UT-84321, USA" con_phone = '435-567-3245' con_homepage = 'http://usu.edu/homepage/009' con_res_id = 'http://research.org/009' con_res_gate_id = 'http://research-gate.org/009' resource.create_metadata_element(self.resTimeSeries.short_id,'contributor', name=con_name, description=con_des, organization=con_org, email=con_email, address=con_address, phone=con_phone, homepage=con_homepage, researcherID=con_res_id, researchGateID=con_res_gate_id) # add another creator with only the name resource.create_metadata_element(self.resTimeSeries.short_id,'contributor', name='Andrew Smith') # add a period type coverage # add a period type coverage value_dict = {'name':'Name for period coverage' , 'start':'1/1/2000', 'end':'12/12/2012'} resource.create_metadata_element(self.resTimeSeries.short_id,'coverage', type='period', value=value_dict) # add a point type coverage value_dict = {'name':'Name for point coverage', 'east':'56.45678', 'north':'12.6789'} resource.create_metadata_element(self.resTimeSeries.short_id,'coverage', type='point', value=value_dict) # add date of type 'valid' resource.create_metadata_element(self.resTimeSeries.short_id,'date', type='valid', start_date='8/10/2011', end_date='8/11/2012') # add a format element format_nc = 'netcdf' resource.create_metadata_element(self.resTimeSeries.short_id,'format', value=format_nc) # add 'DOI' identifier #resource.create_metadata_element(self.resTimeSeries.short_id,'identifier', name='DOI', url="http://dx.doi.org/001") # add a language element resource.create_metadata_element(self.resTimeSeries.short_id,'language', code='eng') # add 'Publisher' element original_file_name = 'original.txt' original_file = open(original_file_name, 'w') original_file.write("original text") original_file.close() original_file = open(original_file_name, 'r') # add the file to the resource hydroshare.add_resource_files(self.resTimeSeries.short_id, original_file) resource.create_metadata_element(self.resTimeSeries.short_id,'publisher', name="HydroShare", url="http://hydroshare.org") # add a relation element of uri type resource.create_metadata_element(self.resTimeSeries.short_id,'relation', type='isPartOf', value='http://hydroshare.org/resource/001') # add another relation element of non-uri type resource.create_metadata_element(self.resTimeSeries.short_id,'relation', type='isDataFor', value='This resource is for another resource') # add a source element of uri type resource.create_metadata_element(self.resTimeSeries.short_id,'source', derived_from='http://hydroshare.org/resource/0002') # add a rights element resource.create_metadata_element(self.resTimeSeries.short_id,'rights', statement='This is the rights statement for this resource', url='http://rights.ord/001') # add a subject element resource.create_metadata_element(self.resTimeSeries.short_id,'subject', value='sub-1') # add another subject element resource.create_metadata_element(self.resTimeSeries.short_id,'subject', value='sub-2') # add time series specific metadata elements self.resTimeSeries.metadata.create_element('site', site_code='LR_WaterLab_AA', site_name='Logan River at the Utah Water Research Laboratory ' 'west bridge', elevation_m=1414, elevation_datum='EGM96', site_type='Stream') self.resTimeSeries.metadata.create_element('variable', variable_code='ODO', variable_name='Oxygen, dissolved', variable_type='Concentration', no_data_value=-9999, variable_definition='Concentration of oxygen gas dissolved in water.', speciation='Not Applicable') self.resTimeSeries.metadata.create_element('method', method_code=59, method_name='Optical DO', method_type='Instrument deployment', method_description='Dissolved oxygen concentration measured ' 'optically using a YSI EXO multi-parameter water ' 'quality sonde.', method_link='http://www.exowater.com') exp_text = """Raw and unprocessed data and data products that have not undergone quality control. Depending on the variable, data type, and data transmission system, raw data may be available within seconds or minutes after the measurements have been made. Examples include real time precipitation, streamflow and water quality measurements.""" self.resTimeSeries.metadata.create_element('processinglevel', processing_level_code=0, definition='Raw data', explanation=exp_text) self.resTimeSeries.metadata.create_element('timeseriesresult', units_type='Concentration', units_name='milligrams per liter', units_abbreviation='mg/L', status='Complete', sample_medium='Surface water', value_count=11283, aggregation_statistics="Average") print self.resTimeSeries.metadata.get_xml() print(bad)
def generate_files(request, shortkey, *args, **kwargs): res = hydroshare.get_resource_by_shortkey(shortkey) ts, csv_link, csv_size, xml_link, xml_size = {}, '', '', '', '' try: if res.reference_type == 'rest': ts = ts_utils.time_series_from_service(res.url, res.reference_type) else: ts = ts_utils.time_series_from_service(res.url, res.reference_type, site_name_or_code=res.data_site_code, variable_code=res.variable_code) vals = ts['values'] version = ts['wml_version'] d = datetime.date.today() date = '{0}_{1}_{2}'.format(d.month, d.day, d.year) file_base = '{0}-{1}'.format(res.title.replace(" ", ""), date) csv_name = '{0}.{1}'.format(file_base, 'csv') if version == '1': xml_end = 'wml_1' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) elif version == '2.0': xml_end = 'wml_2_0' xml_name = '{0}-{1}.xml'.format(file_base, xml_end) for_csv = [] for k, v in vals.items(): t = (k, v) for_csv.append(t) ResourceFile.objects.filter(object_id=res.pk).delete() with open(csv_name, 'wb') as csv_file: w = csv.writer(csv_file) w.writerow([res.title]) var = '{0}({1})'.format(ts['variable_name'], ts['units']) w.writerow(['time', var]) for r in for_csv: w.writerow(r) with open(xml_name, 'wb') as xml_file: xml_file.write(ts['time_series']) csv_file = open(csv_name, 'r') xml_file = open(xml_name, 'r') files = [csv_file, xml_file] hydroshare.add_resource_files(res.short_id, csv_file, xml_file) create_bag(res) os.remove(csv_name) os.remove(xml_name) files = ResourceFile.objects.filter(object_id=res.pk) for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if xml_end in str(f.resource_file): xml_link = f.resource_file.url xml_size = f.resource_file.size status_code = 200 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_name': csv_name, 'xml_name': xml_name, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # successfully generated new files except Exception: # most likely because the server is unreachable files = ResourceFile.objects.filter(object_id=res.pk) xml_file = None for f in files: if str(f.resource_file).endswith('.csv'): csv_link = f.resource_file.url csv_size = f.resource_file.size if str(f.resource_file).endswith('.xml'): xml_link = f.resource_file.url xml_size = f.resource_file.size xml_file = f.resource_file if xml_file is None: status_code = 404 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, did not find old ones xml_doc = open(str(xml_file), 'r').read() root = etree.XML(xml_doc) os.remove(str(xml_file)) version = ts_utils.get_version(root) if version == '1': ts = ts_utils.parse_1_0_and_1_1(root) status_code = 200 elif version =='2.0': ts = ts_utils.parse_2_0(root) status_code = 200 else: status_code = 503 data = {'for_graph': ts.get('for_graph'), 'values': ts.get('values'), 'units': ts.get('units'), 'site_name': ts.get('site_name'), 'variable_name': ts.get('variable_name'), 'status_code': status_code, 'csv_link': csv_link, 'csv_size': csv_size, 'xml_link': xml_link, 'xml_size': xml_size} return json_or_jsonp(request, data) # did not generate new files, return old ones
def test_unfederated_root_path_checks(self): """ an unfederated file in the root folder has the proper state after state changes """ # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") check_irods_files(self.res, stop_on_error=True) # add one file to the resource hydroshare.add_resource_files(self.res.short_id, self.test_file_1) # should succeed without errors check_irods_files(self.res, stop_on_error=True) # cleaning should not change anything check_irods_files(self.res, stop_on_error=True, log_errors=False, return_errors=True, clean_irods=True, clean_django=True, sync_ispublic=True) # resource should has only one file at this point self.assertEqual(self.res.files.all().count(), 1, msg="resource file count didn't match") # get the handle of the file created above resfile = self.res.files.all()[0] # determine where that file should live shortpath = os.path.join(self.res.short_id, "data", "contents", "file1.txt") self.assertEqual(resfile.file_folder, None) self.assertEqual(resfile.storage_path, shortpath) self.assertTrue(resfile.path_is_acceptable(shortpath)) # now try to intentionally corrupt it resfile.set_short_path("fuzz.txt") # should raise exception with self.assertRaises(ValidationError): check_irods_files(self.res, stop_on_error=True) # now don't raise exception and read error errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS')) self.assertTrue(errors[1].endswith( 'data/contents/file1.txt in iRODs does not exist in Django')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # now try to clean it up errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False, clean_irods=True, clean_django=True) self.assertTrue(errors[0].endswith( 'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)')) self.assertTrue(errors[1].endswith( 'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)')) self.assertTrue(errors[2].endswith( "type is GenericResource, title is 'My Test Resource'")) # resource should not have any files at this point self.assertEqual(self.res.files.all().count(), 0, msg="resource file count didn't match") # now check should succeed errors, ecount = check_irods_files(self.res, stop_on_error=True, log_errors=False) self.assertEqual(ecount, 0) # delete resources to clean up hydroshare.delete_resource(self.res.short_id)
def test_netcdf_metadata(self): # add a type element resource.create_metadata_element(self.resNetCDF.short_id, 'type', url="http://hydroshare.org/netcdf") # add another creator with all sub_elements cr_name = 'Mike Sundar' cr_des = 'http://hydroshare.org/user/001' cr_org = "USU" cr_email = '*****@*****.**' cr_address = "11 River Drive, Logan UT-84321, USA" cr_phone = '435-567-0989' cr_homepage = 'http://usu.edu/homepage/001' cr_res_id = 'http://research.org/001' cr_res_gate_id = 'http://research-gate.org/001' resource.create_metadata_element(self.resNetCDF.short_id,'creator', name=cr_name, description=cr_des, organization=cr_org, email=cr_email, address=cr_address, phone=cr_phone, homepage=cr_homepage, researcherID=cr_res_id, researchGateID=cr_res_gate_id) # add another creator with only the name resource.create_metadata_element(self.resNetCDF.short_id,'creator', name='Lisa Holley') #test adding a contributor with all sub_elements con_name = 'Sujan Peterson' con_des = 'http://hydroshare.org/user/002' con_org = "USU" con_email = '*****@*****.**' con_address = "101 Center St, Logan UT-84321, USA" con_phone = '435-567-3245' con_homepage = 'http://usu.edu/homepage/009' con_res_id = 'http://research.org/009' con_res_gate_id = 'http://research-gate.org/009' resource.create_metadata_element(self.resNetCDF.short_id,'contributor', name=con_name, description=con_des, organization=con_org, email=con_email, address=con_address, phone=con_phone, homepage=con_homepage, researcherID=con_res_id, researchGateID=con_res_gate_id) # add another creator with only the name resource.create_metadata_element(self.resNetCDF.short_id,'contributor', name='Andrew Smith') # add a period type coverage # add a period type coverage value_dict = {'name':'Name for period coverage' , 'start':'1/1/2000', 'end':'12/12/2012'} resource.create_metadata_element(self.resNetCDF.short_id,'coverage', type='period', value=value_dict) # add a point type coverage value_dict = {'name':'Name for point coverage', 'east':'56.45678', 'north':'12.6789'} resource.create_metadata_element(self.resNetCDF.short_id,'coverage', type='point', value=value_dict) # add date of type 'valid' resource.create_metadata_element(self.resNetCDF.short_id,'date', type='valid', start_date='8/10/2011', end_date='8/11/2012') # add a format element format_nc = 'netcdf' resource.create_metadata_element(self.resNetCDF.short_id,'format', value=format_nc) # add 'DOI' identifier #resource.create_metadata_element(self.resNetCDF.short_id,'identifier', name='DOI', url="http://dx.doi.org/001") # add a language element resource.create_metadata_element(self.resNetCDF.short_id,'language', code='eng') # add 'Publisher' element original_file_name = 'original.txt' original_file = open(original_file_name, 'w') original_file.write("original text") original_file.close() original_file = open(original_file_name, 'r') # add the file to the resource hydroshare.add_resource_files(self.resNetCDF.short_id, original_file) resource.create_metadata_element(self.resNetCDF.short_id,'publisher', name="HydroShare", url="http://hydroshare.org") # add a relation element of uri type resource.create_metadata_element(self.resNetCDF.short_id,'relation', type='isPartOf', value='http://hydroshare.org/resource/001') # add another relation element of non-uri type resource.create_metadata_element(self.resNetCDF.short_id,'relation', type='isDataFor', value='This resource is for another resource') # add a source element of uri type resource.create_metadata_element(self.resNetCDF.short_id,'source', derived_from='http://hydroshare.org/resource/0002') # add a rights element resource.create_metadata_element(self.resNetCDF.short_id,'rights', statement='This is the rights statement for this resource', url='http://rights.ord/001') # add a subject element resource.create_metadata_element(self.resNetCDF.short_id,'subject', value='sub-1') # add another subject element resource.create_metadata_element(self.resNetCDF.short_id,'subject', value='sub-2') # add a netcdf specific element (variable) resource.create_metadata_element(self.resNetCDF.short_id,'variable', name='temp', unit='deg C', type='float', shape='shape_unknown') print self.resNetCDF.metadata.get_xml() print(bad)