def transform_file(request, shortkey, *args, **kwargs):
res = hydroshare.get_resource_by_shortkey(shortkey)
if res.reference_type == 'soap':
client = Client(res.url)
response = client.service.GetValues(':'+res.data_site_code, ':'+res.variable_code, '', '', '')
elif res.reference_type == 'rest':
r = requests.get(res.url)
response = str(r.text)
waterml_1 = etree.XML(response)
wml_string = etree.tostring(waterml_1)
s = StringIO(wml_string)
dom = etree.parse(s)
module_dir = os.path.dirname(__file__)
xsl_location = os.path.join(module_dir, "static/ref_ts/xslt/WaterML1_1_timeSeries_to_WaterML2.xsl")
xslt = etree.parse(xsl_location)
transform = etree.XSLT(xslt)
newdom = transform(dom)
d = datetime.date.today()
date = '{0}_{1}_{2}'.format(d.month, d.day, d.year)
xml_name = '{0}-{1}-{2}'.format(res.title.replace(" ", ""), date, 'wml_2_0.xml')
with open(xml_name, 'wb') as f:
f.write(newdom)
xml_file = open(xml_name, 'r')
ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0').delete()
hydroshare.add_resource_files(res.short_id, xml_file)
f = ResourceFile.objects.filter(object_id=res.pk, resource_file__contains='wml_2_0')[0].resource_file
data = {
'status_code': 200,
'xml_name': xml_name,
'xml_size': f.size,
'xml_link': f.url
}
os.remove(xml_name)
# print(etree.tostring(newdom, pretty_print=True))
return json_or_jsonp(request, data)
def test_move_to_folder_basic(self):
group, _ = Group.objects.get_or_create(name='Hydroshare Author')
user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[]
)
resource = hydroshare.create_resource(
'GenericResource',
user,
'test resource',
)
resource.save()
open('myfile.txt', "w").close()
file = open('myfile.txt', 'r')
hydroshare.add_resource_files(resource.short_id, file)
create_folder(resource.short_id, "data/contents/test_folder")
move_to_folder(user, resource.short_id,
src_paths=['data/contents/myfile.txt'],
tgt_path="data/contents/test_folder",
validate_move=True)
folder_contents = list_folder(resource.short_id, "data/contents/test_folder")
self.assertTrue(['myfile.txt'] in folder_contents)
resource.delete()
def test_unfederated_folder_path_setting(self):
""" an unfederated file in a subfolder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
ResourceFile.create_folder(self.res, 'foo')
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo')
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data",
"contents", "foo", "file1.txt")
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# non-existent files should raise error
otherpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file2.txt")
with self.assertRaises(ValidationError):
resfile.path_is_acceptable(otherpath)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("foo/file1.txt")
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
with self.assertRaises(ValidationError):
resfile.set_storage_path(otherpath)
# should not change
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# TODO: how to eliminate this particular error.
# dumbpath = 'x' + shortpath
# dumbpath = self.res.short_id + "file1.txt"
# clean up after folder test
# ResourceFile.remove_folder(self.res, 'foo', self.user)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_move_to_folder_basic(self):
group, _ = Group.objects.get_or_create(name='Hydroshare Author')
user = hydroshare.create_account('*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[])
resource = hydroshare.create_resource(
'GenericResource',
user,
'test resource',
)
resource.save()
open('myfile.txt', "w").close()
file = open('myfile.txt', 'rb')
hydroshare.add_resource_files(resource.short_id, file)
create_folder(resource.short_id, "data/contents/test_folder")
move_to_folder(user,
resource.short_id,
src_paths=['data/contents/myfile.txt'],
tgt_path="data/contents/test_folder",
validate_move=True)
folder_contents = list_folder(resource.short_id,
"data/contents/test_folder")
self.assertTrue(['myfile.txt'] in folder_contents)
resource.delete()
def migrate_tif_file(apps, schema_editor):
# create a vrt file from tif file for each of the Raster Resources
log = logging.getLogger()
istorage = IrodsStorage()
for res in RasterResource.objects.all():
try:
if len(res.files.all()) == 1:
res_file = res.files.all().first()
vrt_file_path = create_vrt_file(res_file.resource_file)
if os.path.isfile(vrt_file_path):
files = (UploadedFile(file=open(vrt_file_path, 'r'),
name=os.path.basename(vrt_file_path)))
hydroshare.add_resource_files(res.short_id, files)
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
print("Deleted bag for resource ID:" + str(res.short_id))
resource_modified(res, res.creator)
log.info('Tif file conversion to VRT successful for resource:ID:{} '
'Title:{}'.format(res.short_id, res.metadata.title.value))
else:
log.error('Tif file conversion to VRT unsuccessful for resource:ID:{} '
'Title:{}'.format(res.short_id, res.metadata.title.value))
if os.path.exists(vrt_file_path):
shutil.rmtree(os.path.dirname(vrt_file_path))
except:
pass
def test_resource_file_put(self):
myfile = open(self.filename, 'w')
myfile.write('hello world!\n')
myfile.close()
myfile = open(self.filename, 'r')
res1 = hydroshare.create_resource('GenericResource', self.user, 'res1')
hydroshare.add_resource_files(res1.short_id, myfile)
mynewfile = open(self.filename, 'w')
mynewfile.write('anyone there?\n')
mynewfile.close()
mynewfile = open(self.filename, 'r')
url = self.url_proto.format(res1.short_id, self.filename)
put_data = { 'resource_file': mynewfile
}
resp = self.api_client.put(url, data=put_data)
self.assertHttpAccepted(resp)
resp = self.api_client.get(url)
self.assertIn(resp.status_code, [201, 200])
def test_resource_file_put(self):
myfile = open(self.filename, 'w')
myfile.write('hello world!\n')
myfile.close()
myfile = open(self.filename, 'r')
res1 = hydroshare.create_resource('GenericResource', self.user, 'res1')
hydroshare.add_resource_files(res1.short_id, myfile)
mynewfile = open(self.filename, 'w')
mynewfile.write('anyone there?\n')
mynewfile.close()
mynewfile = open(self.filename, 'r')
url = self.url_proto.format(res1.short_id, self.filename)
put_data = {'resource_file': mynewfile}
resp = self.api_client.put(url, data=put_data)
self.assertHttpAccepted(resp)
resp = self.api_client.get(url)
self.assertIn(resp.status_code, [201, 200])
def test_unfederated_folder_path_setting(self):
""" an unfederated file in a subfolder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
ResourceFile.create_folder(self.res, 'foo')
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo')
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data",
"contents", "foo", "file1.txt")
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# non-existent files should raise error
otherpath = os.path.join(self.res.short_id, "data", "contents", "foo", "file2.txt")
with self.assertRaises(ValidationError):
resfile.path_is_acceptable(otherpath)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("foo/file1.txt")
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
with self.assertRaises(ValidationError):
resfile.set_storage_path(otherpath)
# should not change
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# TODO: how to eliminate this particular error.
# dumbpath = 'x' + shortpath
# dumbpath = self.res.short_id + "file1.txt"
# clean up after folder test
# ResourceFile.remove_folder(self.res, 'foo', self.user)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def generate_files(shortkey, ts):
''' creates the calls the make files fxn and adds the files as resource files
called by view: create_ref_time_series, update_files
:param shortkey: res shortkey
:param ts: parsed time series dict
'''
res = hydroshare.get_resource_by_shortkey(shortkey)
if ts is None: #called by update_files in view
if res.metadata.referenceURLs.all()[0].type == 'rest':
ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value, res.metadata.referenceURLs.all()[0].type)
else:
ts = time_series_from_service(res.metadata.referenceURLs.all()[0].value,
res.metadata.referenceURLs.all()[0].type,
site_name_or_code=res.metadata.sites.all()[0].code,
variable_code=res.metadata.variables.all()[0].code)
tempdir = None
try:
tempdir = tempfile.mkdtemp()
files = make_files(res, tempdir, ts)
hydroshare.add_resource_files(res.short_id, *files)
except Exception as e:
raise e
finally:
if tempdir is not None:
shutil.rmtree(tempdir)
def setUp(self):
super(TestTickets, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
self.user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[]
)
self.res = hydroshare.create_resource(
'GenericResource',
self.user,
'test resource',
)
# create a file
self.test_file_name1 = 'file1.txt'
self.file_name_list = [self.test_file_name1, ]
# put predictable contents into the file
test_file = open(self.test_file_name1, 'w')
test_file.write("Test text file in file1.txt")
test_file.close()
self.test_file_1 = open(self.test_file_name1, 'r')
# add one file to the resource: necessary so data/contents is created.
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
def setUp(self):
super(TestNewVersionResource, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
# create a user who is the owner of the resource to be versioned
self.owner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='owner_firstname',
last_name='owner_lastname',
superuser=False,
groups=[]
)
# create a user who is NOT the owner of the resource to be versioned
self.nonowner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='nonowner_firstname',
last_name='nonowner_lastname',
superuser=False,
groups=[]
)
# create a generic resource
self.res_generic = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.owner,
title='Test Generic Resource'
)
test_file1 = open('test1.txt', 'w')
test_file1.write("Test text file in test1.txt")
test_file1.close()
test_file2 = open('test2.txt', 'w')
test_file2.write("Test text file in test2.txt")
test_file2.close()
self.test_file1 = open('test1.txt', 'r')
self.test_file2 = open('test2.txt', 'r')
hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2)
# create a raster resource that represents a specific resource type
raster_file = 'hs_core/tests/data/cea.tif'
temp_dir = tempfile.mkdtemp()
temp_raster_file = os.path.join(temp_dir, 'cea.tif')
shutil.copy(raster_file, temp_raster_file)
self.raster_obj = open(temp_raster_file, 'r')
files = [UploadedFile(file=self.raster_obj, name='cea.tif')]
self.res_raster = hydroshare.create_resource(
resource_type='RasterResource',
owner=self.owner,
title='Test Raster Resource',
files=files,
metadata=[]
)
# call the post creation process here for the metadata to be
# extracted
utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def setUp(self):
super(TestNewVersionResource, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
# create a user who is the owner of the resource to be versioned
self.owner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='owner_firstname',
last_name='owner_lastname',
superuser=False,
groups=[]
)
# create a user who is NOT the owner of the resource to be versioned
self.nonowner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='nonowner_firstname',
last_name='nonowner_lastname',
superuser=False,
groups=[]
)
# create a generic resource
self.res_generic = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.owner,
title='Test Generic Resource'
)
test_file1 = open('test1.txt', 'w')
test_file1.write("Test text file in test1.txt")
test_file1.close()
test_file2 = open('test2.txt', 'w')
test_file2.write("Test text file in test2.txt")
test_file2.close()
self.test_file1 = open('test1.txt', 'r')
self.test_file2 = open('test2.txt', 'r')
hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2)
# create a raster resource that represents a specific resource type
raster_file = 'hs_core/tests/data/cea.tif'
temp_dir = tempfile.mkdtemp()
temp_raster_file = os.path.join(temp_dir, 'cea.tif')
shutil.copy(raster_file, temp_raster_file)
self.raster_obj = open(temp_raster_file, 'r')
files = [UploadedFile(file=self.raster_obj, name='cea.tif')]
self.res_raster = hydroshare.create_resource(
resource_type='RasterResource',
owner=self.owner,
title='Test Raster Resource',
files=files,
metadata=[]
)
# call the post creation process here for the metadata to be
# extracted
utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_resource_operations_in_user_zone(self):
super(TestUserZoneIRODSFederation,
self).assert_federated_irods_available()
# test adding files from federated user zone to an empty resource
# created in hydroshare zone
res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in HydroShare Zone')
self.assertEqual(res.files.all().count(),
0,
msg="Number of content files is not equal to 0")
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
hydroshare.add_resource_files(
res.short_id,
source_names=[fed_test_file1_full_path, fed_file2_full_path])
# test resource has two files
self.assertEqual(res.files.all().count(),
2,
msg="Number of content files is not equal to 2")
user_path = '/{zone}/home/testuser/'.format(
zone=settings.HS_USER_IRODS_ZONE)
file_list = []
for f in res.files.all():
file_list.append(os.path.basename(f.storage_path))
self.assertTrue(
self.file_one in file_list,
msg='file 1 has not been added in the resource in hydroshare zone')
self.assertTrue(
self.file_two in file_list,
msg='file 2 has not been added in the resource in hydroshare zone')
# test original file in user test zone still exist after adding it to the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
# test replication of this resource to user zone even if the bag_modified AVU for this
# resource is wrongly set to False when the bag for this resource does not exist and
# need to be recreated
res.setAVU('bag_modified', 'false')
hydroshare.resource.replicate_resource_bag_to_user_zone(
self.user, res.short_id)
self.assertTrue(self.irods_storage.exists(user_path + res.short_id +
'.zip'),
msg='replicated resource bag is not in the user zone')
# test resource deletion
hydroshare.resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(),
0,
msg='Number of resources not equal to 0')
# test to make sure original files still exist after resource deletion
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
def test_bag_ingestion(self):
from hs_core.views.utils import unzip_file
def normalize_metadata(metadata_str):
"""Prepares metadata string to match resource id and hydroshare url of original"""
return metadata_str\
.replace(current_site_url(), "http://www.hydroshare.org")\
.replace(res.short_id, "97523bdb7b174901b3fc2d89813458f1")
# create empty resource
res = resource.create_resource(
'CompositeResource',
self.user,
'My Test Resource'
)
full_paths = {}
files_to_upload = [UploadedFile(file=open('hs_core/tests/data/test_resource_metadata_files.zip', 'rb'),
name="test_resource_metadata_files.zip")]
add_resource_files(res.short_id, *files_to_upload, full_paths=full_paths)
unzip_file(self.user, res.short_id, "data/contents/test_resource_metadata_files.zip", True,
overwrite=True, auto_aggregate=True, ingest_metadata=True)
def compare_metadatas(new_metadata_str, original_metadata_file):
original_graph = Graph()
with open(os.path.join(self.extracted_directory, original_metadata_file), "r") as f:
original_graph = original_graph.parse(data=f.read())
new_graph = Graph()
new_graph = new_graph.parse(data=normalize_metadata(new_metadata_str))
# remove modified date, they'll never match
subject = new_graph.value(predicate=RDF.type, object=DCTERMS.modified)
new_graph.remove((subject, None, None))
subject = original_graph.value(predicate=RDF.type, object=DCTERMS.modified)
original_graph.remove((subject, None, None))
for (new_triple, original_triple) in _squashed_graphs_triples(new_graph, original_graph):
self.assertEquals(new_triple, original_triple, "Ingested resource metadata does not match original")
res.refresh_from_db()
compare_metadatas(res.metadata.get_xml(), "resourcemetadata.xml")
compare_metadatas(res.get_logical_files(GenericLogicalFile.type_name())[0].metadata.get_xml(),
"test_meta.xml")
compare_metadatas(res.get_logical_files(FileSetLogicalFile.type_name())[0].metadata.get_xml(),
"asdf/asdf_meta.xml")
compare_metadatas(res.get_logical_files(GeoFeatureLogicalFile.type_name())[0].metadata.get_xml(),
"watersheds_meta.xml")
compare_metadatas(res.get_logical_files(GeoRasterLogicalFile.type_name())[0].metadata.get_xml(),
"logan_meta.xml")
compare_metadatas(res.get_logical_files(NetCDFLogicalFile.type_name())[0].metadata.get_xml(),
"SWE_time_meta.xml")
compare_metadatas(res.get_logical_files(RefTimeseriesLogicalFile.type_name())[0].metadata.get_xml(),
"msf_version.refts_meta.xml")
compare_metadatas(res.get_logical_files(TimeSeriesLogicalFile.type_name())[0].metadata.get_xml(),
"ODM2_Multi_Site_One_Variable_meta.xml")
def add_resource_file(self, pk, filename=None):
authorize(self.request, pk, edit=True)
if filename:
rf = self.request.FILES.values()[0]
rf.name = filename
f = hydroshare.add_resource_files(pk, rf)
return HttpResponse(f.resource_file.url, content_type='text/plain')
else:
fs = hydroshare.add_resource_files(pk, self.request.FILES.values())
return json_or_jsonp(self.request, { f.name: f.resource_file.url for f in fs})
def add_resource_file(self, pk, filename=None):
authorize(self.request, pk, edit=True)
if filename:
rf = self.request.FILES.values()[0]
rf.name = filename
f = hydroshare.add_resource_files(pk, rf)
return HttpResponse(f.resource_file.url, content_type='text/plain')
else:
fs = hydroshare.add_resource_files(pk, self.request.FILES.values())
return json_or_jsonp(self.request,
{f.name: f.resource_file.url
for f in fs})
def test_resource_file_get(self):
myfile = open(self.filename, 'w')
myfile.write('hello world!\n')
myfile.close()
myfile = open(self.filename, 'r')
res1 = hydroshare.create_resource('GenericResource', self.user, 'res1')
hydroshare.add_resource_files(res1.short_id, myfile)
url = self.url_proto.format(res1.short_id, self.filename)
resp = self.api_client.get(url)
self.assertIn(resp.status_code, [201, 200])
def test_post_add_files_to_resource(self):
# test: add all files
for f in ResourceFile.objects.filter(object_id=self.resGeoFeature.id):
f.resource_file.delete()
f.delete()
# add files first
files = []
target = 'hs_geographic_feature_resource/tests/gis.osm_adminareas_v06_with_folder.zip'
files.append(UploadedFile(file=open(target, 'r'),
name='gis.osm_adminareas_v06_with_folder.zip'))
hydroshare.utils.resource_file_add_pre_process(self.resGeoFeature, files, self.user,)
hydroshare.add_resource_files(self.resGeoFeature.short_id, *files)
self.assertEqual(ResourceFile.objects.filter(object_id=self.resGeoFeature.id).count(), 5)
# remove .prj
for res_f_obj in ResourceFile.objects.filter(object_id=self.resGeoFeature.id):
del_f_fullname = res_f_obj.resource_file.name.lower()
del_f_fullname = del_f_fullname[del_f_fullname.rfind('/')+1:]
del_f_name, del_f_ext = os.path.splitext(del_f_fullname)
if del_f_ext == ".prj":
hydroshare.delete_resource_file(self.resGeoFeature.short_id,
res_f_obj.id, self.user)
self.assertEqual(ResourceFile.objects.filter
(object_id=self.resGeoFeature.id).count(), 4)
for res_f_obj_2 in ResourceFile.objects.filter(object_id=self.resGeoFeature.id):
del_f_fullname = res_f_obj_2.resource_file.name.lower()
del_f_fullname = del_f_fullname[del_f_fullname.rfind('/')+1:]
del_f_name, del_f_ext = os.path.splitext(del_f_fullname)
self.assertNotEqual(del_f_ext, ".prj")
originalcoverage_obj = self.resGeoFeature.metadata.\
originalcoverage.all().first()
self.assertEqual(originalcoverage_obj.projection_string, UNKNOWN_STR)
self.assertEqual(self.resGeoFeature.metadata.coverages.all().count(), 0)
# add .prj
add_files = []
target = 'hs_geographic_feature_resource/' \
'tests/gis.osm_adminareas_v06/gis.osm_adminareas_v06.prj'
add_files.append(UploadedFile(file=open(target, 'r'), name='gis.osm_adminareas_v06.prj'))
hydroshare.add_resource_files(self.resGeoFeature.short_id, *add_files)
self.assertEqual(ResourceFile.objects.filter(object_id=self.resGeoFeature.id).count(), 5)
geofeature_post_add_files_to_resource_handler(sender=GeographicFeatureResource,
resource=self.resGeoFeature, files=add_files,
validate_files={'are_files_valid': True,
'message': ''})
originalcoverage_obj = self.resGeoFeature.metadata.originalcoverage.all().first()
self.assertNotEqual(originalcoverage_obj.projection_string, UNKNOWN_STR)
def test_resource_file_get(self):
myfile = open(self.filename, 'w')
myfile.write('hello world!\n')
myfile.close()
myfile = open(self.filename, 'r')
res1 = hydroshare.create_resource('GenericResource', self.user, 'res1')
hydroshare.add_resource_files(res1.short_id, myfile)
url = self.url_proto.format(res1.short_id, self.filename)
resp = self.api_client.get(url)
self.assertIn(resp.status_code, [201, 200])
def test_resource_file_folder_oprs(self):
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add the three files to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1, self.test_file_2,
self.test_file_3)
# resource should has only three files at this point
self.assertEqual(self.res.files.all().count(), 3,
msg="resource file count didn't match")
super(TestResourceFileFolderOprsAPI, self).resource_file_oprs()
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def add_files_to_resource(self, files_to_add, upload_folder=None):
files_to_upload = []
for fl in files_to_add:
file_to_upload = UploadedFile(file=open(fl, 'rb'), name=os.path.basename(fl))
files_to_upload.append(file_to_upload)
added_resource_files = add_resource_files(self.composite_resource.short_id,
*files_to_upload, folder=upload_folder)
return added_resource_files
def add_files_to_resource(self, files_to_add, upload_folder=None):
files_to_upload = []
for fl in files_to_add:
file_to_upload = UploadedFile(file=open(fl, 'rb'), name=os.path.basename(fl))
files_to_upload.append(file_to_upload)
added_resource_files = add_resource_files(self.composite_resource.short_id,
*files_to_upload, folder=upload_folder)
return added_resource_files
def test_resource_file_folder_oprs(self):
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(),
0,
msg="resource file count didn't match")
# add the three files to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1,
self.test_file_2, self.test_file_3)
# resource should has only three files at this point
self.assertEqual(self.res.files.all().count(),
3,
msg="resource file count didn't match")
super(TestResourceFileFolderOprsAPI, self).resource_file_oprs()
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_pdf_classification(self):
""" files with recognized extensions are classified properly """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add one file to the resource
test_file_handle = open(self.filenames[0], 'r')
hydroshare.add_resource_files(self.res.short_id, test_file_handle)
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
types = get_content_types(self.res)
self.assertTrue(is_equal_to_as_set(types[0], ['Composite', self.content_types[0]]))
self.assertTrue(is_equal_to_as_set(types[1], [])) # no left-over extensions
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def setUp(self):
super(TestGetResourceFile, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
self.user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='test',
last_name='user',
superuser=False)
self.res = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.user,
title='Test File')
test_file = open('test1.txt', 'w')
test_file.write("Test text file in test1.txt")
test_file.close()
self.file = open('test1.txt', 'rb')
hydroshare.add_resource_files(self.res.short_id, self.file)
def setUp(self):
super(TestGetResourceFile, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
self.user = hydroshare.create_account('*****@*****.**',
username='******',
first_name='Tian',
last_name='Gan',
superuser=False,
groups=[])
self.res = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My resource',
metadata=[],
)
open('myfile.txt', "w").close()
self.file = open('myfile.txt', 'r')
hydroshare.add_resource_files(self.res.short_id, self.file)
def setUp(self):
self.user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Tian',
last_name='Gan',
superuser=False,
groups=[]
)
self.res = GenericResource.objects.create(
user=self.user,
title='My resource',
creator=self.user,
last_changed_by=self.user,
)
self.file = open('myfile.txt', "w")
self.file = open('myfile.txt','r')
hydroshare.add_resource_files(self.res.short_id, self.file)
def migrate_tif_file(apps, schema_editor):
# create a vrt file from tif file for each of the Raster Resources
log = logging.getLogger()
istorage = IrodsStorage()
for res in RasterResource.objects.all():
try:
if len(res.files.all()) == 1:
res_file = res.files.all().first()
vrt_file_path = create_vrt_file(res_file.resource_file)
if os.path.isfile(vrt_file_path):
files = (UploadedFile(
file=open(vrt_file_path, 'r'),
name=os.path.basename(vrt_file_path)))
hydroshare.add_resource_files(res.short_id, files)
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
print("Deleted bag for resource ID:" +
str(res.short_id))
resource_modified(res, res.creator)
log.info(
'Tif file conversion to VRT successful for resource:ID:{} '
'Title:{}'.format(res.short_id,
res.metadata.title.value))
else:
log.error(
'Tif file conversion to VRT unsuccessful for resource:ID:{} '
'Title:{}'.format(res.short_id,
res.metadata.title.value))
if os.path.exists(vrt_file_path):
shutil.rmtree(os.path.dirname(vrt_file_path))
except:
pass
def test_weird_extensions(self):
""" files with unrecognized extensions are classified properly """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add all files to the resource
for f in self.weird_filenames:
test_file_handle = open(f, 'r')
hydroshare.add_resource_files(self.res.short_id, test_file_handle)
self.assertEqual(self.res.files.all().count(), 2,
msg="resource file count didn't match")
types = get_content_types(self.res)
# no extensions match content types
self.assertTrue(is_equal_to_as_set(types[0], ['Composite', 'Generic Data']))
# extensions are flagged as not matching
self.assertTrue(is_equal_to_as_set(types[1], self.weird_extensions))
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def setUp(self):
super(TestHideOldVersions, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
# create a user who is the owner of the resource to be versioned
self.owner = hydroshare.create_account('*****@*****.**',
username='******',
first_name='owner_firstname',
last_name='owner_lastname',
superuser=False,
groups=[])
# create a generic resource
self.version0 = hydroshare.create_resource(
resource_type='CompositeResource',
owner=self.owner,
title='Test Composite Resource')
test_file1 = open('test1.txt', 'w')
test_file1.write("Test text file in test1.txt")
test_file1.close()
test_file2 = open('test2.txt', 'w')
test_file2.write("Test text file in test2.txt")
test_file2.close()
self.test_file1 = open('test1.txt', 'rb')
self.test_file2 = open('test2.txt', 'rb')
hydroshare.add_resource_files(self.version0.short_id, self.test_file1,
self.test_file2)
# make one version
self.version1 = hydroshare.create_empty_resource(
self.version0.short_id, self.owner)
self.version1 = hydroshare.create_new_version_resource(
self.version0, self.version1, self.owner)
# and then make a version of that
self.version2 = hydroshare.create_empty_resource(
self.version1.short_id, self.owner)
self.version2 = hydroshare.create_new_version_resource(
self.version1, self.version2, self.owner)
def setUp(self):
super(TestGetResourceFile, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
self.user = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Tian',
last_name='Gan',
superuser=False,
groups=[]
)
self.res = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My resource',
metadata=[],
)
open('myfile.txt', "w").close()
self.file = open('myfile.txt', 'r')
hydroshare.add_resource_files(self.res.short_id, self.file)
def test_file_extension_classification(self):
""" files with recognized extensions are classified properly """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add all files to the resource
for f in self.filenames:
test_file_handle = open(f, 'r')
hydroshare.add_resource_files(self.res.short_id, test_file_handle)
self.assertEqual(self.res.files.all().count(), 5,
msg="resource file count didn't match")
types = get_content_types(self.res)
self.assertTrue(is_equal_to_as_set(
types[0],
['Composite', 'Document', 'Presentation', 'Spreadsheet',
'Jupyter Notebook']))
self.assertTrue(is_equal_to_as_set(types[1], [])) # no left-over extensions
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_resource_operations_in_user_zone(self):
# only do federation testing when REMOTE_USE_IRODS is True and irods docker containers
# are set up properly
if not super(TestUserZoneIRODSFederation,
self).is_federated_irods_available():
return
# test resource creation and "move" option in federated user zone
fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted)
res = resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in User Zone',
source_names=[fed_test_file_full_path],
move=True)
self.assertEqual(res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
user_path = '/{zone}/home/testuser/'.format(
zone=settings.HS_USER_IRODS_ZONE)
self.assertEqual(res.resource_federation_path, fed_path)
# test original file in user test zone is removed after resource creation
# since True is used for move when creating the resource
self.assertFalse(
self.irods_storage.exists(user_path + self.file_to_be_deleted))
# test resource file deletion
res.files.all().delete()
self.assertEqual(res.files.all().count(),
0,
msg="Number of content files is not equal to 0")
# test add multiple files and 'copy' option in federated user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
hydroshare.add_resource_files(
res.short_id,
source_names=[fed_test_file1_full_path, fed_test_file2_full_path],
move=False)
# test resource has two files
self.assertEqual(res.files.all().count(),
2,
msg="Number of content files is not equal to 2")
file_list = []
for f in res.files.all():
file_list.append(f.storage_path.split('/')[-1])
self.assertTrue(
self.file_one in file_list,
msg='file 1 has not been added in the resource in user zone')
self.assertTrue(
self.file_two in file_list,
msg='file 2 has not been added in the resource in user zone')
# test original two files in user test zone still exist after adding them to the resource
# since False is used for move when creating the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
# test resource deletion
resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(),
0,
msg='Number of resources not equal to 0')
# test create new version resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user)
new_res = hydroshare.create_new_version_resource(
ori_res, new_res, self.user)
# only need to test file-related attributes
# ensure new versioned resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path,
new_res.resource_federation_path)
# ensure new versioned resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(),
new_res.files.all().count())
# delete resources to clean up
resource.delete_resource(new_res.short_id)
resource.delete_resource(ori_res.short_id)
# test copy resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id,
self.user,
action='copy')
new_res = hydroshare.copy_resource(ori_res, new_res)
# only need to test file-related attributes
# ensure new copied resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path,
new_res.resource_federation_path)
# ensure new copied resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(),
new_res.files.all().count())
# delete resources to clean up
resource.delete_resource(new_res.short_id)
resource.delete_resource(ori_res.short_id)
# test folder operations in user zone
fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three)
self.res = resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[
fed_file1_full_path, fed_file2_full_path, fed_file3_full_path
],
move=False)
# make sure self.res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(self.res.resource_federation_path, fed_path)
# resource should has only three files at this point
self.assertEqual(self.res.files.all().count(),
3,
msg="resource file count didn't match")
self.file_name_list = [self.file_one, self.file_two, self.file_three]
super(TestUserZoneIRODSFederation, self).resource_file_oprs()
# delete resources to clean up
resource.delete_resource(self.res.short_id)
# test adding files from federated user zone to an empty resource
# created in hydroshare zone
res = resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in HydroShare Zone')
self.assertEqual(res.files.all().count(),
0,
msg="Number of content files is not equal to 0")
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
hydroshare.add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
# test resource has one file
self.assertEqual(res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
file_list = []
for f in res.files.all():
file_list.append(os.path.basename(f.storage_path))
self.assertTrue(
self.file_one in file_list,
msg='file 1 has not been added in the resource in hydroshare zone')
# test original file in user test zone still exist after adding it to the resource
# since 'copy' is used for fed_copy_or_move when adding the file to the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
# test replication of this resource to user zone
hydroshare.replicate_resource_bag_to_user_zone(self.user, res.short_id)
self.assertTrue(self.irods_storage.exists(user_path + res.short_id +
'.zip'),
msg='replicated resource bag is not in the user zone')
# test resource deletion
resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(),
0,
msg='Number of resources not equal to 0')
# test to make sure original file still exist after resource deletion
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_resource_operations_in_user_zone(self):
super(TestUserZoneIRODSFederation, self).assert_federated_irods_available()
# test resource creation and "move" option in federated user zone
fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted)
res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in User Zone',
source_names=[fed_test_file_full_path],
move=True
)
self.assertEqual(res.files.all().count(), 1,
msg="Number of content files is not equal to 1")
fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
user_path = '/{zone}/home/testuser/'.format(zone=settings.HS_USER_IRODS_ZONE)
self.assertEqual(res.resource_federation_path, fed_path)
# test original file in user test zone is removed after resource creation
# since True is used for move when creating the resource
file_path_name = user_path + self.file_to_be_deleted
self.assertFalse(self.irods_storage.exists(file_path_name))
# test django_irods CopyFiles() with an iRODS resource name being passed in
# as input parameter to verify the file gets copied to the pass-in iRODS resource
istorage = res.get_irods_storage()
src_path = os.path.join(res.root_path, 'data', 'contents', self.file_to_be_deleted)
dest_path = file_path_name
istorage.copyFiles(src_path, dest_path, settings.HS_IRODS_LOCAL_ZONE_DEF_RES)
# assert file did get copied over
self.assertTrue(self.irods_storage.exists(file_path_name))
stdout = self.irods_storage.session.run("ils", None, "-l", file_path_name)[0].split()
# assert copied file gets written to the iRODS resource being passed into copyFiles() call
self.assertEqual(stdout[2], settings.HS_IRODS_LOCAL_ZONE_DEF_RES)
# test resource file deletion
res.files.all().delete()
self.assertEqual(res.files.all().count(), 0,
msg="Number of content files is not equal to 0")
# test add multiple files and 'copy' option in federated user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
hydroshare.add_resource_files(
res.short_id,
source_names=[fed_test_file1_full_path, fed_test_file2_full_path],
move=False)
# test resource has two files
self.assertEqual(res.files.all().count(), 2,
msg="Number of content files is not equal to 2")
file_list = []
for f in res.files.all():
file_list.append(f.storage_path.split('/')[-1])
self.assertTrue(self.file_one in file_list,
msg='file 1 has not been added in the resource in user zone')
self.assertTrue(self.file_two in file_list,
msg='file 2 has not been added in the resource in user zone')
# test original two files in user test zone still exist after adding them to the resource
# since False is used for move when creating the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
# test resource deletion
hydroshare.resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(), 0,
msg='Number of resources not equal to 0')
# test create new version resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False
)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(), 1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user)
new_res = hydroshare.create_new_version_resource(ori_res, new_res, self.user)
# only need to test file-related attributes
# ensure new versioned resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path)
# ensure new versioned resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(), new_res.files.all().count())
# delete resources to clean up
hydroshare.resource.delete_resource(new_res.short_id)
hydroshare.resource.delete_resource(ori_res.short_id)
# test copy resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False
)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(), 1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user, action='copy')
new_res = hydroshare.copy_resource(ori_res, new_res)
# only need to test file-related attributes
# ensure new copied resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path, new_res.resource_federation_path)
# ensure new copied resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(), new_res.files.all().count())
# delete resources to clean up
hydroshare.resource.delete_resource(new_res.short_id)
hydroshare.resource.delete_resource(ori_res.short_id)
# test folder operations in user zone
fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three)
self.res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_file1_full_path, fed_file2_full_path, fed_file3_full_path],
move=False
)
# make sure self.res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(self.res.resource_federation_path, fed_path)
# resource should has only three files at this point
self.assertEqual(self.res.files.all().count(), 3,
msg="resource file count didn't match")
self.file_name_list = [self.file_one, self.file_two, self.file_three]
super(TestUserZoneIRODSFederation, self).resource_file_oprs()
# delete resources to clean up
hydroshare.resource.delete_resource(self.res.short_id)
# test adding files from federated user zone to an empty resource
# created in hydroshare zone
res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in HydroShare Zone'
)
self.assertEqual(res.files.all().count(), 0,
msg="Number of content files is not equal to 0")
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
hydroshare.add_resource_files(
res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
# test resource has one file
self.assertEqual(res.files.all().count(), 1,
msg="Number of content files is not equal to 1")
file_list = []
for f in res.files.all():
file_list.append(os.path.basename(f.storage_path))
self.assertTrue(self.file_one in file_list,
msg='file 1 has not been added in the resource in hydroshare zone')
# test original file in user test zone still exist after adding it to the resource
# since 'copy' is used for fed_copy_or_move when adding the file to the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
# test replication of this resource to user zone even if the bag_modified AVU for this
# resource is wrongly set to False when the bag for this resource does not exist and
# need to be recreated
res.setAVU('bag_modified', 'false')
hydroshare.resource.replicate_resource_bag_to_user_zone(self.user, res.short_id)
self.assertTrue(self.irods_storage.exists(user_path + res.short_id + '.zip'),
msg='replicated resource bag is not in the user zone')
# test resource deletion
hydroshare.resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(), 0,
msg='Number of resources not equal to 0')
# test to make sure original file still exist after resource deletion
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_federated_folder_path_logic(self):
""" a federated file in a subfolder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
ResourceFile.create_folder(self.res, 'foo')
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo')
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
self.assertEqual(resfile.resource_file.name, os.path.join(self.res.short_id,
"data", "contents",
"foo", "file1.txt"))
self.assertEqual(resfile.file_folder, "foo")
# cheat: set a fake federated path to test path logic
fedpath = "/myzone/home/myuser"
self.res.resource_federation_path = fedpath
self.res.save()
resfile.content_object.refresh_from_db()
resfile.set_storage_path('foo/file1.txt', test_exists=False)
# determine where that file should live
shortpath = os.path.join(fedpath, self.res.short_id, "data",
"contents", "foo", "file1.txt")
self.assertEqual(shortpath, resfile.storage_path)
# this should result in an exact path
resfile.set_storage_path(shortpath, test_exists=False)
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False))
# non-existent files should raise error
otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file2.txt")
resfile.path_is_acceptable(otherpath, test_exists=False)
# This won't work because federation path is fake.
# with self.assertRaises(ValidationError):
# resfile.path_is_acceptable(otherpath, test_exists=True)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("foo/file1.txt", test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath, test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
resfile.set_storage_path(otherpath, test_exists=False)
# conclusion: unfederate the resource
self.res.resource_federation_path = ""
self.res.save()
resfile.content_object.refresh_from_db()
resfile.set_storage_path("foo/file1.txt", test_exists=False)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir', store[0], msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg="resource does not contain the updated file new_" + file_name_list[2])
self.assertNotIn(file_name_list[2], updated_res_file_names,
msg='resource still contains the old file ' + file_name_list[2] +
' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] + ' moved to a folder')
self.assertNotIn(file_name_list[0], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[0] +
'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1], updated_res_file_names,
msg='resource still contains the old ' + file_name_list[1] +
' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(), 2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, uname=user.username, fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id, source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', False)
# Resource should still contain 5 files: file3_new.txt (2), sub_test_dir.zip,
# and file1.txt (2)
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 5, msg="resource file count didn't match - " +
str(file_cnt) + " != 5")
# remove all files except the zippped file
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
remove_folder(user, res.short_id, 'data/contents/sub_test_dir-1')
# Now resource should contain two files: file3_new.txt sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt, 2, msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(), 3, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir.zip', updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[0])
self.assertIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain unzipped file ' + file_name_list[1])
self.assertIn('new_' + file_name_list[2], updated_res_file_names,
msg='resource does not contain unzipped file new_' + file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[0], updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0], updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/sub_test_dir/' + file_name_list[1], updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1], updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(), 1, msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def migrate_tif_file(apps, schema_editor):
log = logging.getLogger()
istorage = IrodsStorage()
copy_res_fail = []
vrt_update_fail = []
vrt_update_success = []
meta_update_fail = []
meta_update_success = []
# start migration for each raster resource that has raster files
for res in RasterResource.objects.all():
if res.files.all():
# copy all the resource files to temp dir
try:
temp_dir = tempfile.mkdtemp()
for res_file in res.files.all():
shutil.copy(res_file.resource_file.file.name,
os.path.join(temp_dir, os.path.basename(res_file.resource_file.name)))
vrt_file_path = [os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.vrt' == f[-4:]].pop()
except Exception as e:
log.exception(e.message)
copy_res_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value))
continue
# update vrt file if the raster resource that has a single tif file
try:
if len(os.listdir(temp_dir)) == 2:
# create new vrt file
tif_file_path = [os.path.join(temp_dir, f) for f in os.listdir(temp_dir) if '.tif' == f[-4:]].pop()
with open(os.devnull, 'w') as fp:
subprocess.Popen(['gdal_translate', '-of', 'VRT', tif_file_path, vrt_file_path], stdout=fp, stderr=fp).wait() # remember to add .wait()
# modify the vrt file contents
tree = ET.parse(vrt_file_path)
root = tree.getroot()
for element in root.iter('SourceFilename'):
element.attrib['relativeToVRT'] = '1'
tree.write(vrt_file_path)
# delete vrt res file
for f in res.files.all():
if 'vrt' == f.resource_file.name[-3:]:
f.resource_file.delete()
f.delete()
# add new vrt file to resource
new_file = UploadedFile(file=open(vrt_file_path, 'r'), name=os.path.basename(vrt_file_path))
hydroshare.add_resource_files(res.short_id, new_file)
# update the bag
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
resource_modified(res, res.creator)
vrt_update_success.append('{}:{}'.format(res.short_id,res.metadata.title.value))
except Exception as e:
log.exception(e.message)
vrt_update_fail.append('{}:{}'.format(res.short_id,res.metadata.title.value))
# update the metadata for the band information of all the raster resources
try:
meta_updated = False
# extract meta
ori_dir = os.getcwd()
os.chdir(temp_dir)
res_md_dict = raster_meta_extract.get_raster_meta_dict(vrt_file_path)
os.chdir(ori_dir)
shutil.rmtree(temp_dir)
# update band information metadata in django
if res_md_dict['band_info']:
for i, band_meta in res_md_dict['band_info'].items():
band_obj = res.metadata.bandInformation.filter(name='Band_{}'.format(i)).first()
if band_obj:
res.metadata.update_element('bandInformation',
band_obj.id,
maximumValue=band_meta['maximumValue'],
minimumValue=band_meta['minimumValue'],
noDataValue=band_meta['noDataValue'],
)
meta_updated = True
# update the bag if meta is updated
if meta_updated:
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
resource_modified(res, res.creator)
meta_update_success.append('{}:{}'.format(res.short_id, res.metadata.title.value))
except Exception as e:
log.exception(e.message)
meta_update_fail.append('{}:{}'.format(res.short_id, res.metadata.title.value))
# Print migration results
print 'Copy resource to temp folder failure: Number: {} List: {}'.format(len(copy_res_fail), copy_res_fail)
print 'VRT file update success: Number: {} List{}'.format(len(vrt_update_success), vrt_update_success)
print 'VRT file update fail: Number: {} List{}'.format(len(vrt_update_fail), vrt_update_fail)
print 'Meta update success: Number: {} List {}'.format(len(meta_update_success), meta_update_success)
print 'Meta update fail: Number: {} List {}'.format(len(meta_update_fail), meta_update_fail)
def test_federated_root_path_logic(self):
""" a federated file path in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# cheat: set a fake federated path to test path logic
oldfedpath = self.res.resource_federation_path
oldpath = resfile.storage_path
# intentionally break path logic by setting an unused federation path
fedpath = "/myzone/home/myuser"
self.res.resource_federation_path = fedpath
self.res.save()
# must load changes into resfile from self.res before setting storage path
resfile.content_object.refresh_from_db()
resfile.set_storage_path('file1.txt', test_exists=False)
self.assertEqual(self.res.resource_federation_path, fedpath)
self.assertEqual(resfile.storage_path, get_path(resfile, 'file1.txt'))
# determine where that file should live; THIS IS FAKE
shortpath = os.path.join(fedpath, self.res.short_id, "data",
"contents", "file1.txt")
# intentionally break the resource file path
resfile.set_storage_path(shortpath, test_exists=False)
self.assertEqual(shortpath, resfile.storage_path)
self.assertEqual(resfile.file_folder, '')
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False))
otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file2.txt")
resfile.path_is_acceptable(otherpath, test_exists=False)
# non-existent files should raise error
# This won't work because federation path is fake
# with self.assertRaises(ValidationError):
# resfile.path_is_acceptable(otherpath, test_exists=True)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("file1.txt", test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, '')
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath, test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, '')
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
resfile.set_storage_path(otherpath, test_exists=False)
# conclusion: strip off federation path
self.res.resource_federation_path = oldfedpath
self.res.save()
# must load changes into resfile from self.res before setting storage path
resfile.content_object.refresh_from_db()
resfile.set_storage_path(oldpath, test_exists=False)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_get_revisions(self):
# create a user to be used for creating the resource
user_creator = hydroshare.create_account(
"*****@*****.**",
username="******",
first_name="Creator_FirstName",
last_name="Creator_LastName",
superuser=False,
groups=[],
)
resource_changed_by = hydroshare.create_account(
"*****@*****.**",
username="******",
first_name="Pabitra",
last_name="Dash",
superuser=False,
groups=[],
)
# create a resource
resource = hydroshare.create_resource("GenericResource", user_creator, "My resource")
# test that we have only one revision at this point - this is the api call we are testing
res_revisions = hydroshare.get_revisions(resource.short_id)
self.assertEqual(len(res_revisions), 1)
# set the resource last changed by a different user - to a create another revision of the resource
hydroshare.utils.resource_modified(resource, resource_changed_by)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 2 revisions
self.assertEqual(len(res_revisions), 2)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
# test that each resource revision has the same resource id
for bags in res_revisions:
self.assertEqual(resource.id, bags.object_id)
# add a file to the resource to generate another revision of the resource
# create a file
original_file_name = "original.txt"
original_file = open(original_file_name, "w")
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, "r")
# add the file to the resource
hydroshare.add_resource_files(resource.short_id, original_file)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 3 revisions
self.assertEqual(len(res_revisions), 3)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp)
self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp)
# delete the file in the resource to create another revision of the resource
hydroshare.delete_resource_file(resource.short_id, original_file_name)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 4 revisions
self.assertEqual(len(res_revisions), 4)
def test_unfederated_root_path_checks(self):
""" an unfederated file in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
self.res.check_irods_files(stop_on_error=True)
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# should succeed without errors
self.res.check_irods_files(stop_on_error=True)
# cleaning should not change anything
self.res.check_irods_files(stop_on_error=True, log_errors=False, return_errors=True,
clean_irods=True, clean_django=True, sync_ispublic=True)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data", "contents", "file1.txt")
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# now try to intentionally corrupt it
resfile.set_short_path("fuzz.txt")
# should raise exception
with self.assertRaises(ValidationError):
self.res.check_irods_files(stop_on_error=True)
# now don't raise exception and read error
errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# now try to clean it up
errors, ecount = self.res.check_irods_files(return_errors=True, log_errors=False,
clean_irods=True, clean_django=True)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# now check should succeed
errors, ecount = self.res.check_irods_files(stop_on_error=True, log_errors=False)
self.assertEqual(ecount, 0)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_get_revisions(self):
url = 'hsapi/revisions/{0}/'.format(self.res.short_id)
resp = self.api_client.get(url)
res_revisions = self.deserialize(resp)
self.assertValidJSONResponse(resp)
self.assertEqual(len(res_revisions), 1)
self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions)
resource_changed_by = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='User1_FirstName',
last_name='User1_LastName'
)
hydroshare.utils.resource_modified(self.res, resource_changed_by)
resp = self.api_client.get(url)
res_revisions = self.deserialize(resp)
self.assertValidJSONResponse(resp)
self.assertEqual(len(res_revisions), 2)
self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
# test that each resource revision has the same resource id
for bags in res_revisions:
self.assertEqual(self.res.id, bags.object_id)
# add a file to the resource to generate another revision of the resource
# create a file
original_file_name = 'original.txt'
original_file = open(original_file_name, 'w')
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, 'r')
# add the file to the resource
hydroshare.add_resource_files(self.res.short_id, original_file)
resp = self.api_client.get(url)
res_revisions = self.deserialize(resp)
self.assertValidJSONResponse(resp)
# test that we now have 3 revisions
self.assertEqual(len(res_revisions), 3)
self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp)
self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp)
# delete the file in the resource to create another revision of the resource
hydroshare.delete_resource_file(self.res.short_id, original_file_name)
resp = self.api_client.get(url)
res_revisions = self.deserialize(resp)
self.assertValidJSONResponse(resp)
self.assertEqual(hydroshare.get_revisions(self.res.short_id), res_revisions)
# test that we now have 4 revisions
self.assertEqual(len(res_revisions), 4)
def setUp(self):
super(TestCopyResource, self).setUp()
self.group, _ = Group.objects.get_or_create(name='Hydroshare Author')
# create a user who is the owner of the resource to be copied
self.owner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='owner_firstname',
last_name='owner_lastname',
superuser=False,
groups=[]
)
# create a user who is NOT the owner of the resource to be copied
self.nonowner = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='nonowner_firstname',
last_name='nonowner_lastname',
superuser=False,
groups=[]
)
# create a generic resource
self.res_generic = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.owner,
title='Test Generic Resource'
)
test_file1 = open('test1.txt', 'w')
test_file1.write("Test text file in test1.txt")
test_file1.close()
test_file2 = open('test2.txt', 'w')
test_file2.write("Test text file in test2.txt")
test_file2.close()
self.test_file1 = open('test1.txt', 'r')
self.test_file2 = open('test2.txt', 'r')
hydroshare.add_resource_files(self.res_generic.short_id, self.test_file1, self.test_file2)
# create a generic empty resource with one license that prohibits derivation
statement = 'This resource is shared under the Creative Commons Attribution-NoDerivs CC ' \
'BY-ND.'
url = 'http://creativecommons.org/licenses/by-nd/4.0/'
metadata = []
metadata.append({'rights': {'statement': statement, 'url': url}})
self.res_generic_lic_nd = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.owner,
title='Test Generic Resource',
metadata=metadata
)
# create a generic empty resource with another license that prohibits derivation
statement = 'This resource is shared under the Creative Commons ' \
'Attribution-NoCommercial-NoDerivs CC BY-NC-ND.'
url = 'http://creativecommons.org/licenses/by-nc-nd/4.0/'
metadata = []
metadata.append({'rights': {'statement': statement, 'url': url}})
self.res_generic_lic_nc_nd = hydroshare.create_resource(
resource_type='GenericResource',
owner=self.owner,
title='Test Generic Resource',
metadata=metadata
)
# create a raster resource that represents a specific resource type
raster_file = 'hs_core/tests/data/cea.tif'
temp_dir = tempfile.mkdtemp()
self.temp_raster_file = os.path.join(temp_dir, 'cea.tif')
shutil.copy(raster_file, self.temp_raster_file)
self.raster_obj = open(self.temp_raster_file, 'r')
files = [UploadedFile(file=self.raster_obj, name='cea.tif')]
self.res_raster = hydroshare.create_resource(
resource_type='RasterResource',
owner=self.owner,
title='Test Raster Resource',
files=files,
metadata=[]
)
# call the post creation process here for the metadata to be
# extracted
utils.resource_post_create_actions(resource=self.res_raster, user=self.owner, metadata=[])
def test_federated_root_path_logic(self):
""" a federated file path in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# cheat: set a fake federated path to test path logic
oldfedpath = self.res.resource_federation_path
oldpath = resfile.storage_path
# intentionally break path logic by setting an unused federation path
fedpath = "/myzone/home/myuser"
self.res.resource_federation_path = fedpath
self.res.save()
# must load changes into resfile from self.res before setting storage path
resfile.content_object.refresh_from_db()
resfile.set_storage_path('file1.txt', test_exists=False)
self.assertEqual(self.res.resource_federation_path, fedpath)
self.assertEqual(resfile.storage_path, get_path(resfile, 'file1.txt'))
# determine where that file should live; THIS IS FAKE
shortpath = os.path.join(fedpath, self.res.short_id, "data",
"contents", "file1.txt")
# intentionally break the resource file path
resfile.set_storage_path(shortpath, test_exists=False)
self.assertEqual(shortpath, resfile.storage_path)
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False))
otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "file2.txt")
resfile.path_is_acceptable(otherpath, test_exists=False)
# non-existent files should raise error
# This won't work because federation path is fake
# with self.assertRaises(ValidationError):
# resfile.path_is_acceptable(otherpath, test_exists=True)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("file1.txt", test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath, test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
resfile.set_storage_path(otherpath, test_exists=False)
# conclusion: strip off federation path
self.res.resource_federation_path = oldfedpath
self.res.save()
# must load changes into resfile from self.res before setting storage path
resfile.content_object.refresh_from_db()
resfile.set_storage_path(oldpath, test_exists=False)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_federated_folder_path_logic(self):
""" a federated file in a subfolder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
ResourceFile.create_folder(self.res, 'foo')
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1, folder='foo')
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
self.assertEqual(resfile.resource_file.name, os.path.join(self.res.short_id,
"data", "contents",
"foo", "file1.txt"))
self.assertEqual(resfile.file_folder, "foo")
# cheat: set a fake federated path to test path logic
fedpath = "/myzone/home/myuser"
self.res.resource_federation_path = fedpath
self.res.save()
resfile.content_object.refresh_from_db()
resfile.set_storage_path('foo/file1.txt', test_exists=False)
# determine where that file should live
shortpath = os.path.join(fedpath, self.res.short_id, "data",
"contents", "foo", "file1.txt")
self.assertEqual(shortpath, resfile.storage_path)
# this should result in an exact path
resfile.set_storage_path(shortpath, test_exists=False)
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath, test_exists=False))
# non-existent files should raise error
otherpath = os.path.join(fedpath, self.res.short_id, "data", "contents", "foo", "file2.txt")
resfile.path_is_acceptable(otherpath, test_exists=False)
# This won't work because federation path is fake.
# with self.assertRaises(ValidationError):
# resfile.path_is_acceptable(otherpath, test_exists=True)
# try setting to an unqualified name; should qualify it
resfile.set_storage_path("foo/file1.txt", test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to what it is already
resfile.set_storage_path(shortpath, test_exists=False)
# should match computed path
self.assertEqual(resfile.file_folder, "foo")
self.assertEqual(resfile.storage_path, shortpath)
# now try to change that path to a good path to a non-existent object
resfile.set_storage_path(otherpath, test_exists=False)
# conclusion: unfederate the resource
self.res.resource_federation_path = ""
self.res.save()
resfile.content_object.refresh_from_db()
resfile.set_storage_path("foo/file1.txt", test_exists=False)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_update_resource_file(self):
# create a user to be used for creating the resource
user_creator = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[]
)
# create a resource without any owner
resource = GenericResource.objects.create(
user=user_creator,
title='My resource',
creator=user_creator,
last_changed_by=user_creator,
doi='doi1000100010001'
)
# resource should not have any files at this point
self.assertEqual(resource.files.all().count(), 0, msg="resource file count didn't match")
# create a file
original_file_name = 'original.txt'
original_file = open(original_file_name, 'w')
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, 'r')
# add the file to the resource
added_files = hydroshare.add_resource_files(resource.short_id, original_file)
# resource should have only one file at this point
self.assertEqual(len(added_files), 1)
self.assertEqual(resource.files.all().count(), 1, msg="resource file count didn't match")
self.assertIn(
original_file_name,
[os.path.basename(f.resource_file.name) for f in resource.files.all()],
msg= '%s is not one of the resource files.' % original_file_name
)
# create a file that will be used to update the original file -1st update
new_file_name = 'update.txt' # file has a different name from the file that we will be updating
new_file = open(new_file_name, 'w')
new_file_data = 'data in new file'
new_file.write(new_file_data)
new_file.close()
new_file = open(new_file_name, 'r')
# this is the api call we are testing
rf = hydroshare.update_resource_file(resource.short_id, original_file_name, new_file)
# test if the file name matches
self.assertEqual(os.path.basename(rf.resource_file.name), new_file_name, msg="resource file name didn't match")
# since we are updating a file the number of files in the resource needs to be still 1
self.assertEqual(resource.files.all().count(), 1, msg="resource file count didn't match")
# test if the content of the file matches
resource_file = hydroshare.get_resource_file(resource.short_id, new_file_name)
self.assertEqual(resource_file.read(), new_file_data, msg="resource file content didn't match")
# reset the original resource file name for 2nd time resource file update
original_file_name = new_file_name
# create a file that will be used to update the resource file - 2nd update
new_file_name = 'update.txt' # file has the same name as the file that we will be updating
new_file = open(new_file_name, 'w')
new_file_data = 'data in new file'
new_file.write(new_file_data)
new_file.close()
new_file = open(new_file_name, 'r')
# this is the api call we are testing
rf = hydroshare.update_resource_file(resource.short_id, original_file_name, new_file)
# test if the file name matches
self.assertEqual(os.path.basename(rf.resource_file.name), new_file_name,
msg="{0} != {1}".format(os.path.basename(rf.resource_file.name), new_file_name))
# exception ObjectDoesNotExist should be raised if resource does not have a file
# for the given file name (file_not_in_resource.txt) to update
self.assertRaises(
ObjectDoesNotExist,
lambda: hydroshare.update_resource_file(resource.short_id, 'file_not_in_resource.txt', new_file)
)
def migrate_tif_file(apps, schema_editor):
log = logging.getLogger()
istorage = IrodsStorage()
copy_res_fail = []
vrt_update_fail = []
vrt_update_success = []
meta_update_fail = []
meta_update_success = []
# start migration for each raster resource that has raster files
for res in RasterResource.objects.all():
if res.files.all():
# copy all the resource files to temp dir
try:
temp_dir = tempfile.mkdtemp()
for res_file in res.files.all():
shutil.copy(
res_file.resource_file.file.name,
os.path.join(
temp_dir,
os.path.basename(res_file.resource_file.name)))
vrt_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.vrt' == f[-4:]
].pop()
except Exception as e:
log.exception(e.message)
copy_res_fail.append('{}:{}'.format(res.short_id,
res.metadata.title.value))
continue
# update vrt file if the raster resource that has a single tif file
try:
if len(os.listdir(temp_dir)) == 2:
# create new vrt file
tif_file_path = [
os.path.join(temp_dir, f) for f in os.listdir(temp_dir)
if '.tif' == f[-4:]
].pop()
with open(os.devnull, 'w') as fp:
subprocess.Popen(
[
'gdal_translate', '-of', 'VRT', tif_file_path,
vrt_file_path
],
stdout=fp,
stderr=fp).wait() # remember to add .wait()
# modify the vrt file contents
tree = ET.parse(vrt_file_path)
root = tree.getroot()
for element in root.iter('SourceFilename'):
element.attrib['relativeToVRT'] = '1'
tree.write(vrt_file_path)
# delete vrt res file
for f in res.files.all():
if 'vrt' == f.resource_file.name[-3:]:
f.resource_file.delete()
f.delete()
# add new vrt file to resource
new_file = UploadedFile(
file=open(vrt_file_path, 'r'),
name=os.path.basename(vrt_file_path))
hydroshare.add_resource_files(res.short_id, new_file)
# update the bag
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
resource_modified(res, res.creator)
vrt_update_success.append('{}:{}'.format(
res.short_id, res.metadata.title.value))
except Exception as e:
log.exception(e.message)
vrt_update_fail.append('{}:{}'.format(
res.short_id, res.metadata.title.value))
# update the metadata for the band information of all the raster resources
try:
meta_updated = False
# extract meta
ori_dir = os.getcwd()
os.chdir(temp_dir)
res_md_dict = raster_meta_extract.get_raster_meta_dict(
vrt_file_path)
os.chdir(ori_dir)
shutil.rmtree(temp_dir)
# update band information metadata in django
if res_md_dict['band_info']:
for i, band_meta in res_md_dict['band_info'].items():
band_obj = res.metadata.bandInformation.filter(
name='Band_{}'.format(i)).first()
if band_obj:
res.metadata.update_element(
'bandInformation',
band_obj.id,
maximumValue=band_meta['maximumValue'],
minimumValue=band_meta['minimumValue'],
noDataValue=band_meta['noDataValue'],
)
meta_updated = True
# update the bag if meta is updated
if meta_updated:
bag_name = 'bags/{res_id}.zip'.format(res_id=res.short_id)
if istorage.exists(bag_name):
# delete the resource bag as the old bag is not valid
istorage.delete(bag_name)
resource_modified(res, res.creator)
meta_update_success.append('{}:{}'.format(
res.short_id, res.metadata.title.value))
except Exception as e:
log.exception(e.message)
meta_update_fail.append('{}:{}'.format(
res.short_id, res.metadata.title.value))
# Print migration results
print 'Copy resource to temp folder failure: Number: {} List: {}'.format(
len(copy_res_fail), copy_res_fail)
print 'VRT file update success: Number: {} List{}'.format(
len(vrt_update_success), vrt_update_success)
print 'VRT file update fail: Number: {} List{}'.format(
len(vrt_update_fail), vrt_update_fail)
print 'Meta update success: Number: {} List {}'.format(
len(meta_update_success), meta_update_success)
print 'Meta update fail: Number: {} List {}'.format(
len(meta_update_fail), meta_update_fail)
def test_resource_operations_in_user_zone(self):
super(TestUserZoneIRODSFederation,
self).assert_federated_irods_available()
# test resource creation and "move" option in federated user zone
fed_test_file_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_to_be_deleted)
res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in User Zone',
source_names=[fed_test_file_full_path],
move=True)
self.assertEqual(res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
user_path = '/{zone}/home/testuser/'.format(
zone=settings.HS_USER_IRODS_ZONE)
self.assertEqual(res.resource_federation_path, fed_path)
# test original file in user test zone is removed after resource creation
# since True is used for move when creating the resource
file_path_name = user_path + self.file_to_be_deleted
self.assertFalse(self.irods_storage.exists(file_path_name))
# test django_irods CopyFiles() with an iRODS resource name being passed in
# as input parameter to verify the file gets copied to the pass-in iRODS resource
istorage = res.get_irods_storage()
src_path = os.path.join(res.root_path, 'data', 'contents',
self.file_to_be_deleted)
dest_path = file_path_name
istorage.copyFiles(src_path, dest_path,
settings.HS_IRODS_LOCAL_ZONE_DEF_RES)
# assert file did get copied over
self.assertTrue(self.irods_storage.exists(file_path_name))
stdout = self.irods_storage.session.run("ils", None, "-l",
file_path_name)[0].split()
# assert copied file gets written to the iRODS resource being passed into copyFiles() call
self.assertEqual(stdout[2], settings.HS_IRODS_LOCAL_ZONE_DEF_RES)
# test resource file deletion
res.files.all().delete()
self.assertEqual(res.files.all().count(),
0,
msg="Number of content files is not equal to 0")
# test add multiple files and 'copy' option in federated user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_test_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
hydroshare.add_resource_files(
res.short_id,
source_names=[fed_test_file1_full_path, fed_test_file2_full_path],
move=False)
# test resource has two files
self.assertEqual(res.files.all().count(),
2,
msg="Number of content files is not equal to 2")
file_list = []
for f in res.files.all():
file_list.append(f.storage_path.split('/')[-1])
self.assertTrue(
self.file_one in file_list,
msg='file 1 has not been added in the resource in user zone')
self.assertTrue(
self.file_two in file_list,
msg='file 2 has not been added in the resource in user zone')
# test original two files in user test zone still exist after adding them to the resource
# since False is used for move when creating the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
self.assertTrue(self.irods_storage.exists(user_path + self.file_two))
# test resource deletion
hydroshare.resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(),
0,
msg='Number of resources not equal to 0')
# test create new version resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id, self.user)
new_res = hydroshare.create_new_version_resource(
ori_res, new_res, self.user)
# only need to test file-related attributes
# ensure new versioned resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path,
new_res.resource_federation_path)
# ensure new versioned resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(),
new_res.files.all().count())
# delete resources to clean up
hydroshare.resource.delete_resource(new_res.short_id)
hydroshare.resource.delete_resource(ori_res.short_id)
# test copy resource in user zone
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
ori_res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[fed_test_file1_full_path],
move=False)
# make sure ori_res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(ori_res.resource_federation_path, fed_path)
self.assertEqual(ori_res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
new_res = hydroshare.create_empty_resource(ori_res.short_id,
self.user,
action='copy')
new_res = hydroshare.copy_resource(ori_res, new_res)
# only need to test file-related attributes
# ensure new copied resource is created in the same federation zone as original resource
self.assertEqual(ori_res.resource_federation_path,
new_res.resource_federation_path)
# ensure new copied resource has the same number of content files as original resource
self.assertEqual(ori_res.files.all().count(),
new_res.files.all().count())
# delete resources to clean up
hydroshare.resource.delete_resource(new_res.short_id)
hydroshare.resource.delete_resource(ori_res.short_id)
# test folder operations in user zone
fed_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
fed_file2_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_two)
fed_file3_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_three)
self.res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Original Generic Resource in User Zone',
source_names=[
fed_file1_full_path, fed_file2_full_path, fed_file3_full_path
],
move=False)
# make sure self.res is created in federated user zone
fed_path = '/{zone}/home/{user}'.format(
zone=settings.HS_USER_IRODS_ZONE,
user=settings.HS_LOCAL_PROXY_USER_IN_FED_ZONE)
self.assertEqual(self.res.resource_federation_path, fed_path)
# resource should has only three files at this point
self.assertEqual(self.res.files.all().count(),
3,
msg="resource file count didn't match")
self.file_name_list = [self.file_one, self.file_two, self.file_three]
super(TestUserZoneIRODSFederation, self).resource_file_oprs()
# delete resources to clean up
hydroshare.resource.delete_resource(self.res.short_id)
# test adding files from federated user zone to an empty resource
# created in hydroshare zone
res = hydroshare.resource.create_resource(
resource_type='GenericResource',
owner=self.user,
title='My Test Generic Resource in HydroShare Zone')
self.assertEqual(res.files.all().count(),
0,
msg="Number of content files is not equal to 0")
fed_test_file1_full_path = '/{zone}/home/testuser/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE, fname=self.file_one)
hydroshare.add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
# test resource has one file
self.assertEqual(res.files.all().count(),
1,
msg="Number of content files is not equal to 1")
file_list = []
for f in res.files.all():
file_list.append(os.path.basename(f.storage_path))
self.assertTrue(
self.file_one in file_list,
msg='file 1 has not been added in the resource in hydroshare zone')
# test original file in user test zone still exist after adding it to the resource
# since 'copy' is used for fed_copy_or_move when adding the file to the resource
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
# test replication of this resource to user zone even if the bag_modified AVU for this
# resource is wrongly set to False when the bag for this resource does not exist and
# need to be recreated
res.setAVU('bag_modified', 'false')
hydroshare.resource.replicate_resource_bag_to_user_zone(
self.user, res.short_id)
self.assertTrue(self.irods_storage.exists(user_path + res.short_id +
'.zip'),
msg='replicated resource bag is not in the user zone')
# test resource deletion
hydroshare.resource.delete_resource(res.short_id)
self.assertEquals(BaseResource.objects.all().count(),
0,
msg='Number of resources not equal to 0')
# test to make sure original file still exist after resource deletion
self.assertTrue(self.irods_storage.exists(user_path + self.file_one))
def test_get_revisions(self):
# create a user to be used for creating the resource
user_creator = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Creator_FirstName',
last_name='Creator_LastName',
superuser=False,
groups=[]
)
resource_changed_by = hydroshare.create_account(
'*****@*****.**',
username='******',
first_name='Pabitra',
last_name='Dash',
superuser=False,
groups=[]
)
# create a resource
resource = hydroshare.create_resource('GenericResource', user_creator, 'My resource')
# test that we have only one revision at this point - this is the api call we are testing
res_revisions = hydroshare.get_revisions(resource.short_id)
self.assertEqual(len(res_revisions), 1)
# set the resource last changed by a different user - to a create another revision of the resource
hydroshare.utils.resource_modified(resource, resource_changed_by)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 2 revisions
self.assertEqual(len(res_revisions), 2)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
# test that each resource revision has the same resource id
for bags in res_revisions:
self.assertEqual(resource.id, bags.object_id)
# add a file to the resource to generate another revision of the resource
# create a file
original_file_name = 'original.txt'
original_file = open(original_file_name, 'w')
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, 'r')
# add the file to the resource
hydroshare.add_resource_files(resource.short_id, original_file)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 3 revisions
self.assertEqual(len(res_revisions), 3)
# test that each revision has a different time stamp
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[1].timestamp)
self.assertNotEqual(res_revisions[0].timestamp, res_revisions[2].timestamp)
self.assertNotEqual(res_revisions[1].timestamp, res_revisions[2].timestamp)
# delete the file in the resource to create another revision of the resource
hydroshare.delete_resource_file(resource.short_id, original_file_name)
res_revisions = hydroshare.get_revisions(resource.short_id)
# test that we now have 4 revisions
self.assertEqual(len(res_revisions), 4)
def resource_file_oprs(self):
"""Test common iRODS file operations.
This is a common test utility function to be called by both regular folder operation
testing and federated zone folder operation testing.
Make sure the calling TestCase object has the following attributes defined before calling
this method:
self.res: resource that has been created that contains files listed in file_name_list
self.user: owner of the resource
self.file_name_list: a list of three file names that have been added to the res object
self.test_file_1 needs to be present for the calling object for doing regular folder
operations without involving federated zone so that the same opened file can be re-added
to the resource for testing the case where zipping cannot overwrite existing file
"""
user = self.user
res = self.res
file_name_list = self.file_name_list
# create a folder, if folder is created successfully, no exception is raised, otherwise,
# an iRODS exception will be raised which will be caught by the test runner and mark as
# a test failure
create_folder(res.short_id, 'data/contents/sub_test_dir')
istorage = res.get_irods_storage()
res_path = res.file_path
store = istorage.listdir(res_path)
self.assertIn('sub_test_dir',
store[0],
msg='resource does not contain created sub-folder')
# rename the third file in file_name_list
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/' + file_name_list[2],
'data/contents/new_' + file_name_list[2])
# move the first two files in file_name_list to the new folder
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[1],
'data/contents/sub_test_dir/' + file_name_list[1])
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg="resource does not contain the updated file new_" +
file_name_list[2])
self.assertNotIn(file_name_list[2],
updated_res_file_names,
msg='resource still contains the old file ' +
file_name_list[2] + ' after renaming')
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' moved to a folder')
self.assertNotIn(file_name_list[0],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[0] + 'after moving to a folder')
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
'moved to a new folder')
self.assertNotIn(file_name_list[1],
updated_res_file_names,
msg='resource still contains the old ' +
file_name_list[1] + ' after moving to a folder')
# zip the folder
output_zip_fname, size = \
zip_folder(user, res.short_id, 'data/contents/sub_test_dir',
'sub_test_dir.zip', True)
self.assertGreater(size, 0, msg='zipped file has a size of 0')
# Now resource should contain only two files: new_file3.txt and sub_test_dir.zip
# since the folder is zipped into sub_test_dir.zip with the folder deleted
self.assertEqual(res.files.all().count(),
2,
msg="resource file count didn't match-")
# test unzip does not allow override of existing files
# add an existing file in the zip to the resource
if res.resource_federation_path:
fed_test_file1_full_path = '/{zone}/home/{uname}/{fname}'.format(
zone=settings.HS_USER_IRODS_ZONE,
uname=user.username,
fname=file_name_list[0])
# TODO: why isn't this a method of resource?
# TODO: Why do we repeat the resource_federation_path?
add_resource_files(res.short_id,
source_names=[fed_test_file1_full_path],
move=False)
else:
# TODO: Why isn't this a method of resource?
add_resource_files(res.short_id, self.test_file_1)
# TODO: use ResourceFile.create_folder, which doesn't require data/contents prefix
create_folder(res.short_id, 'data/contents/sub_test_dir')
# TODO: use ResourceFile.rename, which doesn't require data/contents prefix
move_or_rename_file_or_folder(
user, res.short_id, 'data/contents/' + file_name_list[0],
'data/contents/sub_test_dir/' + file_name_list[0])
# Now resource should contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
with self.assertRaises(SessionException):
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip',
False)
# Resource should still contain three files: file3_new.txt, sub_test_dir.zip, and file1.txt
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
3,
msg="resource file count didn't match - " +
str(file_cnt) + " != 3")
# test unzipping the file succeeds now after deleting the existing folder
# TODO: this causes a multiple delete because the paths are valid now.
istorage = res.get_irods_storage()
remove_folder(user, res.short_id, 'data/contents/sub_test_dir')
# Now resource should contain two files: file3_new.txt and sub_test_dir.zip
file_cnt = res.files.all().count()
self.assertEqual(file_cnt,
2,
msg="resource file count didn't match - " +
str(file_cnt) + " != 2")
unzip_file(user, res.short_id, 'data/contents/sub_test_dir.zip', True)
# Now resource should contain three files: file1.txt, file2.txt, and file3_new.txt
self.assertEqual(res.files.all().count(),
3,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn(
'sub_test_dir.zip',
updated_res_file_names,
msg="resource still contains the zip file after unzipping")
self.assertIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[0])
self.assertIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain unzipped file ' +
file_name_list[1])
self.assertIn('new_' + file_name_list[2],
updated_res_file_names,
msg='resource does not contain unzipped file new_' +
file_name_list[2])
# rename a folder
move_or_rename_file_or_folder(user, res.short_id,
'data/contents/sub_test_dir',
'data/contents/sub_dir')
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertNotIn('sub_test_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource still contains ' + file_name_list[0] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[0],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[0] +
' in the new folder after renaming')
self.assertNotIn('sub_test_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource still contains ' + file_name_list[1] +
' in the old folder after renaming')
self.assertIn('sub_dir/' + file_name_list[1],
updated_res_file_names,
msg='resource does not contain ' + file_name_list[1] +
' in the new folder after renaming')
# remove a folder
# TODO: utilize ResourceFile.remove_folder instead. Takes a short path.
remove_folder(user, res.short_id, 'data/contents/sub_dir')
# Now resource only contains one file
self.assertEqual(res.files.all().count(),
1,
msg="resource file count didn't match")
updated_res_file_names = []
for rf in ResourceFile.objects.filter(object_id=res.id):
updated_res_file_names.append(rf.short_path)
self.assertEqual(len(updated_res_file_names), 1)
self.assertEqual(updated_res_file_names[0], 'new_' + file_name_list[2])
def test_metadata(self):
# add a type element
resource.create_metadata_element(self.resTimeSeries.short_id, 'type', url="http://hydroshare.org/netcdf")
# add another creator with all sub_elements
cr_name = 'Mike Sundar'
cr_des = 'http://hydroshare.org/user/001'
cr_org = "USU"
cr_email = '*****@*****.**'
cr_address = "11 River Drive, Logan UT-84321, USA"
cr_phone = '435-567-0989'
cr_homepage = 'http://usu.edu/homepage/001'
cr_res_id = 'http://research.org/001'
cr_res_gate_id = 'http://research-gate.org/001'
resource.create_metadata_element(self.resTimeSeries.short_id,'creator',
name=cr_name,
description=cr_des,
organization=cr_org,
email=cr_email,
address=cr_address,
phone=cr_phone,
homepage=cr_homepage,
researcherID=cr_res_id,
researchGateID=cr_res_gate_id)
# add another creator with only the name
resource.create_metadata_element(self.resTimeSeries.short_id,'creator', name='Lisa Holley')
#test adding a contributor with all sub_elements
con_name = 'Sujan Peterson'
con_des = 'http://hydroshare.org/user/002'
con_org = "USU"
con_email = '*****@*****.**'
con_address = "101 Center St, Logan UT-84321, USA"
con_phone = '435-567-3245'
con_homepage = 'http://usu.edu/homepage/009'
con_res_id = 'http://research.org/009'
con_res_gate_id = 'http://research-gate.org/009'
resource.create_metadata_element(self.resTimeSeries.short_id,'contributor',
name=con_name,
description=con_des,
organization=con_org,
email=con_email,
address=con_address,
phone=con_phone,
homepage=con_homepage,
researcherID=con_res_id,
researchGateID=con_res_gate_id)
# add another creator with only the name
resource.create_metadata_element(self.resTimeSeries.short_id,'contributor', name='Andrew Smith')
# add a period type coverage
# add a period type coverage
value_dict = {'name':'Name for period coverage' , 'start':'1/1/2000', 'end':'12/12/2012'}
resource.create_metadata_element(self.resTimeSeries.short_id,'coverage', type='period', value=value_dict)
# add a point type coverage
value_dict = {'name':'Name for point coverage', 'east':'56.45678', 'north':'12.6789'}
resource.create_metadata_element(self.resTimeSeries.short_id,'coverage', type='point', value=value_dict)
# add date of type 'valid'
resource.create_metadata_element(self.resTimeSeries.short_id,'date', type='valid', start_date='8/10/2011', end_date='8/11/2012')
# add a format element
format_nc = 'netcdf'
resource.create_metadata_element(self.resTimeSeries.short_id,'format', value=format_nc)
# add 'DOI' identifier
#resource.create_metadata_element(self.resTimeSeries.short_id,'identifier', name='DOI', url="http://dx.doi.org/001")
# add a language element
resource.create_metadata_element(self.resTimeSeries.short_id,'language', code='eng')
# add 'Publisher' element
original_file_name = 'original.txt'
original_file = open(original_file_name, 'w')
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, 'r')
# add the file to the resource
hydroshare.add_resource_files(self.resTimeSeries.short_id, original_file)
resource.create_metadata_element(self.resTimeSeries.short_id,'publisher', name="HydroShare", url="http://hydroshare.org")
# add a relation element of uri type
resource.create_metadata_element(self.resTimeSeries.short_id,'relation', type='isPartOf',
value='http://hydroshare.org/resource/001')
# add another relation element of non-uri type
resource.create_metadata_element(self.resTimeSeries.short_id,'relation', type='isDataFor',
value='This resource is for another resource')
# add a source element of uri type
resource.create_metadata_element(self.resTimeSeries.short_id,'source', derived_from='http://hydroshare.org/resource/0002')
# add a rights element
resource.create_metadata_element(self.resTimeSeries.short_id,'rights', statement='This is the rights statement for this resource',
url='http://rights.ord/001')
# add a subject element
resource.create_metadata_element(self.resTimeSeries.short_id,'subject', value='sub-1')
# add another subject element
resource.create_metadata_element(self.resTimeSeries.short_id,'subject', value='sub-2')
# add time series specific metadata elements
self.resTimeSeries.metadata.create_element('site', site_code='LR_WaterLab_AA',
site_name='Logan River at the Utah Water Research Laboratory '
'west bridge', elevation_m=1414, elevation_datum='EGM96',
site_type='Stream')
self.resTimeSeries.metadata.create_element('variable', variable_code='ODO', variable_name='Oxygen, dissolved',
variable_type='Concentration', no_data_value=-9999,
variable_definition='Concentration of oxygen gas dissolved in water.',
speciation='Not Applicable')
self.resTimeSeries.metadata.create_element('method', method_code=59, method_name='Optical DO',
method_type='Instrument deployment',
method_description='Dissolved oxygen concentration measured '
'optically using a YSI EXO multi-parameter water '
'quality sonde.', method_link='http://www.exowater.com')
exp_text = """Raw and unprocessed data and data products that have not undergone quality control.
Depending on the variable, data type, and data transmission system, raw data may be available within seconds
or minutes after the measurements have been made. Examples include real time precipitation, streamflow and
water quality measurements."""
self.resTimeSeries.metadata.create_element('processinglevel', processing_level_code=0, definition='Raw data',
explanation=exp_text)
self.resTimeSeries.metadata.create_element('timeseriesresult', units_type='Concentration',
units_name='milligrams per liter', units_abbreviation='mg/L',
status='Complete', sample_medium='Surface water', value_count=11283,
aggregation_statistics="Average")
print self.resTimeSeries.metadata.get_xml()
print(bad)
def generate_files(request, shortkey, *args, **kwargs):
res = hydroshare.get_resource_by_shortkey(shortkey)
ts, csv_link, csv_size, xml_link, xml_size = {}, '', '', '', ''
try:
if res.reference_type == 'rest':
ts = ts_utils.time_series_from_service(res.url, res.reference_type)
else:
ts = ts_utils.time_series_from_service(res.url,
res.reference_type,
site_name_or_code=res.data_site_code,
variable_code=res.variable_code)
vals = ts['values']
version = ts['wml_version']
d = datetime.date.today()
date = '{0}_{1}_{2}'.format(d.month, d.day, d.year)
file_base = '{0}-{1}'.format(res.title.replace(" ", ""), date)
csv_name = '{0}.{1}'.format(file_base, 'csv')
if version == '1':
xml_end = 'wml_1'
xml_name = '{0}-{1}.xml'.format(file_base, xml_end)
elif version == '2.0':
xml_end = 'wml_2_0'
xml_name = '{0}-{1}.xml'.format(file_base, xml_end)
for_csv = []
for k, v in vals.items():
t = (k, v)
for_csv.append(t)
ResourceFile.objects.filter(object_id=res.pk).delete()
with open(csv_name, 'wb') as csv_file:
w = csv.writer(csv_file)
w.writerow([res.title])
var = '{0}({1})'.format(ts['variable_name'], ts['units'])
w.writerow(['time', var])
for r in for_csv:
w.writerow(r)
with open(xml_name, 'wb') as xml_file:
xml_file.write(ts['time_series'])
csv_file = open(csv_name, 'r')
xml_file = open(xml_name, 'r')
files = [csv_file, xml_file]
hydroshare.add_resource_files(res.short_id, csv_file, xml_file)
create_bag(res)
os.remove(csv_name)
os.remove(xml_name)
files = ResourceFile.objects.filter(object_id=res.pk)
for f in files:
if str(f.resource_file).endswith('.csv'):
csv_link = f.resource_file.url
csv_size = f.resource_file.size
if xml_end in str(f.resource_file):
xml_link = f.resource_file.url
xml_size = f.resource_file.size
status_code = 200
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_name': csv_name,
'xml_name': xml_name,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # successfully generated new files
except Exception: # most likely because the server is unreachable
files = ResourceFile.objects.filter(object_id=res.pk)
xml_file = None
for f in files:
if str(f.resource_file).endswith('.csv'):
csv_link = f.resource_file.url
csv_size = f.resource_file.size
if str(f.resource_file).endswith('.xml'):
xml_link = f.resource_file.url
xml_size = f.resource_file.size
xml_file = f.resource_file
if xml_file is None:
status_code = 404
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # did not generate new files, did not find old ones
xml_doc = open(str(xml_file), 'r').read()
root = etree.XML(xml_doc)
os.remove(str(xml_file))
version = ts_utils.get_version(root)
if version == '1':
ts = ts_utils.parse_1_0_and_1_1(root)
status_code = 200
elif version =='2.0':
ts = ts_utils.parse_2_0(root)
status_code = 200
else:
status_code = 503
data = {'for_graph': ts.get('for_graph'),
'values': ts.get('values'),
'units': ts.get('units'),
'site_name': ts.get('site_name'),
'variable_name': ts.get('variable_name'),
'status_code': status_code,
'csv_link': csv_link,
'csv_size': csv_size,
'xml_link': xml_link,
'xml_size': xml_size}
return json_or_jsonp(request, data) # did not generate new files, return old ones
def test_unfederated_root_path_checks(self):
""" an unfederated file in the root folder has the proper state after state changes """
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
check_irods_files(self.res, stop_on_error=True)
# add one file to the resource
hydroshare.add_resource_files(self.res.short_id, self.test_file_1)
# should succeed without errors
check_irods_files(self.res, stop_on_error=True)
# cleaning should not change anything
check_irods_files(self.res, stop_on_error=True, log_errors=False, return_errors=True,
clean_irods=True, clean_django=True, sync_ispublic=True)
# resource should has only one file at this point
self.assertEqual(self.res.files.all().count(), 1,
msg="resource file count didn't match")
# get the handle of the file created above
resfile = self.res.files.all()[0]
# determine where that file should live
shortpath = os.path.join(self.res.short_id, "data", "contents", "file1.txt")
self.assertEqual(resfile.file_folder, None)
self.assertEqual(resfile.storage_path, shortpath)
self.assertTrue(resfile.path_is_acceptable(shortpath))
# now try to intentionally corrupt it
resfile.set_short_path("fuzz.txt")
# should raise exception
with self.assertRaises(ValidationError):
check_irods_files(self.res, stop_on_error=True)
# now don't raise exception and read error
errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# now try to clean it up
errors, ecount = check_irods_files(self.res, return_errors=True, log_errors=False,
clean_irods=True, clean_django=True)
self.assertTrue(errors[0].endswith(
'data/contents/fuzz.txt does not exist in iRODS (DELETED FROM DJANGO)'))
self.assertTrue(errors[1].endswith(
'data/contents/file1.txt in iRODs does not exist in Django (DELETED FROM IRODS)'))
self.assertTrue(errors[2].endswith(
"type is GenericResource, title is 'My Test Resource'"))
# resource should not have any files at this point
self.assertEqual(self.res.files.all().count(), 0,
msg="resource file count didn't match")
# now check should succeed
errors, ecount = check_irods_files(self.res, stop_on_error=True, log_errors=False)
self.assertEqual(ecount, 0)
# delete resources to clean up
hydroshare.delete_resource(self.res.short_id)
def test_netcdf_metadata(self):
# add a type element
resource.create_metadata_element(self.resNetCDF.short_id, 'type', url="http://hydroshare.org/netcdf")
# add another creator with all sub_elements
cr_name = 'Mike Sundar'
cr_des = 'http://hydroshare.org/user/001'
cr_org = "USU"
cr_email = '*****@*****.**'
cr_address = "11 River Drive, Logan UT-84321, USA"
cr_phone = '435-567-0989'
cr_homepage = 'http://usu.edu/homepage/001'
cr_res_id = 'http://research.org/001'
cr_res_gate_id = 'http://research-gate.org/001'
resource.create_metadata_element(self.resNetCDF.short_id,'creator',
name=cr_name,
description=cr_des,
organization=cr_org,
email=cr_email,
address=cr_address,
phone=cr_phone,
homepage=cr_homepage,
researcherID=cr_res_id,
researchGateID=cr_res_gate_id)
# add another creator with only the name
resource.create_metadata_element(self.resNetCDF.short_id,'creator', name='Lisa Holley')
#test adding a contributor with all sub_elements
con_name = 'Sujan Peterson'
con_des = 'http://hydroshare.org/user/002'
con_org = "USU"
con_email = '*****@*****.**'
con_address = "101 Center St, Logan UT-84321, USA"
con_phone = '435-567-3245'
con_homepage = 'http://usu.edu/homepage/009'
con_res_id = 'http://research.org/009'
con_res_gate_id = 'http://research-gate.org/009'
resource.create_metadata_element(self.resNetCDF.short_id,'contributor',
name=con_name,
description=con_des,
organization=con_org,
email=con_email,
address=con_address,
phone=con_phone,
homepage=con_homepage,
researcherID=con_res_id,
researchGateID=con_res_gate_id)
# add another creator with only the name
resource.create_metadata_element(self.resNetCDF.short_id,'contributor', name='Andrew Smith')
# add a period type coverage
# add a period type coverage
value_dict = {'name':'Name for period coverage' , 'start':'1/1/2000', 'end':'12/12/2012'}
resource.create_metadata_element(self.resNetCDF.short_id,'coverage', type='period', value=value_dict)
# add a point type coverage
value_dict = {'name':'Name for point coverage', 'east':'56.45678', 'north':'12.6789'}
resource.create_metadata_element(self.resNetCDF.short_id,'coverage', type='point', value=value_dict)
# add date of type 'valid'
resource.create_metadata_element(self.resNetCDF.short_id,'date', type='valid', start_date='8/10/2011', end_date='8/11/2012')
# add a format element
format_nc = 'netcdf'
resource.create_metadata_element(self.resNetCDF.short_id,'format', value=format_nc)
# add 'DOI' identifier
#resource.create_metadata_element(self.resNetCDF.short_id,'identifier', name='DOI', url="http://dx.doi.org/001")
# add a language element
resource.create_metadata_element(self.resNetCDF.short_id,'language', code='eng')
# add 'Publisher' element
original_file_name = 'original.txt'
original_file = open(original_file_name, 'w')
original_file.write("original text")
original_file.close()
original_file = open(original_file_name, 'r')
# add the file to the resource
hydroshare.add_resource_files(self.resNetCDF.short_id, original_file)
resource.create_metadata_element(self.resNetCDF.short_id,'publisher', name="HydroShare", url="http://hydroshare.org")
# add a relation element of uri type
resource.create_metadata_element(self.resNetCDF.short_id,'relation', type='isPartOf',
value='http://hydroshare.org/resource/001')
# add another relation element of non-uri type
resource.create_metadata_element(self.resNetCDF.short_id,'relation', type='isDataFor',
value='This resource is for another resource')
# add a source element of uri type
resource.create_metadata_element(self.resNetCDF.short_id,'source', derived_from='http://hydroshare.org/resource/0002')
# add a rights element
resource.create_metadata_element(self.resNetCDF.short_id,'rights', statement='This is the rights statement for this resource',
url='http://rights.ord/001')
# add a subject element
resource.create_metadata_element(self.resNetCDF.short_id,'subject', value='sub-1')
# add another subject element
resource.create_metadata_element(self.resNetCDF.short_id,'subject', value='sub-2')
# add a netcdf specific element (variable)
resource.create_metadata_element(self.resNetCDF.short_id,'variable', name='temp', unit='deg C', type='float', shape='shape_unknown')
print self.resNetCDF.metadata.get_xml()
print(bad)
