def test_update_scimeta(self):
hs = HydroShare(prompt_auth=False)
metadata_to_update = {'title': 'Updated resource title'}
scimeta = hs.updateScienceMetadata('511debf8858a4ea081f78d66870da76c',
metadata=metadata_to_update)
self.assertEqual(scimeta['title'], 'Updated resource title')
self.assertEqual(len(scimeta['creators']), 2)
self.assertEqual(len(scimeta['contributors']), 1)
self.assertEqual(len(scimeta['coverages']), 1)
self.assertEqual(len(scimeta['dates']), 2)
self.assertEqual(
scimeta['description'],
'Time series of level, area and volume in the Great Salt Lake. Volume and area of the Great Salt Lake are derived from recorded levels'
)
self.assertEqual(scimeta['formats'][0]['value'], 'image/tiff')
self.assertEqual(len(scimeta['funding_agencies']), 1)
self.assertEqual(len(scimeta['identifiers']), 1)
self.assertEqual(scimeta['language'], 'eng')
self.assertEqual(
scimeta['rights'],
'This resource is shared under the Creative Commons Attribution CC BY. http://creativecommons.org/licenses/by/4.0/'
)
self.assertEqual(scimeta['type'],
'http://www.hydroshare.org/terms/GenericResource')
self.assertEqual(scimeta['publisher'], None)
self.assertEqual(len(scimeta['sources']), 0)
self.assertEqual(len(scimeta['relations']), 0)
self.assertEqual(len(scimeta['subjects']), 2)
def get_epanet_model_metadata(request):
return_obj = {
'success': False,
'message': None,
'results': "",
}
if request.is_ajax() and request.method == 'GET':
if not request.GET.get('model_id'):
return_obj['message'] = message_template_param_unfilled.format(
param='model_id')
else:
model_id = request.GET['model_id']
try:
hs = get_oauth_hs(request)
except:
hs = HydroShare()
metadata_json = hs.getScienceMetadata(model_id)
return_obj['results'] = metadata_json
return_obj['success'] = True
else:
return_obj['message'] = message_template_wrong_req_method.format(
method="GET")
return JsonResponse(return_obj)
def download_epanet_model(request):
return_obj = {'success': False, 'message': None, 'results': "", 'name': ""}
if request.is_ajax() and request.method == 'GET':
if not request.GET.get('model_id'):
return_obj['message'] = message_template_param_unfilled.format(
param='model_id')
else:
model_id = request.GET['model_id']
try:
hs = get_oauth_hs(request)
except:
hs = HydroShare()
for model_file in hs.getResourceFileList(model_id):
model_url = model_file['url']
model_name = model_url[model_url.find('contents/') + 9:]
model = ""
for line in hs.getResourceFile(model_id, model_name):
model += line.decode("utf-8")
return_obj['name'] = model_name
return_obj['results'] = model
return_obj['success'] = True
else:
return_obj['message'] = message_template_wrong_req_method.format(
method="POST")
return JsonResponse(return_obj)
def test_get_resourcemap(self):
hs = HydroShare(prompt_auth=False)
resourcemap = hs.getResourceMap('6dbb0dfb8f3a498881e4de428cb1587c')
self.assertTrue(
resourcemap.find(
"""<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">"""
) != -1)
def test_get_resource_list_filter_date(self):
hs = HydroShare(hostname=self.url)
from_date = date(2015, 5, 20)
res_list = hs.getResourceList(from_date=from_date)
for (i, r) in enumerate(res_list):
self.assertTrue(
datetime.strptime(r['date_created'], '%m-%d-%Y').date() >=
from_date)
to_date = date(2015, 5, 21) # up to and including 5/21/2015
res_list = hs.getResourceList(to_date=to_date)
for (i, r) in enumerate(res_list):
self.assertTrue(
datetime.strptime(r['date_created'], '%m-%d-%Y').date() <
to_date)
from_date = date(2015, 5, 19)
to_date = date(2015, 5, 22) # up to and including 5/21/2015
res_list = hs.getResourceList(from_date=from_date, to_date=to_date)
for (i, r) in enumerate(res_list):
self.assertTrue(
datetime.strptime(r['date_created'], '%m-%d-%Y').date() >=
from_date)
self.assertTrue(
datetime.strptime(r['date_created'], '%m-%d-%Y').date() <
to_date)
def test_get_scimeta_xml(self):
hs = HydroShare(prompt_auth=False)
scimeta = hs.getScienceMetadataRDF('6dbb0dfb8f3a498881e4de428cb1587c')
self.assertTrue(
scimeta.find(
"""<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">"""
) != -1)
def authenticate(self, username, password, client_id=None, client_secret=None):
"""
Authenticates access to allow read/write access to privileged resource_cache
:param username: username for HydroShare.org
:param password: password associated with username
:param client_id: Client ID obtained from HydroShare
:param client_secret: Client Secret provided by HydroShare
:return: Returns true if authentication was successful, false otherwise
"""
if not all([username, password]):
self.auth = None
return False
if client_id is not None and client_secret is not None:
self.auth = HydroShareAuthOAuth2(client_id, client_secret, username=username, password=password)
else:
self.auth = HydroShareAuthBasic(username, password)
try:
self.client = HydroShare(auth=self.auth) # , verify=False)
self.user_info = self.client.getUserInfo()
return True
except HydroShareException as e: # for incorrect username/password combinations
print('Authentication failed: {}'.format(e))
except InvalidGrantError as e: # for failures when attempting to use OAuth2
print('Credentials could not be validated: {}'.format(e))
except InvalidClientError as e:
print('Invalid client ID and/or client secret: {}'.format(e))
except Exception as e:
print(e)
self.auth = None
return False
def __init__(self, username=None):
self.hs = None
self.content = {}
# load the HS environment variables
# self.load_environment()
uname = username
if uname is None:
uname = os.environ['HS_USR_NAME']
# get a secure connection to hydroshare
auth = self.getSecureConnection(uname)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
display(HTML('<b style="color:green;">Successfully established a connection with HydroShare</b>'))
except HydroShareHTTPException as e:
display(HTML(
'<p style="color:red;"><b>Failed to establish a connection with HydroShare. Please check that you provided the correct credentials</b><br>%s </p>' % e))
# remove the cached authentication
auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth')
os.remove(auth_path)
return None
def test_resource_scimeta_custom(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource('511debf8858a4ea081f78d66870da76c').scimeta.custom({
"foo": "bar",
"foo2": "bar2"
})
self.assertEqual(response.status_code, 200)
def home(request):
"""
Controller for the app home page.
"""
logger = logging.getLogger('django')
logger.error('Entering controller.py...')
res_output = ""
client_id = getattr(settings, "SOCIAL_AUTH_HYDROSHARE_KEY", "None")
client_secret = getattr(settings, "SOCIAL_AUTH_HYDROSHARE_SECRET", "None")
token = request.user.social_auth.get(provider='hydroshare').extra_data['token_dict']
token_dict_str = "Token: {0}".format(str(token))
logger.error(token_dict_str)
auth = HydroShareAuthOAuth2(client_id, client_secret, token=token)
try:
logger.error("Fetching resource list from playground.hydroshare.org")
hs = HydroShare(auth=auth, hostname='playground.hydroshare.org')
for resource in hs.getResourceList():
res_output += str(resource)
context = {"res_output": res_output, "token_dict_str": token_dict_str}
return render(request, 'oauth_user_demo/home.html', context)
except TokenExpiredError as e:
# TODO: redirect back to login view, giving this view as the view to return to
logger.error("TokenExpiredError: TODO: redirect to login view")
raise e
def test_referenced_update(self):
hs = HydroShare(prompt_auth=False)
response = hs.updateReferencedFile(pid='511debf8858a4ea081f78d66870da76c', path='data/contents', name='file.url',
ref_url='https://www.cuahsi.org')
self.assertEqual(response['status'], 'success')
def test_get_resource_list(self):
hs = HydroShare()
res_list = hs.getResourceList()
for (i, r) in enumerate(res_list):
self.assertEquals(r['resource_title'], self.resource_titles[i])
self.assertEquals(r['resource_id'], self.resource_ids[i])
def __init__(self, username=None, password=None, cache=True):
self.hs = None
self.content = {}
# load the HS environment variables
# todo: this should be set as a path variable somehow.
# possibly add JPY_TMP to Dockerfile
self.cache = cache
if cache:
utilities.load_environment()
self.auth_path = '/home/jovyan/.auth'
if password is None:
# try for secure connection
auth = self.getSecureConnection(username)
else:
print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO '
'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT')
auth = HydroShareAuthBasic(username=username, password=password)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
print('Successfully established a connection with HydroShare')
except HydroShareHTTPException as e:
print('Failed to establish a connection with HydroShare.\n '
'Please check that you provided the correct credentials.\n'
'%s' % e)
# remove the cached authentication
if os.path.exists(self.auth_path):
os.remove(self.auth_path)
def test_resource_move_or_rename(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource('511debf8858a4ea081f78d66870da76c').functions.move_or_rename({
"source_path": "/source/path",
"target_path": "/target/path"
})
self.assertEqual(response.status_code, 200)
def test_get_resource_list(self):
hs = HydroShare(prompt_auth=False)
res_list = hs.getResourceList()
for (i, r) in enumerate(res_list):
self.assertEqual(r['resource_title'], self.resource_titles[i])
self.assertEqual(r['resource_id'], self.resource_ids[i])
def test_get_user_info(self):
hs = HydroShare(prompt_auth=False)
user_info = hs.getUserInfo()
self.assertEqual(user_info['username'], 'username')
self.assertEqual(user_info['first_name'], 'First')
self.assertEqual(user_info['last_name'], 'Last')
self.assertEqual(user_info['email'], '*****@*****.**')
def test_resource_zip_folder(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource('511debf8858a4ea081f78d66870da76c').functions.zip({
"input_coll_path": "/source/path",
"output_zip_fname": "filename.zip",
"remove_original_after_zip": True
})
self.assertEqual(response.status_code, 200)
def get_resource_list():
hs = HydroShare()
resource_list = []
for resource in hs.getResourceList(types=['RasterResource']):
res_name = '{} [creator: {}]'.format(resource['resource_title'], resource['creator'])
resource_list.append((res_name, resource['resource_id']))
return resource_list
def test_get_user_info(self):
hs = HydroShare()
user_info = hs.getUserInfo()
self.assertEquals(user_info['username'], 'username')
self.assertEquals(user_info['first_name'], 'First')
self.assertEquals(user_info['last_name'], 'Last')
self.assertEquals(user_info['email'], '*****@*****.**')
def test_get_folder_contents(self):
hs = HydroShare(prompt_auth=False)
folder_contents = hs.getResourceFolderContents('511debf8858a4ea081f78d66870da76c', pathname='model/initial/')
self.assertEqual(folder_contents['resource_id'], '511debf8858a4ea081f78d66870da76c')
self.assertEqual(folder_contents['path'], 'model/initial')
self.assertEqual(folder_contents['files'], ["model.exe", "param.txt"])
self.assertEqual(folder_contents['folders'], ["run/1", "run/2"])
def test_resource_scimeta_custom(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource(
'511debf8858a4ea081f78d66870da76c').scimeta.custom({
"foo": "bar",
"foo2": "bar2"
})
self.assertEqual(response.status_code, 200)
def test_folder_delete(self):
hs = HydroShare(prompt_auth=False)
response = hs.deleteResourceFolder('511debf8858a4ea081f78d66870da76c',
pathname='model/initial/')
self.assertEqual(response['resource_id'],
'511debf8858a4ea081f78d66870da76c')
self.assertEqual(response['path'], 'model/initial')
def test_get_resource_file_list(self):
hs = HydroShare()
res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c')
for (i, r) in enumerate(res_list):
self.assertEquals(r['url'], self.urls[i])
self.assertEquals(r['size'], self.sizes[i])
self.assertEquals(r['content_type'], self.content_types[i])
def test_get_resource_file_list(self):
hs = HydroShare(prompt_auth=False)
res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c')
for (i, r) in enumerate(res_list):
self.assertEqual(r['url'], self.urls[i])
self.assertEqual(r['size'], self.sizes[i])
self.assertEqual(r['content_type'], self.content_types[i])
def test_resource_list_by_bounding_box(self):
hs = HydroShare(prompt_auth=False)
res_list = hs.resources(coverage_type="box",
north="50",
south="30",
east="40",
west="20")
for (i, r) in enumerate(res_list):
self.assertEquals(True, True)
def test_set_file_type(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource(
'511debf8858a4ea081f78d66870da76c').functions.set_file_type({
"file_path":
"file_path",
"hs_file_type":
"NetCDF"
})
self.assertEqual(response.status_code, 202)
def test_resource_move_or_rename(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource(
'511debf8858a4ea081f78d66870da76c').functions.move_or_rename({
"source_path":
"/source/path",
"target_path":
"/target/path"
})
self.assertEqual(response.status_code, 200)
def test_resource_unzip_file(self):
hs = HydroShare()
response = hs.resource(
'511debf8858a4ea081f78d66870da76c').functions.unzip({
"zip_with_rel_path":
"/path/to/zip",
"remove_original_zip":
True
})
self.assertEqual(response.status_code, 200)
def test_resource_list_by_bounding_box(self):
hs = HydroShare(prompt_auth=False)
res_list = hs.resources(coverage_type="box",
north="50",
south="30",
east="40",
west="20")
for (i, r) in enumerate(res_list):
self.assertEquals(True, True)
def test_referenced_update(self):
hs = HydroShare(prompt_auth=False)
response = hs.updateReferencedFile(
pid='511debf8858a4ea081f78d66870da76c',
path='data/contents',
name='file.url',
ref_url='https://www.cuahsi.org')
self.assertEqual(response['status'], 'success')
def test_resource_upload_file(self):
hs = HydroShare()
response = hs.resource('511debf8858a4ea081f78d66870da76c').files({
"file":
'mocks/data/another_resource_file.txt',
"folder":
"/target/folder"
})
self.assertEqual(response.status_code, 200)
def test_get_resource_types(self):
hs = HydroShare(prompt_auth=False)
res_type_proto = {
'GenericResource', 'ModelInstanceResource', 'ModelProgramResource',
'NetcdfResource', 'RasterResource', 'RefTimeSeries',
'SWATModelInstanceResource', 'TimeSeriesResource', 'ToolResource'
}
res_types = hs.getResourceTypes()
self.assertSetEqual(res_type_proto, res_types)
def download_executable_lubuntu_hs(save_filepath):
resource_id = 'a5dbd5b198c9468387f59f3fefc11e22'
hs = HydroShare()
hs.getResource(resource_id, destination=save_filepath, unzip=True)
resource_name = 'summa-master.zip'
executable_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name
shutil.unpack_archive(executable_filepath,
extract_dir=os.path.dirname(executable_filepath))
executable = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/summa-master/bin'
return os.path.dirname(executable)
def test_get_folder_contents(self):
hs = HydroShare(prompt_auth=False)
folder_contents = hs.getResourceFolderContents(
'511debf8858a4ea081f78d66870da76c', pathname='model/initial/')
self.assertEqual(folder_contents['resource_id'],
'511debf8858a4ea081f78d66870da76c')
self.assertEqual(folder_contents['path'], 'model/initial')
self.assertEqual(folder_contents['files'], ["model.exe", "param.txt"])
self.assertEqual(folder_contents['folders'], ["run/1", "run/2"])
def start_file_download(request):
global temp_dir, prediction_data, rp_data, total_prediction_comids, total_rp_comids, sorted_prediction_comids, \
sorted_rp_comids, time
if request.method == 'GET':
get_data = request.GET
this_script_path = inspect.getfile(inspect.currentframe())
try:
if get_data['res_id'] is not None:
temp_dir = tempfile.mkdtemp()
file_path = get_data['res_id']
if get_data['src'] == 'iRODS':
download = requests.get(file_path, stream=True)
filename = os.path.basename(file_path)
local_file_path = os.path.join(temp_dir, filename)
with open(local_file_path, 'wb') as fd:
for chunk in download.iter_content(1024):
fd.write(chunk)
prediction_data = nc.Dataset(local_file_path, mode="r")
elif get_data['src'] == 'hs':
auth = HydroShareAuthBasic(username='******', password='******')
hs = HydroShare(auth=auth, hostname="playground.hydroshare.org", use_https=False)
resource_data = hs.getSystemMetadata(file_path)
filename = resource_data['resource_title']
# this will only work if there is only one file in the resource and if
# the resource title is the same as filename
hs.getResourceFile(file_path, filename, destination=temp_dir)
local_file_path = temp_dir + "/" + filename
prediction_data = nc.Dataset(local_file_path, mode="r")
else:
testfile_path = this_script_path.replace('controllers.py', 'public/data/test.nc')
prediction_data = nc.Dataset(testfile_path, mode="r")
# Sort the RAPID netCDF file by COMID
qout_dimensions = prediction_data.variables['Qout'].dimensions
if qout_dimensions[0].lower() == 'comid' and qout_dimensions[1].lower() == 'time':
sorted_prediction_comids = sorted(enumerate(prediction_data.variables['COMID'][:]),
key=lambda comid: comid[1])
total_prediction_comids = len(sorted_prediction_comids)
else:
return JsonResponse({'error': "Invalid netCDF file"})
variables = prediction_data.variables.keys()
if 'time' in variables:
time = [t * 1000 for t in prediction_data.variables['time'][:]]
else:
return JsonResponse({'error': "Invalid netCDF file"})
rp_data_path = this_script_path.replace('controllers.py', 'public/data/return_period_data.nc')
rp_data = nc.Dataset(rp_data_path, mode="r")
sorted_rp_comids = sorted(enumerate(rp_data.variables['COMID'][:]), key=lambda comid: comid[1])
total_rp_comids = len(sorted_rp_comids)
return JsonResponse({'success': "The file is ready to go."})
except Exception, err:
return JsonResponse({'error': err})
def map(request):
"""
Controller for the app home page.
"""
hs = HydroShare()
res_info_list = []
i =0
for resource in hs.getResourceList(types=['MODFLOWModelInstanceResource']):
xml = hs.getScienceMetadata(resource['resource_id'])
print "iteration number"
print i
dict = parse(xml)
#takes xml turns into string then json dictionary object ^
try:
box = dict['rdf:RDF']['rdf:Description'][0]['dc:coverage'][1]['dcterms:box']['rdf:value']
#['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value']
# ^From the dictionary I pull out the values I need or the data from the "coverage" part of the model
list_points = box.split('; ')
# ^ splits up box to produce a list containing: northlimit, eastlimit, southlimit, westlimit, units
# and porjection.
temp_dict = {} #creates an empty dictionary
for l in list_points:
temp_list = l.split('=') # splits up the list even further into name and value ex: northlimit, 40.5033
# print temp_list
temp_dict[temp_list[0]] = temp_list[1]
res_info = {}
res_info["resource_id"]=resource['resource_id']
res_info["box"]=temp_dict
res_info_list.append(res_info)
print res_info
except Exception, e:
pass
try:
box = dict['rdf:RDF']['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value']
# ['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value']
# ^From the dictionary I pull out the values I need or the data from the "coverage" part of the model
list_points = box.split('; ')
# ^ splits up box to produce a list containing: northlimit, eastlimit, southlimit, westlimit, units
# and porjection.
temp_dict = {} # creates an empty dictionary
for l in list_points:
temp_list = l.split('=') # splits up the list even further into name and value ex: northlimit, 40.5033
# print temp_list
temp_dict[temp_list[0]] = temp_list[1]
res_info = {}
res_info["resource_id"] = resource['resource_id']
res_info["box"] = temp_dict
res_info_list.append(res_info)
print res_info
except Exception, e:
pass
def handle(self, *args, **options): # (f,fileid, databeginson,columnheaderson, cmd):
# cmdline = bool(options['cmdline'][0])
username = str(options['username'][0])
password = str(options['password'][0])
hs_client_id = settings.SOCIAL_AUTH_HYDROSHARE_UP_KEY
hs_client_secret = settings.SOCIAL_AUTH_HYDROSHARE_UP_SECRET
auth = HydroShareAuthOAuth2(hs_client_id, hs_client_secret,
username=username, password=password)
# print(username)
# print(password)
export_complete = True
resource_link = ''
user = str(options['django_user_name'][0])
datasettitle = str(options['datasettitle'][0])
startdate = str(options['startdate'][0])
enddate = str(options['enddate'][0])
datafile = str(options['datafile'][0])
dbfilename = str(options['dbfilename'][0])
# print(request.POST['hydroshareusername'])
# hs = get_oauth_hs(request)
# userInfo = hs.getUserInfo()
#
hs = HydroShare(auth=auth)
# username = hs.getUserInfo()
# print(username)
abstracttext = ''
title = ''
abstracttext += 'ODM2 Admin dataset: ' + str(datasettitle)
title += 'ODM2 Admin dataset ' + str(datasettitle)
abstract = abstracttext
keywords = ['ODM2']
rtype = 'GenericResource'
fpath = datafile # str(exportdb.DATABASES['default']['NAME'])
# # print(fpath)
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"'+entered_start_date +'", "end":"'+ entered_end_date +'"}}}, {"creator":{"name":"Miguel Leon"}}]'
metadata = '[{"coverage":{"type":"period", "value":{"start":"' + str(startdate) + '", "end":"' + str(
enddate) + '"}}}, ' \
'{"creator":{"name":"' + user + '"}}]'
extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# #abstract = 'My abstract'
# #title = 'My resource'
# #keywords = ('my keyword 1', 'my keyword 2')
# #rtype = 'GenericResource'
# #fpath = 'C:/Users/leonmi/Google Drive/ODM2AdminLT2/ODM2SQliteBlank.db'
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2010"}}}, {"creator":{"name":"John Smith"}}, {"creator":{"name":"Lisa Miller"}}]'
# #extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# messages.success(request, 'Profile details updated.')
resource_id = hs.createResource(rtype, title, resource_file=datafile,
resource_filename=dbfilename, keywords=keywords,
abstract=abstract, metadata=metadata, extra_metadata=extra_metadata)
print('resource created')
print(resource_id)
def check_auth(self, auth):
hs = HydroShare(auth=auth)
self.log.info('hs=%s' % str(hs))
try:
info = hs.getUserInfo()
self.settings['hydroshare'] = hs
self.log.info('info=%s' % info)
except:
hs = None
return hs
def makePublic(self, resource, public=True):
"""
Makes a resource public or private
:param resource: The resource you want to modify
:param public: boolean value, True makes the resource public, False makes it private (wowzer)
:return: None
"""
hs = HydroShare(auth=self.auth)
res = hs.resource(resource.id).public(public)
if res.status_code == 200 or res.status_code == 202:
resource.public = public
def install_test_cases_hs(save_filepath):
resource_id = 'a0105d479c334764ba84633c5b9c1c01'
hs = HydroShare()
hs.getResource(resource_id, destination=save_filepath, unzip=True)
resource_name = 'summatestcases-2.x.tar.gz'
testcase_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name
shutil.unpack_archive(testcase_filepath,
extract_dir=os.path.dirname(testcase_filepath))
cmd = "cd {}/summaTestCases_2.x/; ./installTestCases_local.sh".format(
os.path.dirname(testcase_filepath))
subprocess.run(cmd, shell=True)
return os.path.dirname(testcase_filepath)
def download_model_instance(resource_id):
path = (os.getcwd() + '/' + resource_id + '/' + resource_id +
'/data/contents/')
hs = HydroShare()
hs.getResource(resource_id, destination=os.getcwd(), unzip=True)
# unpack the simulation archive and remove unncessary files
Model_Instance_Name = os.listdir(path)[0]
shutil.unpack_archive(path + Model_Instance_Name, extract_dir=os.getcwd())
cmd = "rm -rf " + resource_id
subprocess.run(cmd, shell=True)
return Model_Instance_Name.split('.')[0]
def get_hs_resource(resource_id, file_path):
hs = HydroShare()
hs.getResource(resource_id, destination=file_path, unzip=True)
# unpack the simulation archive and remove unncessary files
hs_resource_dir = os.path.join(file_path, resource_id, resource_id,
'data/contents/')
hs_resource = os.listdir(hs_resource_dir)
shutil.unpack_archive(hs_resource_dir + hs_resource[0],
extract_dir=file_path)
shutil.rmtree(os.path.join(file_path, resource_id))
return hs_resource[0].split(".")[0]
def test_resource_zip_folder(self):
hs = HydroShare(prompt_auth=False)
response = hs.resource(
'511debf8858a4ea081f78d66870da76c').functions.zip({
"input_coll_path":
"/source/path",
"output_zip_fname":
"filename.zip",
"remove_original_after_zip":
True
})
self.assertEqual(response.status_code, 200)
def upload_to_hydroshare(request):
try:
if request.method == 'GET':
get_data = request.GET
download_path = str(get_data['download_path'])
hs_username = str(get_data['hs_username'])
hs_password = str(get_data['hs_password'])
r_title = str(get_data['r_title'])
r_type = str(get_data['r_type'])
r_abstract = str(get_data['r_abstract'])
r_keywords_raw = str(get_data['r_keywords'])
r_keywords = r_keywords_raw.split(',')
#startup a Hydroshare instance with user's credentials
auth = HydroShareAuthBasic(username=hs_username, password=hs_password)
hs = HydroShare(auth=auth, hostname="dev.hydroshare.org", use_https=True)
# try to download a tiny file simply to test the user's credentials
# test_id = '49d01b5b0d0a41b6a5a31d8aace0a36e'
# hs.getResource(test_id, destination=None, unzip=False)
#download the iRODS file to a temp directory
temp_dir = tempfile.mkdtemp()
filename = os.path.basename(download_path)
download_file_path = os.path.join(temp_dir, filename)
download = requests.get(download_path, stream=True)
with open(download_file_path, 'wb') as fd:
for chunk in download.iter_content(1024):
fd.write(chunk)
#upload the temp file to HydroShare
if os.path.exists(download_file_path):
resource_id = hs.createResource(r_type, r_title, resource_file=download_file_path, keywords=r_keywords, abstract=r_abstract)
else:
if temp_dir:
# remove the temp directory/file
shutil.rmtree(temp_dir)
return JsonResponse({'error': 'An error occurred with the file upload.'})
except Exception, err:
if temp_dir:
# remove the temp directory/file
shutil.rmtree(temp_dir)
if "401 Unauthorized" in str(err):
return JsonResponse({'error': 'Username or password invalid.'})
elif "400 Bad Request" in str(err):
return JsonResponse({'error': 'File uploaded successfully despite 400 Bad Request Error.'})
else:
traceback.print_exc()
return JsonResponse({'error': 'HydroShare rejected the upload for some reason.'})
def test_get_resource_types(self):
hs = HydroShare()
res_type_proto = {'GenericResource',
'ModelInstanceResource',
'ModelProgramResource',
'NetcdfResource',
'RasterResource',
'RefTimeSeries',
'SWATModelInstanceResource',
'TimeSeriesResource',
'ToolResource'}
res_types = hs.getResourceTypes()
self.assertSetEqual(res_type_proto, res_types)
def test_create_get_delete_resource_file(self):
hs = HydroShare()
# Add
res_id = '511debf8858a4ea081f78d66870da76c'
fpath = 'mocks/data/another_resource_file.txt'
fname = os.path.basename(fpath)
resp = hs.addResourceFile(res_id, fpath)
self.assertEqual(resp, res_id)
# Get
tmpdir = tempfile.mkdtemp()
res_file = hs.getResourceFile(res_id, fname, destination=tmpdir)
self.assertTrue(filecmp.cmp(res_file, fpath, shallow=False))
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResourceFile(res_id, fname)
self.assertEqual(delres, res_id)
def test_get_resource_list_filter_date(self):
hs = HydroShare()
from_date = date(2015, 5, 20)
res_list = hs.getResourceList(from_date=from_date)
for (i, r) in enumerate(res_list):
self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date)
to_date = date(2015, 5, 21) # up to and including 5/21/2015
res_list = hs.getResourceList(to_date=to_date)
for (i, r) in enumerate(res_list):
self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date)
from_date = date(2015, 5, 19)
to_date = date(2015, 5, 22) # up to and including 5/21/2015
res_list = hs.getResourceList(from_date=from_date, to_date=to_date)
for (i, r) in enumerate(res_list):
self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date)
self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date)
def test_create_get_delete_resource(self):
hs = HydroShare(prompt_auth=False)
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
metadata = json.dumps([{'coverage': {'type': 'period', 'value': {'start': '01/01/2000',
'end': '12/12/2010'}}},
{'creator': {'name': 'John Smith'}},
{'contributor': {'name': 'Lisa Miller'}}])
extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'})
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords,
abstract=abstract, metadata=metadata, extra_metadata=extra_metadata)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile:
self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist())
downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r')
original = open('mocks/data/minimal_resource_file.txt', 'rb')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResource(newres)
self.assertEqual(delres, newres)
def test_get_scimeta_json(self):
hs = HydroShare(prompt_auth=False)
scimeta = hs.getScienceMetadata('511debf8858a4ea081f78d66870da76c')
self.assertEqual(scimeta['title'], 'Great Salt Lake Level and Volume')
self.assertEqual(len(scimeta['creators']), 2)
self.assertEqual(len(scimeta['contributors']), 1)
self.assertEqual(len(scimeta['coverages']), 1)
self.assertEqual(len(scimeta['dates']), 2)
self.assertEqual(scimeta['description'], 'Time series of level, area and volume in the Great Salt Lake. Volume and area of the Great Salt Lake are derived from recorded levels')
self.assertEqual(scimeta['formats'][0]['value'], 'image/tiff')
self.assertEqual(len(scimeta['funding_agencies']), 1)
self.assertEqual(len(scimeta['identifiers']), 1)
self.assertEqual(scimeta['language'], 'eng')
self.assertEqual(scimeta['rights'], 'This resource is shared under the Creative Commons Attribution CC BY. http://creativecommons.org/licenses/by/4.0/')
self.assertEqual(scimeta['type'], 'http://www.hydroshare.org/terms/GenericResource')
self.assertEqual(scimeta['publisher'], None)
self.assertEqual(len(scimeta['sources']), 0)
self.assertEqual(len(scimeta['relations']), 0)
self.assertEqual(len(scimeta['subjects']), 2)
def __init__(self, username=None, password=None, cache=True):
self.hs = None
self.content = {}
# connect to hydroshare using OAUTH2
authfile = os.path.expanduser("~/.hs_auth")
if os.path.exists(authfile):
with open(authfile, 'rb') as f:
token, cid = pickle.load(f)
auth = HydroShareAuthOAuth2(cid, '', token=token)
else:
# connect to hydroshare using Basic Authentication
self.cache = cache
if cache:
utilities.load_environment(os.path.join(
os.environ['NOTEBOOK_HOME'],
'.env'))
self.auth_path = os.environ.get('NOTEBOOK_HOME',
'/home/jovyan/.auth')
uname = username
if uname is None:
if 'HS_USR_NAME' in os.environ.keys():
uname = os.environ['HS_USR_NAME']
if password is None:
# get a secure connection to hydroshare
auth = self.getSecureConnection(uname)
else:
print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO '
'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT')
auth = HydroShareAuthBasic(username=uname, password=password)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
print('Successfully established a connection with HydroShare')
except HydroShareHTTPException as e:
print('Failed to establish a connection with HydroShare.\n '
'Please check that you provided the correct credentials.\n'
'%s' % e)
# remove the cached authentication
if os.path.exists(self.auth_path):
os.remove(self.auth_path)
return None
# set the HS resource download directory
download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'Downloads')
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
self.download_dir = download_dir
def run_service(user_input):
auth = HydroShareAuthBasic(hs_username, hs_password)
hs = HydroShare(auth=auth)
client = HydroDS(username=ci_username, password=ci_password)
# step1: get raster resource files from HydroShare # TODO remember to check if the resource is downloadable
resource_id = user_input['input_raster_url_path']
destination = tempfile.mkdtemp()
hs.getResource(resource_id, destination, unzip=True)
raster_file_folder = '{0}/{1}/{2}/data/contents'.format(destination, resource_id, resource_id)
raster_file_path_list = [os.path.join(raster_file_folder, f) for f in os.listdir(raster_file_folder) if
'.vrt' not in f]
upload_file_url = client.upload_file(file_to_upload=raster_file_path_list[0])
# step2: processing files in HydroDS
output_raster_name = 'process_{}'.format(upload_file_url.split('/')[-1])
output_file = client.project_resample_raster(input_raster_url_path=upload_file_url,
cell_size_dx=int(user_input['cell_size_dx']),
cell_size_dy=int(user_input['cell_size_dy']),
output_raster=output_raster_name,
resample=user_input['resample'],
epsg_code=int(user_input['proj_code']) if user_input['proj_code_type']=='epsg_code' else None,
utm_zone=int(user_input['proj_code']) if user_input['proj_code_type']=='utm_zone' else None,
)
# step3: download file and create HydroShare resource
save_as = '{}/{}'.format(destination, output_file['output_raster'].split('/')[-1])
client.download_file(file_url_path=output_file['output_raster'], save_as=save_as)
title = user_input['output_raster']
rtype = 'RasterResource'
resource_id = hs.createResource(rtype, title, resource_file=save_as)
# step4: clean up the temp in Tethys and HydroDS
shutil.rmtree(destination)
client.delete_my_file(output_file['output_raster'].split('/')[-1])
client.delete_my_file(upload_file_url.split('/')[-1])
return {'is_success': True, 'message': 'http://www.hydroshare.org/resource/{}/'.format(resource_id)}
def test_create_get_delete_resource(self):
hs = HydroShare()
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile:
self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist())
downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r')
original = open('mocks/data/minimal_resource_file.txt', 'r')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResource(newres)
self.assertEqual(delres, newres)
def __init__(self, username=None, password=None, cache=True):
self.hs = None
self.content = {}
# load the HS environment variables
# todo: this should be set as a path variable somehow.
# possibly add JPY_TMP to Dockerfile
self.cache = cache
if cache:
utilities.load_environment(os.path.join(
os.environ['NOTEBOOK_HOME'], '.env'))
self.auth_path = '/home/jovyan/.auth'
# todo: either use JPY_USR or ask them to
# enter their hydroshare username
uname = username
if uname is None:
if 'HS_USR_NAME' in os.environ.keys():
uname = os.environ['HS_USR_NAME']
if password is None:
# get a secure connection to hydroshare
auth = self.getSecureConnection(uname)
else:
print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO '
'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT')
auth = HydroShareAuthBasic(username=uname, password=password)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
print('Successfully established a connection with HydroShare')
except HydroShareHTTPException as e:
print('Failed to establish a connection with HydroShare.\n '
'Please check that you provided the correct credentials.\n'
'%s' % e)
# remove the cached authentication
if os.path.exists(self.auth_path):
os.remove(self.auth_path)
return None
def test_get_resource_list_filter_type(self):
hs = HydroShare()
res_list = hs.getResourceList(types=('RasterResource',))
for (i, r) in enumerate(res_list):
self.assertEquals(r['resource_type'], 'RasterResource')
def Test_All_Resources(request):
try:
start = time()
res_count = 0
num_success = 0
num_errors = 0
error_resource_list = []
ACCEPTABLE_ERRORS_LIST = ['This resource is too large to open in HydroShare GIS',
'There is no projection information associated with this resource',
'Resource contains insufficient geospatial information']
set_currently_testing(True)
auth = HydroShareAuthBasic(username='******', password='******')
hs = HydroShare(auth=auth)
valid_res_types = ['GeographicFeatureResource', 'RasterResource', 'RefTimeSeriesResource',
'GenericResource', 'ScriptResource']
resource_list = hs.getResourceList(types=valid_res_types)
num_resources = 0
for res in hs.getResourceList(types=valid_res_types):
if res['public']:
num_resources += 1
for res in resource_list:
res_id = None
try:
if res['public']:
res_count += 1
res_id = res['resource_id']
print "Currently testing resource %s of %s: %s" % (res_count, num_resources, res_id)
if res['resource_type'] == 'GenericResource':
response = get_res_files_list(hs, res_id)
if response['success']:
res_files_list = response['results']['generic_res_files_list']
num_files_failed = 0
for i, res_file in enumerate(res_files_list):
response = process_generic_res_file(hs, res_id, res_file,
request.user.username, i)
if response['success']:
pass
else:
error_acceptable = False
for error in ACCEPTABLE_ERRORS_LIST:
if error in response['message']:
error_acceptable = True
break
if error_acceptable:
pass
else:
num_files_failed += 1
error_resource_list.append('https://www.hydroshare.org/resource/%s on file %s'
% (res_id, res_file))
print 'ERROR ENCOUNTERED:'
print 'RES_ID: %s' % res_id
print 'MESSAGE: %s' % response['message']
if num_files_failed == 0:
num_success += 1
else:
num_errors += 1
else:
response = process_nongeneric_res(hs, res_id)
if response['success']:
num_success += 1
else:
error_acceptable = False
for error in ACCEPTABLE_ERRORS_LIST:
if error in response['message']:
error_acceptable = True
break
if error_acceptable:
num_success += 1
else:
num_errors += 1
error_resource_list.append('https://www.hydroshare.org/resource/%s' % res_id)
print 'ERROR ENCOUNTERED:'
print 'RES_ID: %s' % res_id
print 'MESSAGE: %s' % response['message']
except Exception as e:
num_errors += 1
error_resource_list.append('https://www.hydroshare.org/resource/%s' % res_id)
print 'ERROR ENCOUNTERED:'
print 'RES_ID: %s' % res_id
print 'MESSAGE: %s' % str(e)
print "%d%% complete" % (res_count * 100 / num_resources)
elapsed = str(timedelta(seconds=time()-start))
results = '''
ALL TESTS COMPLETE!
SUMMARY
TIME ELAPSED: {0}
TOTAL RESOURCES TESTED: {1}
TOTAL FAILED: {2}
PERCENT FAILED: {3}
TOTAL SUCCESSFUL: {4}
PERCENT SUCCESSFUL: {5}
{6}
{7}
'''.format(
elapsed,
res_count,
num_errors,
'%d%%' % (num_errors * 100 / res_count),
num_success,
'%d%%' % (num_success * 100 / res_count),
'LIST OF FAILED RESOURCES:' if len(error_resource_list) != 0 else 'ALL TESTS PASSED!',
'\n'.join(error_resource_list)
)
print results
email_admin('Test Results', custom_msg=results)
context = {'results': '<br>'.join(results.split('\n'))}
set_currently_testing(False)
return render(request, 'hydroshare_gis/test-results.html', context)
finally:
set_currently_testing(False)
def test_get_resource_list_filter_creator(self):
hs = HydroShare()
res_list = hs.getResourceList(creator='bmiles')
for (i, r) in enumerate(res_list):
self.assertEquals(r['creator'], 'bmiles')
def upload_to_hs(id,file):
auth = HydroShareAuthBasic(username='******', password='******')
hs = HydroShare(auth=auth)
fpath = '/path/to/somefile.txt'
resource_id = hs.addResourceFile('id', file)
def test_get_scimeta(self):
hs = HydroShare()
scimeta = hs.getScienceMetadata('6dbb0dfb8f3a498881e4de428cb1587c')
self.assertTrue(scimeta.find("""<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">""") != -1)
class hydroshare():
def __init__(self, username=None, password=None, cache=True):
self.hs = None
self.content = {}
# connect to hydroshare using OAUTH2
authfile = os.path.expanduser("~/.hs_auth")
if os.path.exists(authfile):
with open(authfile, 'rb') as f:
token, cid = pickle.load(f)
auth = HydroShareAuthOAuth2(cid, '', token=token)
else:
# connect to hydroshare using Basic Authentication
self.cache = cache
if cache:
utilities.load_environment(os.path.join(
os.environ['NOTEBOOK_HOME'],
'.env'))
self.auth_path = os.environ.get('NOTEBOOK_HOME',
'/home/jovyan/.auth')
uname = username
if uname is None:
if 'HS_USR_NAME' in os.environ.keys():
uname = os.environ['HS_USR_NAME']
if password is None:
# get a secure connection to hydroshare
auth = self.getSecureConnection(uname)
else:
print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO '
'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT')
auth = HydroShareAuthBasic(username=uname, password=password)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
print('Successfully established a connection with HydroShare')
except HydroShareHTTPException as e:
print('Failed to establish a connection with HydroShare.\n '
'Please check that you provided the correct credentials.\n'
'%s' % e)
# remove the cached authentication
if os.path.exists(self.auth_path):
os.remove(self.auth_path)
return None
# set the HS resource download directory
download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'Downloads')
if not os.path.isdir(download_dir):
os.makedirs(download_dir)
self.download_dir = download_dir
def _addContentToExistingResource(self, resid, content_files):
for f in content_files:
self.hs.addResourceFile(resid, f)
def getSecureConnection(self, username=None):
"""Establishes a secure connection with hydroshare.
args:
-- email: email address associated with hydroshare
returns:
-- hydroshare api connection
"""
if not os.path.exists(self.auth_path):
print('\nThe hs_utils library requires a secure connection to '
'your HydroShare account.')
if username is None:
username = input('Please enter your HydroShare username: '******'Enter the HydroShare password for user '
'\'%s\': ' % username)
auth = HydroShareAuthBasic(username=username, password=p)
if self.cache:
with open(self.auth_path, 'wb') as f:
pickle.dump(auth, f, protocol=2)
else:
with open(self.auth_path, 'rb') as f:
auth = pickle.load(f)
return auth
def getResourceMetadata(self, resid):
"""Gets metadata for a specified resource.
args:
-- resid: hydroshare resource id
returns:
-- resource metadata object
"""
science_meta = self.hs.getScienceMetadata(resid)
system_meta = self.hs.getSystemMetadata(resid)
return resource.ResourceMetadata(system_meta, science_meta)
def createHydroShareResource(self, abstract, title, derivedFromId=None,
keywords=[], resource_type='GenericResource',
content_files=[], public=False):
"""Creates a hydroshare resource.
args:
-- abstract: abstract for resource (str, required)
-- title: title of resource (str, required)
-- derivedFromId: id of parent hydroshare resource (str, default=>None)
-- keywords: list of subject keywords (list, default=>[])
-- resource_type: type of resource to create (str, default=>
'GenericResource')
-- content_files: data to save as resource content (list, default=>[])
-- public: resource sharing status (bool, default=>False)
returns:
-- None
"""
# query the hydroshare resource types and make sure that
# resource_type is valid
restypes = {r.lower(): r for r in self.hs.getResourceTypes()}
try:
res_type = restypes[resource_type]
except KeyError:
display(HTML('<b style="color:red;">[%s] is not a valid '
'HydroShare resource type.</p>' % resource_type))
return None
# get the 'derived resource' metadata
if derivedFromId is not None:
try:
# update the abstract and keyword metadata
meta = self.getResourceMetadata(derivedFromId)
abstract = meta.abstract \
+ '\n\n[Modified in JupyterHub on %s]\n%s' \
% (dt.now(), abstract)
keywords = set(keywords + meta.keywords)
except:
display(HTML('<b style="color:red;">Encountered an error '
' while setting the derivedFrom relationship '
' using id=%s. Make sure this resource is '
' is accessible to your account. '
% derivedFromId))
display(HTML('<a href=%s target="_blank">%s<a>' %
('https://www.hydroshare.org/resource/%s'
% derivedFromId, 'View the "DerivedFrom" '
'Resource')))
return None
f = None if len(content_files) == 0 else content_files[0]
# create the hs resource (1 content file allowed)
resid = threads.runThreadedFunction('Creating HydroShare Resource',
'Resource Created Successfully',
self.hs.createResource,
resource_type=res_type,
title=title,
abstract=abstract,
resource_file=f,
keywords=keywords)
# add the remaining content files to the hs resource
try:
if len(content_files) > 1:
self.addContentToExistingResource(resid, content_files[1:])
except Exception as e:
print(e)
display(HTML('Resource id: %s' % resid))
display(HTML('<a href=%s target="_blank">%s<a>' %
('https://www.hydroshare.org/resource/%s'
% resid, 'Open Resource in HydroShare')))
def getResourceFromHydroShare(self, resourceid, destination='.'):
"""Downloads content of a hydroshare resource.
args:
-- resourceid: id of the hydroshare resource (str)
-- destination: path to save resource, default
/user/[username]/notebooks/data (str)
returns:
-- None
"""
default_dl_path = self.download_dir
dst = os.path.abspath(os.path.join(default_dl_path, destination))
download = True
# check if the data should be overwritten
dst_res_folder = os.path.join(dst, resourceid)
if os.path.exists(dst_res_folder):
print('This resource already exists in your userspace.')
utils.tree(dst_res_folder)
res = input('\nDo you want to overwrite these data [Y/n]? ')
if res != 'n':
shutil.rmtree(dst_res_folder)
else:
download = False
# re-download the content if desired
if download:
try:
# download the resource (threaded)
threads.runThreadedFunction('Downloading Resource',
'Download Finished',
self.hs.getResource,
resourceid,
destination=dst,
unzip=True)
print('Successfully downloaded resource %s' % resourceid)
except Exception as e:
display(HTML('<b style="color:red">Failed to retrieve '
'resource content from HydroShare: %s</b>' % e))
return None
# load the resource content
outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid))
content_files = glob.glob(os.path.join(outdir, 'data/contents/*'))
content = {}
for f in content_files:
fname = os.path.basename(f)
# trim the base name relative to the data directory
dest_folder_name = os.path.dirname(destination).split('/')[-1]
f = os.path.join(dest_folder_name,
os.path.relpath(f, dest_folder_name))
content[fname] = f
# show the resource content files
utilities.display_resource_content_files(content)
# update the content dictionary
self.content.update(content)
def addContentToExistingResource(self, resid, content):
"""Adds content files to an existing hydroshare resource.
args:
-- resid: id of an existing hydroshare resource (str)
-- content: files paths to be added to resource (list)
returns:
-- None
"""
threads.runThreadedFunction('Adding Content to Resource',
'Successfully Added Content Files',
self._addContentToExistingResource,
resid, content)
def loadResource(self, resourceid):
"""Loads the contents of a previously downloaded resource.
args:
-- resourceid: the id of the resource that has been downloaded (str)
returns:
-- {content file name: path}
"""
resdir = utilities.find_resource_directory(resourceid)
if resdir is None:
display(HTML('<b style="color:red">Could not find any resource '
'matching the id [%s].</b> <br> It is likely that '
'this resource has not yet been downloaded from '
'HydroShare.org, or it was removed from the '
'JupyterHub server. Please use the following '
'command to aquire the resource content: '
'<br><br><code>hs.getResourceFromHydroShare(%s)'
'</code>.' % (resourceid, resourceid)))
return
# create search paths. Need to check 2 paths due to hs_restclient bug #63.
search_paths = [os.path.join(resdir, '%s/data/contents/*' % resourceid),
os.path.join(resdir, 'data/contents/*')]
content = {}
found_content = False
for p in search_paths:
content_files = glob.glob(p)
if len(content_files) > 0:
found_content = True
display(HTML('<p>Downloaded content is located at: %s</p>' % resdir))
display(HTML('<p>Found %d content file(s). \n</p>'
% len(content_files)))
for f in content_files:
fname = os.path.basename(f)
content[fname] = f
if len(content.keys()) == 0:
display(HTML('<p>Did not find any content files for resource id: %s</p>' % resourceid))
utilities.display_resource_content_files(content)
self.content = content
def getContentFiles(self, resourceid):
"""Gets the content files for a resource that exists on the
Jupyter Server
args:
-- resourceid: the id of the hydroshare resource
returns:
-- {content file name: path}
"""
content = utilities.get_hs_content(resourceid)
return content
def getContentPath(self, resourceid):
"""Gets the server path of a resources content files.
args:
-- resourceid: the id of the hydroshare resource
returns:
-- server path the the resource content files
"""
path = utilities.find_resource_directory(resourceid)
if path is not None:
return os.path.join(path, resourceid, 'data/contents')
def create_hydroshare_resource(context,
auth,
title,
hydroshare_host='www.hydroshare.org',
hydroshare_port=None, use_https=False,
resource_type='GenericResource',
abstract=None,
keywords=None,
create_callback=None,
verbose=False,
outfp=sys.stderr):
"""
Create HydroShare resource of an EcohydroLib project
@param context ecohydrolib.Context object
@param auth hs_restclient.HydroShareAuth object
@param title string representing the title of the resource
@param hydroshare_host string representing DNS name of the HydroShare
server in which to create the resource
@param hydroshare_port int representing the TCP port of the HydroShare
server
@param use_https True if HTTPS should be used. Default: False
@param resource_type string representing the HydroShare resource type
that should be used to create the resource
@param abstract string representing the abstract of the resource
@param keywords list of strings representing the keywords to assign
to the resource
@param create_callback user-defined callable that takes as input a
file size in bytes, and generates a callable to provide feedback
to the user about the progress of the upload of resource_file.
For more information, see:
http://toolbelt.readthedocs.org/en/latest/uploading-data.html#monitoring-your-streaming-multipart-upload
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@return string representing the ID of the newly created resource
"""
temp_dir = tempfile.mkdtemp()
zip_filename = "{0}.zip".format(os.path.basename(context.projectDir))
zip_filepath = os.path.join(temp_dir, zip_filename)
# Zip up the project for upload ...
if verbose:
outfp.write("Creating archive...")
outfp.flush()
zfile = zipfile.ZipFile(zip_filepath, 'w',
zipfile.ZIP_DEFLATED,
True)
os.path.walk(context.projectDir, _addToZip, (context.projectDir, zfile))
zfile.close()
if verbose:
outfp.write("done\n")
outfp.flush()
# Make upload progress callback
progress_callback = None
if create_callback:
s = os.stat(zip_filepath)
progress_callback = create_callback(s.st_size)
if verbose:
outfp.write("Uploading to HydroShare...")
outfp.flush()
# This next line is needed for some text-based progress indicators
# to properly render.
outfp.write(" \nThis is a hack\r")
outfp.flush()
if hydroshare_host:
hs = HydroShare(hostname=hydroshare_host, port=hydroshare_port,
use_https=use_https, auth=auth)
else:
hs = HydroShare(port=hydroshare_port,
use_https=use_https, auth=auth)
resource_id = None
try:
resource_id = hs.createResource(resource_type, title,
resource_file=zip_filepath, resource_filename=zip_filename,
abstract=abstract, keywords=keywords,
progress_callback=progress_callback)
if verbose:
url = _getResourceURL(resource_id,
hydroshare_host, hs_port=hydroshare_port,
use_https=use_https)
outfp.write("\nNew resource created at:\n{0}\n".format(url))
outfp.flush()
except Exception as e:
raise e
finally:
shutil.rmtree(temp_dir)
return resource_id
