def runSCI(id):
if request.method == 'POST':
auth = HydroShareAuthBasic(username=request.form['username'],
password=request.form['password'])
hs = HydroShare(auth=auth)
hs.getResource('e987ddcf73a94376a4a70e5db0fb7646',
destination='/home/ubuntu/hydroshareLink/',
unzip=True)
subprocess.Popen(
'sciunit open /home/ubuntu/hydroshareLink/e987ddcf73a94376a4a70e5db0fb7646/e987ddcf73a94376a4a70e5db0fb7646/data/contents/modflow.zip',
stdout=subprocess.PIPE,
shell=True)
os.chdir("/home/ubuntu/test/")
hs.getResource(id, destination='/home/ubuntu/test/', unzip=True)
proc = subprocess.Popen('sciunit repeat e2 ' + str(id),
stdout=subprocess.PIPE,
shell=True)
output = proc.stdout.read()
abstract = ''
title = 'MODFLOW_NWT_SCIUNIT_OUTPUT'
keywords = ('my keyword 1', 'my keyword 2')
rtype = 'MODFLOWModelInstanceResource'
output_id = hs.createResource(rtype,
title,
abstract=abstract,
keywords=keywords)
for file in os.listdir("/home/ubuntu/test/MODFLOW/"):
if file != "mfnwt":
hs.addResourceFile(output_id,
"/home/ubuntu/test/MODFLOW/" + file)
title = 'ModflowNwtCollection'
keywords = ('MODFLOW-NWT Input data', 'MODFLOW-NWT Output data',
'MODFLOW-NWT')
rtype = 'CollectionResource'
resource_id = hs.createResource(rtype,
title,
abstract=abstract,
keywords=keywords)
metaData = {'relations': []}
newObject = {}
newObject['type'] = 'hasPart'
newObject['value'] = 'http://www.hydroshare.org/resource/' + str(
id) + '/'
metaData['relations'].append(newObject)
newObject = {}
newObject['type'] = 'hasPart'
newObject['value'] = 'http://www.hydroshare.org/resource/' + str(
output_id) + '/'
metaData['relations'].append(newObject)
hs.updateScienceMetadata(resource_id, metadata=metaData)
return output
return render_template('login.html', id=str(id))
def handle(self, *args, **options): # (f,fileid, databeginson,columnheaderson, cmd):
# cmdline = bool(options['cmdline'][0])
username = str(options['username'][0])
password = str(options['password'][0])
hs_client_id = settings.SOCIAL_AUTH_HYDROSHARE_UP_KEY
hs_client_secret = settings.SOCIAL_AUTH_HYDROSHARE_UP_SECRET
auth = HydroShareAuthOAuth2(hs_client_id, hs_client_secret,
username=username, password=password)
# print(username)
# print(password)
export_complete = True
resource_link = ''
user = str(options['django_user_name'][0])
datasettitle = str(options['datasettitle'][0])
startdate = str(options['startdate'][0])
enddate = str(options['enddate'][0])
datafile = str(options['datafile'][0])
dbfilename = str(options['dbfilename'][0])
# print(request.POST['hydroshareusername'])
# hs = get_oauth_hs(request)
# userInfo = hs.getUserInfo()
#
hs = HydroShare(auth=auth)
# username = hs.getUserInfo()
# print(username)
abstracttext = ''
title = ''
abstracttext += 'ODM2 Admin dataset: ' + str(datasettitle)
title += 'ODM2 Admin dataset ' + str(datasettitle)
abstract = abstracttext
keywords = ['ODM2']
rtype = 'GenericResource'
fpath = datafile # str(exportdb.DATABASES['default']['NAME'])
# # print(fpath)
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"'+entered_start_date +'", "end":"'+ entered_end_date +'"}}}, {"creator":{"name":"Miguel Leon"}}]'
metadata = '[{"coverage":{"type":"period", "value":{"start":"' + str(startdate) + '", "end":"' + str(
enddate) + '"}}}, ' \
'{"creator":{"name":"' + user + '"}}]'
extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# #abstract = 'My abstract'
# #title = 'My resource'
# #keywords = ('my keyword 1', 'my keyword 2')
# #rtype = 'GenericResource'
# #fpath = 'C:/Users/leonmi/Google Drive/ODM2AdminLT2/ODM2SQliteBlank.db'
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2010"}}}, {"creator":{"name":"John Smith"}}, {"creator":{"name":"Lisa Miller"}}]'
# #extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# messages.success(request, 'Profile details updated.')
resource_id = hs.createResource(rtype, title, resource_file=datafile,
resource_filename=dbfilename, keywords=keywords,
abstract=abstract, metadata=metadata, extra_metadata=extra_metadata)
print('resource created')
print(resource_id)
def upload_epanet_model(request):
return_obj = {
'success': False,
'message': None,
'results': "",
}
if request.is_ajax() and request.method == 'POST':
try:
hs = get_oauth_hs(request)
except:
hs = HydroShare()
model_title = request.POST['model_title']
resource_filename = model_title + ".inp"
abstract = request.POST[
'model_description'] + '\n{%EPANET Model Repository%}'
title = model_title
user_keywords = ["EPANET_2.0"]
for keyword in request.POST['model_keywords'].split(","):
user_keywords.append(keyword)
keywords = (tuple(user_keywords))
rtype = 'ModelInstanceResource'
extra_metadata = '{"modelProgram": "EPANET_2.0"}'
fd, path = tempfile.mkstemp()
with os.fdopen(fd, 'w') as tmp:
tmp.write(request.POST['model_file'])
fpath = path
metadata = '[{"creator":{"name":"' + hs.getUserInfo(
)['first_name'] + ' ' + hs.getUserInfo()['last_name'] + '"}}]'
resource_id = hs.createResource(rtype,
title,
resource_file=fpath,
resource_filename=resource_filename,
keywords=keywords,
abstract=abstract,
metadata=metadata,
extra_metadata=extra_metadata)
hs.setAccessRules(resource_id, public=True)
return_obj['results'] = resource_id
return_obj['success'] = True
else:
return_obj['message'] = message_template_wrong_req_method.format(
method="GET")
return JsonResponse(return_obj)
def test_create_get_delete_resource(self):
hs = HydroShare()
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype,
title,
resource_file=fname,
keywords=keywords,
abstract=abstract)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(
os.path.join(tmpdir,
'511debf8858a4ea081f78d66870da76c.zip'),
'r') as zfile:
self.assertTrue(
'511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt'
in zfile.namelist())
downloaded = zfile.open(
'511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt',
'r')
original = open('mocks/data/minimal_resource_file.txt', 'r')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResource(newres)
self.assertEqual(delres, newres)
def upload_to_hydroshare(request):
try:
if request.method == 'GET':
get_data = request.GET
download_path = str(get_data['download_path'])
hs_username = str(get_data['hs_username'])
hs_password = str(get_data['hs_password'])
r_title = str(get_data['r_title'])
r_type = str(get_data['r_type'])
r_abstract = str(get_data['r_abstract'])
r_keywords_raw = str(get_data['r_keywords'])
r_keywords = r_keywords_raw.split(',')
#startup a Hydroshare instance with user's credentials
auth = HydroShareAuthBasic(username=hs_username, password=hs_password)
hs = HydroShare(auth=auth, hostname="dev.hydroshare.org", use_https=True)
# try to download a tiny file simply to test the user's credentials
# test_id = '49d01b5b0d0a41b6a5a31d8aace0a36e'
# hs.getResource(test_id, destination=None, unzip=False)
#download the iRODS file to a temp directory
temp_dir = tempfile.mkdtemp()
filename = os.path.basename(download_path)
download_file_path = os.path.join(temp_dir, filename)
download = requests.get(download_path, stream=True)
with open(download_file_path, 'wb') as fd:
for chunk in download.iter_content(1024):
fd.write(chunk)
#upload the temp file to HydroShare
if os.path.exists(download_file_path):
resource_id = hs.createResource(r_type, r_title, resource_file=download_file_path, keywords=r_keywords, abstract=r_abstract)
else:
if temp_dir:
# remove the temp directory/file
shutil.rmtree(temp_dir)
return JsonResponse({'error': 'An error occurred with the file upload.'})
except Exception, err:
if temp_dir:
# remove the temp directory/file
shutil.rmtree(temp_dir)
if "401 Unauthorized" in str(err):
return JsonResponse({'error': 'Username or password invalid.'})
elif "400 Bad Request" in str(err):
return JsonResponse({'error': 'File uploaded successfully despite 400 Bad Request Error.'})
else:
traceback.print_exc()
return JsonResponse({'error': 'HydroShare rejected the upload for some reason.'})
def test_create_get_delete_resource(self):
hs = HydroShare(prompt_auth=False)
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
metadata = json.dumps([{'coverage': {'type': 'period', 'value': {'start': '01/01/2000',
'end': '12/12/2010'}}},
{'creator': {'name': 'John Smith'}},
{'contributor': {'name': 'Lisa Miller'}}])
extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'})
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords,
abstract=abstract, metadata=metadata, extra_metadata=extra_metadata)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile:
self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist())
downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r')
original = open('mocks/data/minimal_resource_file.txt', 'rb')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResource(newres)
self.assertEqual(delres, newres)
def upload_to_hydroshare(request):
try:
return_json = {}
if request.method == 'POST':
get_data = request.POST
name = str(get_data['name'])
region = str(get_data['region'])
r_title = str(get_data['r_title'])
r_type = str(get_data['r_type'])
r_abstract = str(get_data['r_abstract'])
r_keywords_raw = str(get_data['r_keywords'])
r_keywords = r_keywords_raw.split(',')
auth = HydroShareAuthBasic(username='******',
password='******')
#hs = get_oauth_hs(request)
hs = HydroShare(auth=auth)
directory = os.path.join(thredds_serverpath, region)
nc_file = os.path.join(directory, name)
if os.path.exists(nc_file):
h = netCDF4.Dataset(nc_file, 'r+')
metadata = '[{"coverage":{"type":"period", "value":{"start":' + \
str(h.start_date) + ', "end":' + str(h.end_date) + \
'}}},{"creator":{"name":"*****@*****.**"}}]'
h.close()
# upload the file to HydroShare
if os.path.exists(nc_file):
resource_id = hs.createResource(r_type,
r_title,
resource_file=nc_file,
resource_filename=nc_file,
keywords=r_keywords,
abstract=r_abstract)
return_json['success'] = 'File uploaded successfully!'
return_json['newResource'] = resource_id
return_json['hs_domain'] = hs.hostname
except Exception as e:
print(e)
finally:
return JsonResponse(return_json)
def create_resource(self, uid, title, keywords, abstract):
# load user credentials
self.logger.info('Connecting to the HydroShare API')
token = json.loads(self.current_user)
# connect with HydroShare
auth = HydroShareAuthOAuth2(env.oauth_client_id, '', token=token)
hs = HydroShare(auth=auth)
# compress the subset output
self.logger.info('Compressing subset output as zip')
datapath = os.path.join(env.output_dir, uid)
shutil.make_archive(datapath, 'zip', datapath)
# create the resource using the hsapi
self.logger.info('Creating HydroShare resource')
extra_metadata = '{"appkey": "MyBinder"}'
resource_id = hs.createResource(
'CompositeResource',
title,
resource_file=f'{datapath}.zip',
keywords=keywords,
abstract=abstract,
extra_metadata=extra_metadata,
)
# options = {
# "zip_with_rel_path": f"{uid}.zip",
# "remove_original_zip": True,
# "overwrite": False
# }
# app_log.info('unzipping the HS content')
# app_log.info(options)
# result = hs.resource(resource_id).functions.unzip(options)
# app_log.info(result)
# redirect to the HS page when finished
self.redirect(
f'https://hydroshare.org/resource/{resource_id}?resource-mode=edit'
)
def test_create_get_delete_resource(self):
hs = HydroShare()
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile:
self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist())
downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r')
original = open('mocks/data/minimal_resource_file.txt', 'r')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
delres = hs.deleteResource(newres)
self.assertEqual(delres, newres)
def run_service(user_input):
auth = HydroShareAuthBasic(hs_username, hs_password)
hs = HydroShare(auth=auth)
client = HydroDS(username=ci_username, password=ci_password)
# step1: get raster resource files from HydroShare # TODO remember to check if the resource is downloadable
resource_id = user_input['input_raster_url_path']
destination = tempfile.mkdtemp()
hs.getResource(resource_id, destination, unzip=True)
raster_file_folder = '{0}/{1}/{2}/data/contents'.format(destination, resource_id, resource_id)
raster_file_path_list = [os.path.join(raster_file_folder, f) for f in os.listdir(raster_file_folder) if
'.vrt' not in f]
upload_file_url = client.upload_file(file_to_upload=raster_file_path_list[0])
# step2: processing files in HydroDS
output_raster_name = 'process_{}'.format(upload_file_url.split('/')[-1])
output_file = client.project_resample_raster(input_raster_url_path=upload_file_url,
cell_size_dx=int(user_input['cell_size_dx']),
cell_size_dy=int(user_input['cell_size_dy']),
output_raster=output_raster_name,
resample=user_input['resample'],
epsg_code=int(user_input['proj_code']) if user_input['proj_code_type']=='epsg_code' else None,
utm_zone=int(user_input['proj_code']) if user_input['proj_code_type']=='utm_zone' else None,
)
# step3: download file and create HydroShare resource
save_as = '{}/{}'.format(destination, output_file['output_raster'].split('/')[-1])
client.download_file(file_url_path=output_file['output_raster'], save_as=save_as)
title = user_input['output_raster']
rtype = 'RasterResource'
resource_id = hs.createResource(rtype, title, resource_file=save_as)
# step4: clean up the temp in Tethys and HydroDS
shutil.rmtree(destination)
client.delete_my_file(output_file['output_raster'].split('/')[-1])
client.delete_my_file(upload_file_url.split('/')[-1])
return {'is_success': True, 'message': 'http://www.hydroshare.org/resource/{}/'.format(resource_id)}
def upload_file(request):
# f = open("demofile3.txt", "w")
# f.write("Woops! I have deleted the content!")
# f.close()
# auth = HydroShareAuthBasic(username='******', password='******')
# hs = HydroShare(auth=auth)
#fpath = '/apps/geocode/upload_file/output.txt'
fpath = 'tethysapp/geocode/workspaces/app_workspace/output.txt'
hs = HydroShare(auth=auth)
abstract = 'My abstract'
title = 'My resource'
keywords = ('my keyword 1', 'my keyword 2')
rtype = 'GenericResource'
metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2010"}}}, {"creator":{"name":"John Smith"}}, {"creator":{"name":"Lisa Miller"}}]'
extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
resource_id = hs.createResource(rtype, title, resource_file=fpath, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata)
resource_id = hs.addResourceFile('99319811b9c44f03aa14f47a32aa4111', fpath)
def handle(self, *args,
**options): # (f,fileid, databeginson,columnheaderson, cmd):
# cmdline = bool(options['cmdline'][0])
username = str(options['username'][0])
password = str(options['password'][0])
hs_client_id = settings.SOCIAL_AUTH_HYDROSHARE_UP_KEY
hs_client_secret = settings.SOCIAL_AUTH_HYDROSHARE_UP_SECRET
auth = HydroShareAuthOAuth2(hs_client_id,
hs_client_secret,
username=username,
password=password)
# print(username)
# print(password)
export_complete = True
resource_link = ''
user = str(options['django_user_name'][0])
datasettitle = str(options['datasettitle'][0])
startdate = str(options['startdate'][0])
enddate = str(options['enddate'][0])
datafile = str(options['datafile'][0])
dbfilename = str(options['dbfilename'][0])
# print(request.POST['hydroshareusername'])
# hs = get_oauth_hs(request)
# userInfo = hs.getUserInfo()
#
hs = HydroShare(auth=auth)
# username = hs.getUserInfo()
# print(username)
abstracttext = ''
title = ''
abstracttext += 'CZ Manager dataset: ' + str(datasettitle)
title += 'CZ Manager dataset ' + str(datasettitle)
abstract = abstracttext
keywords = ['ODM2']
rtype = 'GenericResource'
fpath = datafile # str(exportdb.DATABASES['default']['NAME'])
# # print(fpath)
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"'+entered_start_date +'", "end":"'+ entered_end_date +'"}}}, {"creator":{"name":"Miguel Leon"}}]'
metadata = '[{"coverage":{"type":"period", "value":{"start":"' + str(startdate) + '", "end":"' + str(
enddate) + '"}}}, ' \
'{"creator":{"name":"' + user + '"}}]'
extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# #abstract = 'My abstract'
# #title = 'My resource'
# #keywords = ('my keyword 1', 'my keyword 2')
# #rtype = 'GenericResource'
# #fpath = 'C:/Users/leonmi/Google Drive/ODM2AdminLT2/ODM2SQliteBlank.db'
# #metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2010"}}}, {"creator":{"name":"John Smith"}}, {"creator":{"name":"Lisa Miller"}}]'
# #extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
# messages.success(request, 'Profile details updated.')
resource_id = hs.createResource(rtype,
title,
resource_file=datafile,
resource_filename=dbfilename,
keywords=keywords,
abstract=abstract,
metadata=metadata,
extra_metadata=extra_metadata)
print('resource created')
print(resource_id)
class HydroShareUtility:
def __init__(self):
self.client = None # type: HydroShare
self.auth = None
self.user_info = None
self.re_period = re.compile(r'(?P<tag_start>^start=)(?P<start>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_end>end=)'
r'(?P<end>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_scheme>scheme=)(?P<scheme>.+$)', re.I)
self.xml_ns = {
'dc': "http://purl.org/dc/elements/1.1/",
'dcterms': "http://purl.org/dc/terms/",
'hsterms': "http://hydroshare.org/terms/",
'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#",
'rdfs1': "http://www.w3.org/2001/01/rdf-schema#"}
self.time_format = '%Y-%m-%dT%H:%M:%S'
self.xml_coverage = 'start={start}; end={end}; scheme=W3C-DTF'
def authenticate(self, username, password, client_id=None, client_secret=None):
"""
Authenticates access to allow read/write access to privileged resource_cache
:param username: username for HydroShare.org
:param password: password associated with username
:param client_id: Client ID obtained from HydroShare
:param client_secret: Client Secret provided by HydroShare
:return: Returns true if authentication was successful, false otherwise
"""
if not all([username, password]):
self.auth = None
return False
if client_id is not None and client_secret is not None:
self.auth = HydroShareAuthOAuth2(client_id, client_secret, username=username, password=password)
else:
self.auth = HydroShareAuthBasic(username, password)
try:
self.client = HydroShare(auth=self.auth) # , verify=False)
self.user_info = self.client.getUserInfo()
return True
except HydroShareException as e: # for incorrect username/password combinations
print('Authentication failed: {}'.format(e))
except InvalidGrantError as e: # for failures when attempting to use OAuth2
print('Credentials could not be validated: {}'.format(e))
except InvalidClientError as e:
print('Invalid client ID and/or client secret: {}'.format(e))
except Exception as e:
print(e)
self.auth = None
return False
def purgeDuplicateGamutFiles(self, resource_id, regex, confirm_delete=False):
"""
Removes all files that have a duplicate-style naming pattern (e.g. ' (1).csv', '_ASDFGJK9.csv'
:param resource_id: Resource to inspect for duplicates
:type resource_id: Resource object received from the HydroShare API client
:param confirm_delete: If true, requires input that confirm file should be deleted
:type confirm_delete: bool
"""
from collections import defaultdict
re_breakdown = re.compile(regex, re.I)
resource_files = self.getResourceFileList(resource_id) # type: [dict]
duplicates_list = []
for remote_file in resource_files:
url = remote_file.get('url', '')
results = re_breakdown.match(url) # Check the file URL for expected patterns
temp_dict = defaultdict(lambda: '') # type: dict
if results:
temp_dict['duplicated'] = results.groupdict()
if len(temp_dict.get('duplicated', '')): # A non-duplicated file will match with length 0
duplicates_list.append(temp_dict) # Save the file name so we can remove it later
for file_detail in duplicates_list:
if not confirm_delete:
delete_me = True
else:
user_answer = raw_input("Delete file {} [Y/n]: ".format(file_detail['name']))
if user_answer != 'N' and user_answer != 'n':
delete_me = True
else:
delete_me = False
if delete_me:
self.client.deleteResourceFile(resource_id, file_detail['name'])
print('Deleting file {}...'.format(file_detail['name']))
else:
print('Skipping duplicate file {}...'.format(file_detail['name']))
def getResourceFileList(self, resource_id):
"""
:param resource_id: ID of resource for which to retrieve a file list
:type resource_id: str
:return: List of files in resource
:rtype: list of str
"""
try:
return list(self.client.getResourceFileList(resource_id))
except Exception as e:
print('Error while fetching resource files {}'.format(e))
return []
def getAllResources(self):
filtered_resources = {}
owner = self.user_info['username']
if self.auth is None:
raise HydroShareUtilityException("Cannot query resources without authentication")
all_resources = self.client.resources(owner=owner)
for resource in all_resources:
resource_object = HydroShareResource(resource)
filtered_resources[resource_object.id] = resource_object
return filtered_resources
def getMetadataForResource(self, resource):
"""
:type resource: HydroShareResource
"""
metadata = self.client.getScienceMetadata(resource.id)
resource.title = metadata.get('title', '')
resource.subjects = [item['value'] for item in metadata.get('subjects', [])]
resource.abstract = metadata.get('description', '')
if resource.abstract is None:
resource.abstract = ''
if 'funding_agencies' in metadata and len(metadata['funding_agencies']) > 0:
funding_agency = metadata['funding_agencies'][0]
resource.agency_url = funding_agency['agency_url'] if 'agency_url' in funding_agency else ''
resource.funding_agency = funding_agency['agency_name'] if 'agency_name' in funding_agency else ''
resource.award_number = funding_agency['award_number'] if 'award_number' in funding_agency else ''
resource.award_title = funding_agency['award_title'] if 'award_title' in funding_agency else ''
def updateResourceMetadata(self, resource):
"""
:type resource: HydroShareResource
"""
return self.client.updateScienceMetadata(resource.id, resource.get_metadata())
def _request(self, method, url, params=None, data=None, files=None, headers=None, stream=False):
request = self.client.session.request(method, url, params=params, data=data, files=files, headers=headers,
stream=stream)
return request
def requestAccessRules(self, resource):
"""
Get access rule for a resource.
"""
url = "{url_base}/resource/{pid}/sysmeta/".format(url_base=self.client.url_base, pid=resource.id)
r = self._request('GET', url)
if r.status_code != 200:
raise Exception("Failed to get system metadata for resource: {}".format(resource.id))
data = r.json()
resource.public = data.get('public', False)
resource.shareable = data.get('shareable', False)
def makePublic(self, resource, public=True):
"""
Makes a resource public or private
:param resource: The resource you want to modify
:param public: boolean value, True makes the resource public, False makes it private (wowzer)
:return: None
"""
hs = HydroShare(auth=self.auth)
res = hs.resource(resource.id).public(public)
if res.status_code == 200 or res.status_code == 202:
resource.public = public
def updateKeywords(self, resource, keywords=None):
if keywords is None:
keywords = resource.keywords
# remove leading/trailing whitespaces from keywords
keywords = map(lambda x: x.strip(), keywords)
url = "{url_base}/resource/{id}/scimeta/elements/".format(url_base=self.client.url_base, id=resource.id)
subjects = []
for keyword in keywords:
subjects.append({"value": keyword})
r = self.client.session.request('PUT', url, json={"subjects": subjects})
if r.status_code != 202:
raise HydroShareException((url, 'PUT', r.status_code, keywords))
return r.json()
def getFileListForResource(self, resource):
resource.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource.id)]
return resource.files
def getFilesByResourceId(self, resource_id):
return [os.path.basename(f['url']) for f in self.getResourceFileList(resource_id)]
def filterResourcesByRegex(self, regex_string=None, owner=None, regex_flags=re.IGNORECASE):
"""
Apply a regex filter to all available resource_cache. Useful for finding GAMUT resource_cache
:param owner: username of the owner of the resource
:type owner: string
:param regex_string: String to be used as the regex filter
:param regex_flags: Flags to be passed to the regex search
:return: A list of resource_cache that matched the filter
"""
filtered_resources = []
if owner is None:
owner = self.user_info['username']
if self.auth is None:
raise HydroShareUtilityException("Cannot query resources without authentication")
all_resources = self.client.resources(owner=owner)
regex_filter = re.compile(regex_string, regex_flags)
for resource in all_resources:
if regex_string is not None and regex_filter.search(resource['resource_title']) is None:
continue
resource_object = HydroShareResource(resource)
resource_object.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource_object.id)]
filtered_resources.append(resource_object)
return filtered_resources
def UploadFiles(self, files, resource): # type: ([str], HydroShareResource) -> bool
if self.auth is None:
raise HydroShareUtilityException("Cannot modify resources without authentication")
try:
for csv_file in files:
try:
self.client.deleteResourceFile(resource.id, os.path.basename(csv_file))
except HydroShareNotFound:
pass
# except Exception as e:
# if APP_SETTINGS.H2O_DEBUG and APP_SETTINGS.VERBOSE:
# print 'File did not exist in remote: {}, {}'.format(type(e), e)
if type(csv_file) != str:
csv_file = str(csv_file)
self.client.addResourceFile(resource.id, csv_file)
msg = "File {} uploaded to remote {}".format(os.path.basename(csv_file), repr(resource))
print(msg)
pub.sendMessage('logger', message=msg)
except HydroShareException as e:
print("Upload failed - could not complete upload to HydroShare due to exception: {}".format(e))
return False
except KeyError as e:
print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e))
return False
return True
def setResourcesAsPublic(self, resource_ids):
if self.auth is None:
raise HydroShareUtilityException("Cannot modify resources without authentication")
for resource_id in resource_ids:
try:
print('Setting resource {} as public'.format(resource_id))
self.client.setAccessRules(resource_id, public=True)
except HydroShareException as e:
print("Access rule edit failed - could not set to public due to exception: {}".format(e))
except KeyError as e:
print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e))
def deleteFilesInResource(self, resource):
if self.auth is None:
raise HydroShareUtilityException("Cannot modify resources without authentication")
try:
file_list = self.getResourceFileList(resource.id)
for file_info in file_list:
msg = 'Deleting resource file: {}'.format(os.path.basename(file_info['url']))
print(msg)
pub.sendMessage('logger', message=msg)
self.client.deleteResourceFile(resource.id, os.path.basename(file_info['url']))
except Exception as e:
print('Could not delete files in resource {}\n{}'.format(resource.id, e))
def getResourceCoveragePeriod(self, resource_id):
metadata = self.client.getScienceMetadata(resource_id)
period_start = None
period_end = None
try:
xml_tree = ElementTree.fromstring(metadata)
description_node = xml_tree.find('rdf:Description', namespaces=self.xml_ns)
coverage_node = description_node.find('dc:coverage', namespaces=self.xml_ns)
period_node = coverage_node.find('dcterms:period', namespaces=self.xml_ns)
value_node = period_node.find('rdf:value', namespaces=self.xml_ns)
match = self.re_period.match(value_node.text)
if match is not None:
period_start = dateutil.parser.parse(match.group('start'))
period_end = dateutil.parser.parse(match.group('end'))
except Exception as e:
print("Unable to find coverage data - encountered exception {}".format(e))
return period_start, period_end
def deleteResource(self, resource_id, confirm=True):
if self.auth is None:
raise HydroShareUtilityException("Cannot modify resources without authentication")
try:
if confirm:
user_input = raw_input('Are you sure you want to delete the resource {}? (y/N): '.format(resource_id))
if user_input.lower() != 'y':
return
print('Deleting resource {}'.format(resource_id))
self.client.deleteResource(resource_id)
except Exception as e:
print('Exception encountered while deleting resource {}: {}'.format(resource_id, e))
def createNewResource(self, resource): # type: (ResourceTemplate) -> HydroShareResource
"""
:param resource:
:type resource: ResourceTemplate
:return:
:rtype: str
"""
if self.auth is None:
raise HydroShareUtilityException("Cannot create resource without authentication")
# http://hs-restclient.readthedocs.io/en/latest/
if resource is not None:
metadata = []
fundingagency = {}
fundingagency_attr_map = (
('funding_agency', 'agency_name'),
('award_title', 'award_title'),
('award_number', 'award_number'),
('agency_url', 'agency_url')
)
for reskey, fakey in fundingagency_attr_map:
# reskey - resource attribute name
# fakey - funding agency attribute name
value = getattr(resource, reskey).strip()
if len(value):
fundingagency[fakey] = value
if len(fundingagency.keys()):
metadata.append({'fundingagency': fundingagency})
resource_id = self.client.createResource(resource_type='CompositeResource',
title=resource.title,
abstract=resource.abstract,
keywords=resource.keywords,
metadata=json.dumps(metadata, encoding='ascii'))
hs_resource = HydroShareResource({'resource_id': resource_id})
self.getMetadataForResource(hs_resource)
return hs_resource
return None
class hydroshare():
def __init__(self, username=None):
self.hs = None
self.content = {}
# load the HS environment variables
# self.load_environment()
uname = username
if uname is None:
uname = os.environ['HS_USR_NAME']
# get a secure connection to hydroshare
auth = self.getSecureConnection(uname)
try:
self.hs = HydroShare(auth=auth)
self.hs.getUserInfo()
display(HTML('<b style="color:green;">Successfully established a connection with HydroShare</b>'))
except HydroShareHTTPException as e:
display(HTML(
'<p style="color:red;"><b>Failed to establish a connection with HydroShare. Please check that you provided the correct credentials</b><br>%s </p>' % e))
# remove the cached authentication
auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth')
os.remove(auth_path)
return None
def _getResourceFromHydroShare(self, resourceid, destination='.', unzip=True):
# download the resource
pid = self.hs.getResource(resourceid, destination=destination, unzip=unzip)
threadResults.put(pid)
def _createHydroShareResource(self, res_type, title, abstract, content_file,
keywords=[]):
resid = self.hs.createResource(res_type, title, resource_file=content_file,
keywords=keywords, abstract=abstract)
threadResults.put(resid)
def _addContentToExistingResource(self, resid, content_files):
for f in content_files:
self.hs.addResourceFile(resid, f)
def load_environment(self):
env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'env')
with open(env_path, 'r') as f:
lines = f.readlines()
print('Adding the following system variables:')
for line in lines:
k, v = line.strip().split('=')
os.environ[k] = v
print(' %s = %s' % (k, v))
print('\nThese can be accessed using the following command: ')
print(' os.environ[key]')
print('\n (e.g.)\n os.environ["HS_USR_NAME"] => %s' % os.environ['HS_USR_NAME'])
def getSecureConnection(self, username):
"""
Establishes a secure connection with HydroShare.
Args:
email: email address associated with HydroShare
Returns:
HydroShare connection
"""
auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth')
if not os.path.exists(auth_path):
print('\nThe hs_utils library requires a secure connection to your HydroShare account.')
# p = getpass.getpass('Enter the HydroShare password for user \'%s\': ' % username)
p = '7jmftUpata'
auth = HydroShareAuthBasic(username=username, password=p)
with open(auth_path, 'wb') as f:
pickle.dump(auth, f, protocol=2)
else:
with open(auth_path, 'rb') as f:
auth = pickle.load(f)
return auth
def getResourceMetadata(self, resid):
science_meta = self.hs.getScienceMetadata(resid)
system_meta = self.hs.getSystemMetadata(resid)
return ResourceMetadata(system_meta, science_meta)
def createHydroShareResource(self, abstract, title, derivedFromId, keywords=[], resource_type='GenericResource',
content_files=[], public=False):
# query the hydroshare resource types and make sure that resource_type is valid
restypes = {r.lower(): r for r in self.hs.getResourceTypes()}
try:
res_type = restypes[resource_type]
except KeyError:
display(HTML('<b style="color:red;">[%s] is not a valid HydroShare resource type.</p>' % resource_type))
return None
# get the 'derived resource' metadata
if derivedFromId is not None:
try:
# update the abstract and keyword metadata
meta = self.getResourceMetadata(derivedFromId)
abstract = meta.abstract + '\n\n[Modified in JupyterHub on %s]\n%s' % (dt.now(), abstract)
keywords = set(keywords + meta.keywords)
except:
display(HTML(
'<b style="color:red;">[%s] is not a valid HydroShare resource id for setting the "derivedFrom" attribute.</p>' % derivedFromId))
return None
else:
response = input(
'You have indicated that this resource is NOT derived from any existing HydroShare resource. Are you sure that this is what you intended? [Y/n]')
if response == 'n':
display(HTML('<b style="color:red;">Resource creation aborted.</p>'))
return
f = None if len(content_files) == 0 else content_files[0]
# create the hs resource (1 content file allowed)
t = threading.Thread(target=self._createHydroShareResource, args=(res_type, title, abstract, f),
kwargs={'keywords': keywords})
resid = runThreadedFunction(t, msg='Creating HydroShare Resource', success='Resource Creation Successful')
# add the remaining content files to the hs resource
self.addContentToExistingResource(resid, content_files[1:])
display(HTML('Resource id: %s' % resid))
display(HTML('<a href=%s target="_blank">%s<a>' % (
'https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare')))
def getResourceFromHydroShare(self, resourceid, destination='.'):
"""
Downloads the content of HydroShare resource to the JupyterHub userspace
Args:
resourceid: id of the HydroShare resource to query
destination: path relative to /user/[username]/notebooks/data
"""
default_dl_path = 'C:\\Users\\12672\\Box\\data\\NEON\\lczodata\\' # os.environ['DATA']
dst = os.path.abspath(os.path.join(default_dl_path, destination))
download = True
# check if the data should be overwritten
dst_res_folder = os.path.join(dst, resourceid)
if os.path.exists(dst_res_folder):
res = input(
'This resource already exists in your userspace.\nWould you like to overwrite this data [Y/n]? ')
if res != 'n':
shutil.rmtree(dst_res_folder)
else:
download = False
# re-download the content if desired
if download:
try:
# get some metadata about the resource that will be downloaded
res_meta = self.hs.getSystemMetadata(resourceid)
header = requests.head(res_meta['bag_url'])
# download the resource (threaded)
t = threading.Thread(target=self._getResourceFromHydroShare,
args=(resourceid,), kwargs={'destination': dst, 'unzip': True})
runThreadedFunction(t, msg='Downloading', success='Download Completed Successfully')
except Exception as e:
display(HTML('<b style="color:red">Failed to retrieve resource content from HydroShare: %s</b>' % e))
return None
# load the resource content
outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid))
content_files = glob.glob(os.path.join(outdir, 'data/contents/*'))
# display(HTML('Your Content is located at: %s' % outdir))
content = {}
for f in content_files:
fname = os.path.basename(f)
content[fname] = f
display_resource_content_files(content)
# check_for_ipynb(content_files)
# update the content dictionary
self.content.update(content)
def addContentToExistingResource(self, resid, content):
t = threading.Thread(target=self._addContentToExistingResource, args=(resid, content))
runThreadedFunction(t, msg='Adding Content to Resource', success='Successfully Added Content Files')
def loadResource(self, resourceid):
resdir = find_resource_directory(resourceid)
if resdir is None:
display(HTML(
'<b style="color:red">Could not find any resource matching the id [%s].</b> <br> It is likely that this resource has not yet been downloaded from HydroShare.org, or it was removed from the JupyterHub server. Please use the following command to aquire the resource content: <br><br> <code> hs.getResourceFromHydroShare(%s)</code>.' % (
resourceid, resourceid)))
return
content_files = glob.glob(os.path.join(resdir, '%s/data/contents/*' % resourceid))
display(HTML('<p>Downloaded content is located at: %s</p>' % resdir))
display(HTML('<p>Found %d content file(s). \n</p>' % len(content_files)))
content = {}
for f in content_files:
fname = os.path.basename(f)
content[fname] = f
display_resource_content_files(content)
self.content = content
def test_create_get_delete_resource(self):
hs = HydroShare(prompt_auth=False)
abstract = 'Abstract for hello world resource'
title = 'Minimal hello world resource'
keywords = ('hello', 'world')
rtype = 'GenericResource'
fname = 'mocks/data/minimal_resource_file.txt'
metadata = json.dumps([{
'coverage': {
'type': 'period',
'value': {
'start': '01/01/2000',
'end': '12/12/2010'
}
}
}, {
'creator': {
'name': 'John Smith'
}
}, {
'contributor': {
'name': 'Lisa Miller'
}
}])
extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'})
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Create
newres = hs.createResource(rtype,
title,
resource_file=fname,
keywords=keywords,
abstract=abstract,
metadata=metadata,
extra_metadata=extra_metadata)
self.assertIsNotNone(newres)
sysmeta = hs.getSystemMetadata(newres)
self.assertEqual(sysmeta['resource_id'], newres)
self.assertEqual(sysmeta['resource_type'], rtype)
self.assertFalse(sysmeta['public'])
with HTTMock(mocks.hydroshare.accessRules_put):
# Make resource public
hs.setAccessRules(newres, public=True)
sysmeta = hs.getSystemMetadata(newres)
self.assertTrue(sysmeta['public'])
with HTTMock(mocks.hydroshare.createResourceCRUD):
# Get
tmpdir = tempfile.mkdtemp()
hs.getResource(newres, destination=tmpdir)
with ZipFile(
os.path.join(tmpdir,
'511debf8858a4ea081f78d66870da76c.zip'),
'r') as zfile:
self.assertTrue(
'511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt'
in zfile.namelist())
downloaded = zfile.open(
'511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt',
'r')
original = open('mocks/data/minimal_resource_file.txt', 'rb')
self.assertEqual(downloaded.read(), original.read())
downloaded.close()
original.close()
shutil.rmtree(tmpdir)
# Delete
hs.deleteResource(newres)
class ResourceManager:
""" Class that defines a handler for working with all of a user's resources.
This is where they will be able to delete a resource, create a new one, or
just get the list of a user's resources in hydroshare or jupyterhub.
"""
def __init__(self):
"""Makes an output folder for storing HS files locally, if none exists,
and sets up authentication on hydroshare API.
"""
config = get_config_values(['dataPath', 'hydroShareHostname'])
self.hs_hostname = 'www.hydroshare.org'
# TODO: Rename to hydroshare_resource_data
self.output_folder = Path('local_hs_resources')
if config:
if 'hydroShareHostname' in config:
self.hs_hostname = config['hydroShareHostname']
if 'dataPath' in config:
self.output_folder = Path(config['dataPath'])
if not self.output_folder.is_dir():
# Let any exceptions that occur bubble up
self.output_folder.mkdir(parents=True)
self.hs_api_conn = None # Use 'authenticate' to initialize
self.username = None
def authenticate(self, username=None, password=None, save=False):
""" Attempts to authenticate with HydroShare.
:param username: the user's HydroShare username
:type username: str
:param password: the user's HydroShare password
:type password: str
:param save: whether or not to save the credentials to the config file if the authentication succeeds
:type save: bool
:return: the user's information (name, email, etc) from HydroShare if successful, None otherwise
"""
if not username or not password:
# Check the config file for a username and password
config = get_config_values(['u', 'p'])
if not config or 'u' not in config or 'p' not in config:
return None # No passed credentials and no saved credentials -- can't authenticate
username = config['u']
password = base64.b64decode(config['p']).decode('utf-8')
# Try to authenticate
auth = HydroShareAuthBasic(username=username, password=password)
self.hs_api_conn = HydroShare(auth=auth, hostname=self.hs_hostname)
try:
user_info = self.hs_api_conn.getUserInfo()
self.username = user_info.get('username')
except HydroShareHTTPException as e:
if e.status_code == 401: # Unauthorized
return None # Authentication failed
raise e # Some other error -- bubble it up
# Authentication succeeded
if save:
# Save the username and password
saved_successfully = set_config_values({
'u':
username,
'p':
str(
base64.b64encode(
password.encode('utf-8')).decode('utf-8')),
})
if saved_successfully:
logging.info(
'Successfully saved HydroShare credentials to config file.'
)
return user_info # Authenticated successfully
def is_authenticated(self):
if self.hs_api_conn is None:
self.authenticate()
return self.hs_api_conn is not None
def save_resource_locally(self, res_id):
""" Downloads a resource to the local filesystem if a copy does not already exist locally
:param res_id: the resource ID
:type res_id: str
"""
# Get resource from HS if it doesn't already exist locally
if not (self.output_folder / res_id).exists():
logging.info(f"Downloading resource {res_id} from HydroShare...")
self.hs_api_conn.getResource(res_id,
destination=self.output_folder,
unzip=True)
def get_user_info(self):
"""Gets information about the user currently logged into HydroShare
"""
user_info = None
error = None
try:
user_info = self.hs_api_conn.getUserInfo()
except HydroShareHTTPException as e:
if e.status_code == 401: # Unauthorized
error = HYDROSHARE_AUTHENTICATION_ERROR
else:
error = {
'type': 'GenericHydroShareHTTPException',
'message': e.status_msg,
}
return user_info, error
def delete_resource_locally(self, res_id):
""" Attempts to delete the local copy of the resource files from the disk
:param res_id: the ID of the resource to delete
:type res_id: str
"""
resource_path = self.output_folder / res_id
if not resource_path.exists():
return {
'type':
'FileNotFoundError',
'message':
f'Could not find a local copy of resource {res_id} to delete.',
}
try:
shutil.rmtree(str(resource_path))
except IOError as e:
return {
'type':
'IOError',
'message':
f'Something went wrong. Could not delete resource {res_id}.',
}
return None # No error
def delete_resource_from_hs(self, res_id):
try:
self.hs_api_conn.deleteResource(res_id)
except HydroShareHTTPException as e:
return {
'type': 'GenericHydroShareHTTPException',
'message': e.status_msg,
}
return None # No error
def get_local_resource_ids(self):
""" Gets a list of IDs of all the resources saved locally
:return a set of strings containing all of the resource IDs
:rtype set<str>
"""
return set(map(lambda p: p.name, self.output_folder.glob('*')))
def get_list_of_user_resources(self):
# TODO: speed this up
"""Gets list of all the resources for the logged in user, including
those stored on hydroshare and those stored locally on jupyterhub
and information about each one including whether HS ones are stored
locally.
Assumes there are no local resources that don't exist in HS
"""
error = None
resources = {}
# Get local res_ids
self.local_res_ids = self.get_local_resource_ids()
# Get the user's resources from HydroShare
user_hs_resources = self.hs_api_conn.resources(owner=self.username)
try:
hs_resources = list(
user_hs_resources) # Resources can't be listed if auth fails
except AttributeError:
return None, HYDROSHARE_AUTHENTICATION_ERROR
for res in hs_resources:
res_id = res['resource_id']
resources[res_id] = {
'abstract': res.get('abstract'),
'authors': res.get('authors'),
'creator': res.get('creator'),
'created': res.get('date_created'),
'lastUpdated': res.get('date_last_updated'),
'localCopyExists': res_id in self.local_res_ids,
'localFiles': res.get('localFiles'),
'id': res_id,
'isPublic': res.get('public'),
'published': res.get('published'),
'status': res.get('status'),
'title': res.get('resource_title'),
'url': res.get('resource_url'),
}
return list(resources.values()), error
def create_HS_resource(self,
resource_title,
creators,
abstract="",
privacy="Private"):
"""
Creates a hydroshare resource from the metadata specified in a dict
using the given resource_title and specified creators
"""
error = None
resource_id = None
# Type errors for resource_title and creators
if not isinstance(resource_title, str):
error = {
'type': 'IncorrectType',
'message': 'Resource title should be a string.'
}
return resource_id, error
if not isinstance(creators, list) or not all(
isinstance(creator, str) for creator in creators):
error = {
'type': 'IncorrectType',
'message': '"Creators" object should be a list of strings.'
}
return resource_id, error
if abstract is None:
abstract = ""
if privacy == "Public":
public = True
else:
public = False
meta_metadata = []
for creator in creators:
meta_metadata.append({"creator": {"name": creator}})
meta_metadata = json.dumps(meta_metadata)
metadata = {
'abstract': abstract,
'title': resource_title,
'keywords': (),
'rtype': 'GenericResource',
'fpath': '',
'metadata': meta_metadata,
'extra_metadata': ''
}
resource_id = ''
try:
resource_id = self.hs_api_conn.createResource(
metadata['rtype'],
metadata['title'],
resource_file=metadata['fpath'],
keywords=metadata['keywords'],
abstract=metadata['abstract'],
metadata=metadata['metadata'],
extra_metadata=metadata['extra_metadata'])
self.hs_api_conn.setAccessRules(resource_id, public=public)
except:
error = {
'type': 'UnknownError',
'message': 'Unable to create resource.'
}
return resource_id, error
def create_copy_of_resource_in_hs(self, src_res_id):
""" Makes a copy of a resource in HydroShare and updates the local copy of the
old resource to point to then ew copy.
:param src_res_id: the ID of the resource in HydroShare to duplicate
:type src_res_id: str
:returns the new resource ID
"""
response = self.hs_api_conn.resource(src_res_id).copy()
new_id = response.content.decode("utf-8") # b'id'
is_local = src_res_id in self.local_res_ids
# if a local version exists under the old resource ID, rename it to point to the new one
if is_local:
og_path = self.output_folder / src_res_id / src_res_id
new_path = self.output_folder / new_id / new_id
shutil.move(str(og_path), str(new_path))
# Change name to 'Copy of []'
title = self.hs_api_conn.getScienceMetadata(new_id)['title']
new_title = "Copy of " + title
self.hs_api_conn.updateScienceMetadata(new_id, {"title": new_title})
return new_id
def get_archive_message(self):
""" Gets the message to display on the resource page prompting the user to archive
their resources to HydroShare
"""
loaded_archive_message = get_config_values(['archiveMessage'])
if loaded_archive_message and loaded_archive_message['archiveMessage']:
archive_message = loaded_archive_message['archiveMessage']
else:
archive_message = input("Please enter the archive message: ")
#Save the archive message
saved_successfully = set_config_values({
'archiveMessage':
archive_message,
})
if saved_successfully:
logging.info(
'Successfully saved archive message to config file.')
return archive_message
def create_hydroshare_resource(context,
auth,
title,
hydroshare_host='www.hydroshare.org',
hydroshare_port=None, use_https=False,
resource_type='GenericResource',
abstract=None,
keywords=None,
create_callback=None,
verbose=False,
outfp=sys.stderr):
"""
Create HydroShare resource of an EcohydroLib project
@param context ecohydrolib.Context object
@param auth hs_restclient.HydroShareAuth object
@param title string representing the title of the resource
@param hydroshare_host string representing DNS name of the HydroShare
server in which to create the resource
@param hydroshare_port int representing the TCP port of the HydroShare
server
@param use_https True if HTTPS should be used. Default: False
@param resource_type string representing the HydroShare resource type
that should be used to create the resource
@param abstract string representing the abstract of the resource
@param keywords list of strings representing the keywords to assign
to the resource
@param create_callback user-defined callable that takes as input a
file size in bytes, and generates a callable to provide feedback
to the user about the progress of the upload of resource_file.
For more information, see:
http://toolbelt.readthedocs.org/en/latest/uploading-data.html#monitoring-your-streaming-multipart-upload
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@return string representing the ID of the newly created resource
"""
temp_dir = tempfile.mkdtemp()
zip_filename = "{0}.zip".format(os.path.basename(context.projectDir))
zip_filepath = os.path.join(temp_dir, zip_filename)
# Zip up the project for upload ...
if verbose:
outfp.write("Creating archive...")
outfp.flush()
zfile = zipfile.ZipFile(zip_filepath, 'w',
zipfile.ZIP_DEFLATED,
True)
os.path.walk(context.projectDir, _addToZip, (context.projectDir, zfile))
zfile.close()
if verbose:
outfp.write("done\n")
outfp.flush()
# Make upload progress callback
progress_callback = None
if create_callback:
s = os.stat(zip_filepath)
progress_callback = create_callback(s.st_size)
if verbose:
outfp.write("Uploading to HydroShare...")
outfp.flush()
# This next line is needed for some text-based progress indicators
# to properly render.
outfp.write(" \nThis is a hack\r")
outfp.flush()
if hydroshare_host:
hs = HydroShare(hostname=hydroshare_host, port=hydroshare_port,
use_https=use_https, auth=auth)
else:
hs = HydroShare(port=hydroshare_port,
use_https=use_https, auth=auth)
resource_id = None
try:
resource_id = hs.createResource(resource_type, title,
resource_file=zip_filepath, resource_filename=zip_filename,
abstract=abstract, keywords=keywords,
progress_callback=progress_callback)
if verbose:
url = _getResourceURL(resource_id,
hydroshare_host, hs_port=hydroshare_port,
use_https=use_https)
outfp.write("\nNew resource created at:\n{0}\n".format(url))
outfp.flush()
except Exception as e:
raise e
finally:
shutil.rmtree(temp_dir)
return resource_id
from hs_restclient import HydroShare, HydroShareAuthBasic
auth = HydroShareAuthBasic(username="******", password="******")
hs = HydroShare(auth=auth)
abstract = 'My short abstract.'
title = 'My resource'
keywords = ('my keyword 1', 'my keyword 2')
rtype = 'GenericResource'
fpath = '/home/christian/Desktop/hsJson'
metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2018"}}}, {"creator":{"name":" Christian E. Camacho"}}]'
extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}'
resource_id = hs.createResource(rtype,
title,
resource_file=fpath,
keywords=keywords,
abstract=abstract,
metadata=metadata,
extra_metadata=extra_metadata)
def upload_to_hs(uploadtype, modelname, resource_name, resource_abstract,
resource_key):
dbs = {
'zone': zone,
'mult': mult,
'pval': pval,
'bas6': bas6,
'dis': dis,
'disu': disu,
'bcf6': bcf6,
'lpf': lpf,
'hfb6': hfb6,
'chd': chd,
'fhb': fhb,
'wel': wel,
'mnw1': mnw1,
'mnw2': mnw2,
'mnwi': mnwi,
'drn': drn,
'rch': rch,
'evt': evt,
'ghb': ghb,
'gmg': gmg,
'lmt6': lmt6,
'lmt7': lmt7,
'riv': riv,
'str': str,
'swi2': swi2,
'pcg': pcg,
'pcgn': pcgn,
'nwt': nwt,
'pks': pks,
'sms': sms,
'sfr': sfr,
'lak': lak,
'gage': gage,
'sip': sip,
'sor': sor,
'de4': de4,
'oc': oc,
'uzf': uzf,
'upw': upw,
'sub': sub,
'swt': swt,
'hyd': hyd,
'hob': hob,
'vdf': vdf,
'vsc': vsc,
'drt': drt,
'pvl': pvl,
'ets': ets,
'bas': bas,
'nam': nam
}
hs = HydroShare()
Session = app.get_persistent_store_database('primary_db',
as_sessionmaker=True)
session = Session()
fileliststr = session.query(Model).filter(
Model.displayname == modelname).first()
filelist = [i for i in fileliststr.modelfiles.strip('{}').split(',')]
mainid = fileliststr.id
resourceid = fileliststr.resourceid
if uploadtype == 'new':
abstract = resource_abstract
title = resource_name
keywords = (i for i in resource_key.split(','))
rtype = 'ModelInstanceResource'
new_resource_id = hs.createResource(rtype,
title,
keywords=keywords,
abstract=abstract)
for fi in filelist:
parts = fi.split(".")
ext_data = session.query(
dbs[parts[1]]).filter(dbs[parts[1]].id == mainid).first().data
if uploadtype == 'add':
date = dt.now().strftime("%m-%d-%Y-%X")
filename = "{}_{}.{}".format(parts[0], date, parts[1])
else:
filename = fi
if uploadtype == 'new':
hs.addResourceFile(new_resource_id,
ext_data,
resource_filename=filename)
elif uploadtype == 'overwrite':
hs.deleteResourceFile(resourceid, filename)
hs.addResourceFile(resourceid,
ext_data,
resource_filename=filename)
else:
hs.addResourceFile(resourceid,
ext_data,
resource_filename=filename)
session.close()
return_obj = {'success': True}
return JsonResponse(return_obj)
def post_hs(ctx, username, password, modelrun_dir, include_shear_nc,
resource_title, keyword):
"""Post the model run data to HydroShare"""
# iterate over files and folders of interest, adding files to resource
# RipCAS files are the vegetation .asc; TODO XXX include base RipCAS XXX TODO
export_dir = os.path.join(modelrun_dir, 'export')
veg_export_dir = os.path.join(export_dir, 'vegetation')
if os.path.isdir(export_dir):
shutil.rmtree(export_dir)
os.mkdir(export_dir)
os.mkdir(veg_export_dir)
veg_pattern = os.path.join(modelrun_dir, 'ripcas-*', 'vegetation.asc')
for tstep, veg_map in enumerate(glob.glob(veg_pattern)):
veg_map_path = os.path.join(
export_dir, 'vegetation', 'vegetation-%s.asc' % tstep
)
shutil.copy(veg_map, veg_map_path)
shutil.make_archive(veg_export_dir, 'zip', veg_export_dir)
# connect
hs = HydroShare(
auth=HydroShareAuthBasic(username=username, password=password)
)
# create new resource
rtype = 'GenericResource'
r_id = hs.createResource(
rtype, resource_title, keywords=keyword # , abstract=abstract
)
print "adding vegmap archive file %s to resource %s" % (veg_map_path, r_id)
hs.addResourceFile(r_id, os.path.join(export_dir, 'vegetation.zip'))
inputs_dir = os.path.join(modelrun_dir, 'inputs')
inputs_export_basename = os.path.join(export_dir, 'inputs')
shutil.make_archive(inputs_export_basename, 'zip', inputs_dir)
print "adding inputs archive file %s to resource %s" % (veg_map_path, r_id)
hs.addResourceFile(
r_id, os.path.join(export_dir, 'inputs.zip')
)
shear_export_dir = os.path.join(export_dir, 'shear')
os.mkdir(shear_export_dir)
shear_pattern = os.path.join(modelrun_dir, 'dflow-*', 'shear_out.asc')
for tstep, shear_map in enumerate(glob.glob(shear_pattern)):
shear_map_path = os.path.join(
export_dir, 'shear', 'shear-%s.asc' % tstep
)
shutil.copy(shear_map, shear_map_path)
shutil.make_archive(shear_export_dir, 'zip', shear_export_dir)
print "adding shear archive file %s to resource %s" % (veg_map_path, r_id)
hs.addResourceFile(
r_id, os.path.join(export_dir, 'shear.zip')
)
def create_hydroshare_resource(context,
auth,
title,
hydroshare_host='www.hydroshare.org',
hydroshare_port=None,
use_https=False,
resource_type='GenericResource',
abstract=None,
keywords=None,
create_callback=None,
verbose=False,
outfp=sys.stderr):
"""
Create HydroShare resource of an EcohydroLib project
@param context ecohydrolib.Context object
@param auth hs_restclient.HydroShareAuth object
@param title string representing the title of the resource
@param hydroshare_host string representing DNS name of the HydroShare
server in which to create the resource
@param hydroshare_port int representing the TCP port of the HydroShare
server
@param use_https True if HTTPS should be used. Default: False
@param resource_type string representing the HydroShare resource type
that should be used to create the resource
@param abstract string representing the abstract of the resource
@param keywords list of strings representing the keywords to assign
to the resource
@param create_callback user-defined callable that takes as input a
file size in bytes, and generates a callable to provide feedback
to the user about the progress of the upload of resource_file.
For more information, see:
http://toolbelt.readthedocs.org/en/latest/uploading-data.html#monitoring-your-streaming-multipart-upload
@param verbose Boolean True if detailed output information should be printed to outfp
@param outfp File-like object to which verbose output should be printed
@return string representing the ID of the newly created resource
"""
temp_dir = tempfile.mkdtemp()
zip_filename = "{0}.zip".format(os.path.basename(context.projectDir))
zip_filepath = os.path.join(temp_dir, zip_filename)
# Zip up the project for upload ...
if verbose:
outfp.write("Creating archive...")
outfp.flush()
zfile = zipfile.ZipFile(zip_filepath, 'w', zipfile.ZIP_DEFLATED, True)
os.path.walk(context.projectDir, _addToZip, (context.projectDir, zfile))
zfile.close()
if verbose:
outfp.write("done\n")
outfp.flush()
# Make upload progress callback
progress_callback = None
if create_callback:
s = os.stat(zip_filepath)
progress_callback = create_callback(s.st_size)
if verbose:
outfp.write("Uploading to HydroShare...")
outfp.flush()
# This next line is needed for some text-based progress indicators
# to properly render.
outfp.write(" \nThis is a hack\r")
outfp.flush()
if hydroshare_host:
hs = HydroShare(hostname=hydroshare_host,
port=hydroshare_port,
use_https=use_https,
auth=auth)
else:
hs = HydroShare(port=hydroshare_port, use_https=use_https, auth=auth)
resource_id = None
try:
resource_id = hs.createResource(resource_type,
title,
resource_file=zip_filepath,
resource_filename=zip_filename,
abstract=abstract,
keywords=keywords,
progress_callback=progress_callback)
if verbose:
url = _getResourceURL(resource_id,
hydroshare_host,
hs_port=hydroshare_port,
use_https=use_https)
outfp.write("\nNew resource created at:\n{0}\n".format(url))
outfp.flush()
except Exception as e:
raise e
finally:
shutil.rmtree(temp_dir)
return resource_id
