def test_create_get_delete_resource(self): hs = HydroShare() abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile( os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'r') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def test_create_get_delete_resource(self): hs = HydroShare(prompt_auth=False) abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' metadata = json.dumps([{'coverage': {'type': 'period', 'value': {'start': '01/01/2000', 'end': '12/12/2010'}}}, {'creator': {'name': 'John Smith'}}, {'contributor': {'name': 'Lisa Miller'}}]) extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'}) with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'rb') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def start_file_download(request): global temp_dir, prediction_data, rp_data, total_prediction_comids, total_rp_comids, sorted_prediction_comids, \ sorted_rp_comids, time if request.method == 'GET': get_data = request.GET this_script_path = inspect.getfile(inspect.currentframe()) try: if get_data['res_id'] is not None: temp_dir = tempfile.mkdtemp() file_path = get_data['res_id'] if get_data['src'] == 'iRODS': download = requests.get(file_path, stream=True) filename = os.path.basename(file_path) local_file_path = os.path.join(temp_dir, filename) with open(local_file_path, 'wb') as fd: for chunk in download.iter_content(1024): fd.write(chunk) prediction_data = nc.Dataset(local_file_path, mode="r") elif get_data['src'] == 'hs': auth = HydroShareAuthBasic(username='******', password='******') hs = HydroShare(auth=auth, hostname="playground.hydroshare.org", use_https=False) resource_data = hs.getSystemMetadata(file_path) filename = resource_data['resource_title'] # this will only work if there is only one file in the resource and if # the resource title is the same as filename hs.getResourceFile(file_path, filename, destination=temp_dir) local_file_path = temp_dir + "/" + filename prediction_data = nc.Dataset(local_file_path, mode="r") else: testfile_path = this_script_path.replace('controllers.py', 'public/data/test.nc') prediction_data = nc.Dataset(testfile_path, mode="r") # Sort the RAPID netCDF file by COMID qout_dimensions = prediction_data.variables['Qout'].dimensions if qout_dimensions[0].lower() == 'comid' and qout_dimensions[1].lower() == 'time': sorted_prediction_comids = sorted(enumerate(prediction_data.variables['COMID'][:]), key=lambda comid: comid[1]) total_prediction_comids = len(sorted_prediction_comids) else: return JsonResponse({'error': "Invalid netCDF file"}) variables = prediction_data.variables.keys() if 'time' in variables: time = [t * 1000 for t in prediction_data.variables['time'][:]] else: return JsonResponse({'error': "Invalid netCDF file"}) rp_data_path = this_script_path.replace('controllers.py', 'public/data/return_period_data.nc') rp_data = nc.Dataset(rp_data_path, mode="r") sorted_rp_comids = sorted(enumerate(rp_data.variables['COMID'][:]), key=lambda comid: comid[1]) total_rp_comids = len(sorted_rp_comids) return JsonResponse({'success': "The file is ready to go."}) except Exception, err: return JsonResponse({'error': err})
def test_create_get_delete_resource(self): hs = HydroShare() abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'r') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def loadresource(request): res_id = request.GET['res_id'] hs = HydroShare() resource_md = hs.getSystemMetadata(res_id) resource_files = hs.getResourceFileList(res_id).list[0] context = { 'path': request.path, 'method': request.method, 'query': request.META['QUERY_STRING'], 'res_id': res_id, 'resource_title': resource_md["resource_title"], 'resource_files': resource_files, } return render(request, 'colab_launcher/loadresource.html', context)
def test_get_resource(self): hs = HydroShare(hostname=self.url) # ideally, look a system metadata, and if file is small, then load it into memory. sysmeta = hs.getSystemMetadata(self.a_resource_id) self.assertEqual(sysmeta['resource_id'], self.a_resource_id) self.assertEqual(sysmeta['resource_type'], self.a_resource_type) self.assertTrue(sysmeta['public']) # if (sysmeta['size']) < 1000000: stream = hs.getResource(self.a_resource_id) in_memory = StringIO.StringIO() for chunk in stream: in_memory.write(chunk) self.assertTrue(zipfile.is_zipfile(in_memory)) with zipfile.ZipFile(in_memory, 'r') as myzip: filelist = myzip.infolist() success = False for f in filelist: if self.a_resource_filename in f.filename: success = True break self.assertTrue(success, "did not find file")
class hydroshare(): def __init__(self, username=None, password=None, cache=False): self.hs = None self.content = {} # connect to hydroshare using OAUTH2 authfile = os.path.expanduser("~/.hs_auth") if os.path.exists(authfile): with open(authfile, 'rb') as f: token, cid = pickle.load(f) auth = HydroShareAuthOAuth2(cid, '', token=token) else: # connect to hydroshare using Basic Authentication self.cache = cache notebook_home = os.environ.get('NOTEBOOK_HOME', '.') if cache: utilities.load_environment(os.path.join(notebook_home, '.env')) self.auth_path = os.path.join(notebook_home, '.auth') uname = username if uname is None: uname = os.environ.get('HS_USR_NAME', None) if password is None: # get a secure connection to hydroshare auth = self.getSecureConnection(uname) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=uname, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path) return None # set the HS resource download directory download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'hs_downloads') if not os.path.isdir(download_dir): print('Creating a directory to store your HS downloads') os.makedirs(download_dir) self.download_dir = download_dir def _addContentToExistingResource(self, resid, content_files): for f in content_files: self.hs.addResourceFile(resid, f) def getSecureConnection(self, username=None): """Establishes a secure connection with hydroshare. args: -- email: email address associated with hydroshare returns: -- hydroshare api connection """ if not os.path.exists(self.auth_path): print('\nThe hs_utils library requires a secure connection to ' 'your HydroShare account.') if username is None: username = input('Please enter your HydroShare username: '******'Enter the HydroShare password for user ' '\'%s\': ' % username) auth = HydroShareAuthBasic(username=username, password=p) save = input('Do you want to save this password for future use?\n' 'Note: the password will be stored in plain text ' '(not recommended) [y/N]?') if save.lower() == 'y': self.cache = True else: self.cache = False if self.cache: with open(self.auth_path, 'wb') as f: pickle.dump(auth, f, protocol=2) else: with open(self.auth_path, 'rb') as f: auth = pickle.load(f) return auth def getResourceMetadata(self, resid): """Gets metadata for a specified resource. args: -- resid: hydroshare resource id returns: -- resource metadata object """ science_meta = self.hs.getScienceMetadata(resid) system_meta = self.hs.getSystemMetadata(resid) return resource.ResourceMetadata(system_meta, science_meta) def createHydroShareResource(self, abstract, title, derivedFromId=None, keywords=[], resource_type='GenericResource', content_files=[], public=False): """Creates a hydroshare resource. args: -- abstract: abstract for resource (str, required) -- title: title of resource (str, required) -- derivedFromId: id of parent hydroshare resource (str, default=>None) -- keywords: list of subject keywords (list, default=>[]) -- resource_type: type of resource to create (str, default=> 'GenericResource') -- content_files: data to save as resource content (list, default=>[]) -- public: resource sharing status (bool, default=>False) returns: -- None """ # query the hydroshare resource types and make sure that # resource_type is valid restypes = {r.lower(): r for r in self.hs.getResourceTypes()} try: res_type = restypes[resource_type] except KeyError: display( HTML('<b style="color:red;">[%s] is not a valid ' 'HydroShare resource type.</p>' % resource_type)) return None # get the 'derived resource' metadata if derivedFromId is not None: try: # update the abstract and keyword metadata meta = self.getResourceMetadata(derivedFromId) abstract = meta.abstract \ + '\n\n[Modified in JupyterHub on %s]\n%s' \ % (dt.now(), abstract) keywords = set(keywords + meta.keywords) except: display( HTML('<b style="color:red;">Encountered an error ' ' while setting the derivedFrom relationship ' ' using id=%s. Make sure this resource is ' ' is accessible to your account. ' % derivedFromId)) display( HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % derivedFromId, 'View the "DerivedFrom" ' 'Resource'))) return None f = None if len(content_files) == 0 else content_files[0] # create the hs resource (1 content file allowed) resid = threads.runThreadedFunction('Creating HydroShare Resource', 'Resource Created Successfully', self.hs.createResource, resource_type=res_type, title=title, abstract=abstract, resource_file=f, keywords=keywords) display(HTML('Resource id: %s' % resid)) display( HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare'))) # add the remaining content files to the hs resource try: if len(content_files) > 1: self.addContentToExistingResource(resid, content_files[1:]) except Exception as e: print(e) def getResourceFromHydroShare(self, resourceid, destination='.'): """Downloads content of a hydroshare resource. args: -- resourceid: id of the hydroshare resource (str) -- destination: path to save resource, default /user/[username]/notebooks/data (str) returns: -- None """ default_dl_path = self.download_dir dst = os.path.abspath(os.path.join(default_dl_path, destination)) download = True # check if the data should be overwritten dst_res_folder = os.path.join(dst, resourceid) if os.path.exists(dst_res_folder): print('This resource already exists in your userspace.') utilities.tree(dst_res_folder) res = input('\nDo you want to overwrite these data [Y/n]? ') if res != 'n': shutil.rmtree(dst_res_folder) else: download = False # re-download the content if desired if download: try: # download the resource (threaded) threads.runThreadedFunction('Downloading Resource', 'Download Finished', self.hs.getResource, resourceid, destination=dst, unzip=True) print('Successfully downloaded resource %s' % resourceid) except Exception as e: display( HTML('<b style="color:red">Failed to retrieve ' 'resource content from HydroShare: %s</b>' % e)) return None # load the resource content outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid)) content_files = glob.glob(os.path.join(outdir, 'data/contents/*')) content = {} for f in content_files: fname = os.path.basename(f) # trim the base name relative to the data directory dest_folder_name = os.path.dirname(destination).split('/')[-1] f = os.path.join(dest_folder_name, os.path.relpath(f, dest_folder_name)) content[fname] = f # show the resource content files utilities.display_resource_content_files(content) # update the content dictionary self.content.update(content) def addContentToExistingResource(self, resid, content): """Adds content files to an existing hydroshare resource. args: -- resid: id of an existing hydroshare resource (str) -- content: files paths to be added to resource (list) returns: -- None """ threads.runThreadedFunction('Adding Content to Resource', 'Successfully Added Content Files', self._addContentToExistingResource, resid, content) def loadResource(self, resourceid): """Loads the contents of a previously downloaded resource. args: -- resourceid: the id of the resource that has been downloaded (str) returns: -- {content file name: path} """ resdir = utilities.find_resource_directory(resourceid) if resdir is None: display( HTML('<b style="color:red">Could not find any resource ' 'matching the id [%s].</b> <br> It is likely that ' 'this resource has not yet been downloaded from ' 'HydroShare.org, or it was removed from the ' 'JupyterHub server. Please use the following ' 'command to aquire the resource content: ' '<br><br><code>hs.getResourceFromHydroShare(%s)' '</code>.' % (resourceid, resourceid))) return # create search paths. Need to check 2 paths due to hs_restclient bug #63. search_paths = [ os.path.join(resdir, '%s/data/contents/*' % resourceid), os.path.join(resdir, 'data/contents/*') ] content = {} found_content = False for p in search_paths: content_files = glob.glob(p) if len(content_files) > 0: found_content = True display( HTML('<p>Downloaded content is located at: %s</p>' % resdir)) display( HTML('<p>Found %d content file(s). \n</p>' % len(content_files))) for f in content_files: fname = os.path.basename(f) content[fname] = f if len(content.keys()) == 0: display( HTML( '<p>Did not find any content files for resource id: %s</p>' % resourceid)) utilities.display_resource_content_files(content) self.content = content def getContentFiles(self, resourceid): """Gets the content files for a resource that exists on the Jupyter Server args: -- resourceid: the id of the hydroshare resource returns: -- {content file name: path} """ content = utilities.get_hs_content(resourceid) return content def getContentPath(self, resourceid): """Gets the server path of a resources content files. args: -- resourceid: the id of the hydroshare resource returns: -- server path the the resource content files """ path = utilities.find_resource_directory(resourceid) if path is not None: return os.path.join(path, resourceid, 'data/contents')
class hydroshare(): def __init__(self, username=None, password=None, cache=True): self.hs = None self.content = {} # connect to hydroshare using OAUTH2 authfile = os.path.expanduser("~/.hs_auth") if os.path.exists(authfile): with open(authfile, 'rb') as f: token, cid = pickle.load(f) auth = HydroShareAuthOAuth2(cid, '', token=token) else: # connect to hydroshare using Basic Authentication self.cache = cache if cache: utilities.load_environment(os.path.join( os.environ['NOTEBOOK_HOME'], '.env')) self.auth_path = os.environ.get('NOTEBOOK_HOME', '/home/jovyan/.auth') uname = username if uname is None: if 'HS_USR_NAME' in os.environ.keys(): uname = os.environ['HS_USR_NAME'] if password is None: # get a secure connection to hydroshare auth = self.getSecureConnection(uname) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=uname, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path) return None # set the HS resource download directory download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'Downloads') if not os.path.isdir(download_dir): os.makedirs(download_dir) self.download_dir = download_dir def _addContentToExistingResource(self, resid, content_files): for f in content_files: self.hs.addResourceFile(resid, f) def getSecureConnection(self, username=None): """Establishes a secure connection with hydroshare. args: -- email: email address associated with hydroshare returns: -- hydroshare api connection """ if not os.path.exists(self.auth_path): print('\nThe hs_utils library requires a secure connection to ' 'your HydroShare account.') if username is None: username = input('Please enter your HydroShare username: '******'Enter the HydroShare password for user ' '\'%s\': ' % username) auth = HydroShareAuthBasic(username=username, password=p) if self.cache: with open(self.auth_path, 'wb') as f: pickle.dump(auth, f, protocol=2) else: with open(self.auth_path, 'rb') as f: auth = pickle.load(f) return auth def getResourceMetadata(self, resid): """Gets metadata for a specified resource. args: -- resid: hydroshare resource id returns: -- resource metadata object """ science_meta = self.hs.getScienceMetadata(resid) system_meta = self.hs.getSystemMetadata(resid) return resource.ResourceMetadata(system_meta, science_meta) def createHydroShareResource(self, abstract, title, derivedFromId=None, keywords=[], resource_type='GenericResource', content_files=[], public=False): """Creates a hydroshare resource. args: -- abstract: abstract for resource (str, required) -- title: title of resource (str, required) -- derivedFromId: id of parent hydroshare resource (str, default=>None) -- keywords: list of subject keywords (list, default=>[]) -- resource_type: type of resource to create (str, default=> 'GenericResource') -- content_files: data to save as resource content (list, default=>[]) -- public: resource sharing status (bool, default=>False) returns: -- None """ # query the hydroshare resource types and make sure that # resource_type is valid restypes = {r.lower(): r for r in self.hs.getResourceTypes()} try: res_type = restypes[resource_type] except KeyError: display(HTML('<b style="color:red;">[%s] is not a valid ' 'HydroShare resource type.</p>' % resource_type)) return None # get the 'derived resource' metadata if derivedFromId is not None: try: # update the abstract and keyword metadata meta = self.getResourceMetadata(derivedFromId) abstract = meta.abstract \ + '\n\n[Modified in JupyterHub on %s]\n%s' \ % (dt.now(), abstract) keywords = set(keywords + meta.keywords) except: display(HTML('<b style="color:red;">Encountered an error ' ' while setting the derivedFrom relationship ' ' using id=%s. Make sure this resource is ' ' is accessible to your account. ' % derivedFromId)) display(HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % derivedFromId, 'View the "DerivedFrom" ' 'Resource'))) return None f = None if len(content_files) == 0 else content_files[0] # create the hs resource (1 content file allowed) resid = threads.runThreadedFunction('Creating HydroShare Resource', 'Resource Created Successfully', self.hs.createResource, resource_type=res_type, title=title, abstract=abstract, resource_file=f, keywords=keywords) # add the remaining content files to the hs resource try: if len(content_files) > 1: self.addContentToExistingResource(resid, content_files[1:]) except Exception as e: print(e) display(HTML('Resource id: %s' % resid)) display(HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare'))) def getResourceFromHydroShare(self, resourceid, destination='.'): """Downloads content of a hydroshare resource. args: -- resourceid: id of the hydroshare resource (str) -- destination: path to save resource, default /user/[username]/notebooks/data (str) returns: -- None """ default_dl_path = self.download_dir dst = os.path.abspath(os.path.join(default_dl_path, destination)) download = True # check if the data should be overwritten dst_res_folder = os.path.join(dst, resourceid) if os.path.exists(dst_res_folder): print('This resource already exists in your userspace.') utils.tree(dst_res_folder) res = input('\nDo you want to overwrite these data [Y/n]? ') if res != 'n': shutil.rmtree(dst_res_folder) else: download = False # re-download the content if desired if download: try: # download the resource (threaded) threads.runThreadedFunction('Downloading Resource', 'Download Finished', self.hs.getResource, resourceid, destination=dst, unzip=True) print('Successfully downloaded resource %s' % resourceid) except Exception as e: display(HTML('<b style="color:red">Failed to retrieve ' 'resource content from HydroShare: %s</b>' % e)) return None # load the resource content outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid)) content_files = glob.glob(os.path.join(outdir, 'data/contents/*')) content = {} for f in content_files: fname = os.path.basename(f) # trim the base name relative to the data directory dest_folder_name = os.path.dirname(destination).split('/')[-1] f = os.path.join(dest_folder_name, os.path.relpath(f, dest_folder_name)) content[fname] = f # show the resource content files utilities.display_resource_content_files(content) # update the content dictionary self.content.update(content) def addContentToExistingResource(self, resid, content): """Adds content files to an existing hydroshare resource. args: -- resid: id of an existing hydroshare resource (str) -- content: files paths to be added to resource (list) returns: -- None """ threads.runThreadedFunction('Adding Content to Resource', 'Successfully Added Content Files', self._addContentToExistingResource, resid, content) def loadResource(self, resourceid): """Loads the contents of a previously downloaded resource. args: -- resourceid: the id of the resource that has been downloaded (str) returns: -- {content file name: path} """ resdir = utilities.find_resource_directory(resourceid) if resdir is None: display(HTML('<b style="color:red">Could not find any resource ' 'matching the id [%s].</b> <br> It is likely that ' 'this resource has not yet been downloaded from ' 'HydroShare.org, or it was removed from the ' 'JupyterHub server. Please use the following ' 'command to aquire the resource content: ' '<br><br><code>hs.getResourceFromHydroShare(%s)' '</code>.' % (resourceid, resourceid))) return # create search paths. Need to check 2 paths due to hs_restclient bug #63. search_paths = [os.path.join(resdir, '%s/data/contents/*' % resourceid), os.path.join(resdir, 'data/contents/*')] content = {} found_content = False for p in search_paths: content_files = glob.glob(p) if len(content_files) > 0: found_content = True display(HTML('<p>Downloaded content is located at: %s</p>' % resdir)) display(HTML('<p>Found %d content file(s). \n</p>' % len(content_files))) for f in content_files: fname = os.path.basename(f) content[fname] = f if len(content.keys()) == 0: display(HTML('<p>Did not find any content files for resource id: %s</p>' % resourceid)) utilities.display_resource_content_files(content) self.content = content def getContentFiles(self, resourceid): """Gets the content files for a resource that exists on the Jupyter Server args: -- resourceid: the id of the hydroshare resource returns: -- {content file name: path} """ content = utilities.get_hs_content(resourceid) return content def getContentPath(self, resourceid): """Gets the server path of a resources content files. args: -- resourceid: the id of the hydroshare resource returns: -- server path the the resource content files """ path = utilities.find_resource_directory(resourceid) if path is not None: return os.path.join(path, resourceid, 'data/contents')
class hydroshare(): def __init__(self, username=None): self.hs = None self.content = {} # load the HS environment variables # self.load_environment() uname = username if uname is None: uname = os.environ['HS_USR_NAME'] # get a secure connection to hydroshare auth = self.getSecureConnection(uname) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() display(HTML('<b style="color:green;">Successfully established a connection with HydroShare</b>')) except HydroShareHTTPException as e: display(HTML( '<p style="color:red;"><b>Failed to establish a connection with HydroShare. Please check that you provided the correct credentials</b><br>%s </p>' % e)) # remove the cached authentication auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth') os.remove(auth_path) return None def _getResourceFromHydroShare(self, resourceid, destination='.', unzip=True): # download the resource pid = self.hs.getResource(resourceid, destination=destination, unzip=unzip) threadResults.put(pid) def _createHydroShareResource(self, res_type, title, abstract, content_file, keywords=[]): resid = self.hs.createResource(res_type, title, resource_file=content_file, keywords=keywords, abstract=abstract) threadResults.put(resid) def _addContentToExistingResource(self, resid, content_files): for f in content_files: self.hs.addResourceFile(resid, f) def load_environment(self): env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'env') with open(env_path, 'r') as f: lines = f.readlines() print('Adding the following system variables:') for line in lines: k, v = line.strip().split('=') os.environ[k] = v print(' %s = %s' % (k, v)) print('\nThese can be accessed using the following command: ') print(' os.environ[key]') print('\n (e.g.)\n os.environ["HS_USR_NAME"] => %s' % os.environ['HS_USR_NAME']) def getSecureConnection(self, username): """ Establishes a secure connection with HydroShare. Args: email: email address associated with HydroShare Returns: HydroShare connection """ auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth') if not os.path.exists(auth_path): print('\nThe hs_utils library requires a secure connection to your HydroShare account.') # p = getpass.getpass('Enter the HydroShare password for user \'%s\': ' % username) p = '7jmftUpata' auth = HydroShareAuthBasic(username=username, password=p) with open(auth_path, 'wb') as f: pickle.dump(auth, f, protocol=2) else: with open(auth_path, 'rb') as f: auth = pickle.load(f) return auth def getResourceMetadata(self, resid): science_meta = self.hs.getScienceMetadata(resid) system_meta = self.hs.getSystemMetadata(resid) return ResourceMetadata(system_meta, science_meta) def createHydroShareResource(self, abstract, title, derivedFromId, keywords=[], resource_type='GenericResource', content_files=[], public=False): # query the hydroshare resource types and make sure that resource_type is valid restypes = {r.lower(): r for r in self.hs.getResourceTypes()} try: res_type = restypes[resource_type] except KeyError: display(HTML('<b style="color:red;">[%s] is not a valid HydroShare resource type.</p>' % resource_type)) return None # get the 'derived resource' metadata if derivedFromId is not None: try: # update the abstract and keyword metadata meta = self.getResourceMetadata(derivedFromId) abstract = meta.abstract + '\n\n[Modified in JupyterHub on %s]\n%s' % (dt.now(), abstract) keywords = set(keywords + meta.keywords) except: display(HTML( '<b style="color:red;">[%s] is not a valid HydroShare resource id for setting the "derivedFrom" attribute.</p>' % derivedFromId)) return None else: response = input( 'You have indicated that this resource is NOT derived from any existing HydroShare resource. Are you sure that this is what you intended? [Y/n]') if response == 'n': display(HTML('<b style="color:red;">Resource creation aborted.</p>')) return f = None if len(content_files) == 0 else content_files[0] # create the hs resource (1 content file allowed) t = threading.Thread(target=self._createHydroShareResource, args=(res_type, title, abstract, f), kwargs={'keywords': keywords}) resid = runThreadedFunction(t, msg='Creating HydroShare Resource', success='Resource Creation Successful') # add the remaining content files to the hs resource self.addContentToExistingResource(resid, content_files[1:]) display(HTML('Resource id: %s' % resid)) display(HTML('<a href=%s target="_blank">%s<a>' % ( 'https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare'))) def getResourceFromHydroShare(self, resourceid, destination='.'): """ Downloads the content of HydroShare resource to the JupyterHub userspace Args: resourceid: id of the HydroShare resource to query destination: path relative to /user/[username]/notebooks/data """ default_dl_path = 'C:\\Users\\12672\\Box\\data\\NEON\\lczodata\\' # os.environ['DATA'] dst = os.path.abspath(os.path.join(default_dl_path, destination)) download = True # check if the data should be overwritten dst_res_folder = os.path.join(dst, resourceid) if os.path.exists(dst_res_folder): res = input( 'This resource already exists in your userspace.\nWould you like to overwrite this data [Y/n]? ') if res != 'n': shutil.rmtree(dst_res_folder) else: download = False # re-download the content if desired if download: try: # get some metadata about the resource that will be downloaded res_meta = self.hs.getSystemMetadata(resourceid) header = requests.head(res_meta['bag_url']) # download the resource (threaded) t = threading.Thread(target=self._getResourceFromHydroShare, args=(resourceid,), kwargs={'destination': dst, 'unzip': True}) runThreadedFunction(t, msg='Downloading', success='Download Completed Successfully') except Exception as e: display(HTML('<b style="color:red">Failed to retrieve resource content from HydroShare: %s</b>' % e)) return None # load the resource content outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid)) content_files = glob.glob(os.path.join(outdir, 'data/contents/*')) # display(HTML('Your Content is located at: %s' % outdir)) content = {} for f in content_files: fname = os.path.basename(f) content[fname] = f display_resource_content_files(content) # check_for_ipynb(content_files) # update the content dictionary self.content.update(content) def addContentToExistingResource(self, resid, content): t = threading.Thread(target=self._addContentToExistingResource, args=(resid, content)) runThreadedFunction(t, msg='Adding Content to Resource', success='Successfully Added Content Files') def loadResource(self, resourceid): resdir = find_resource_directory(resourceid) if resdir is None: display(HTML( '<b style="color:red">Could not find any resource matching the id [%s].</b> <br> It is likely that this resource has not yet been downloaded from HydroShare.org, or it was removed from the JupyterHub server. Please use the following command to aquire the resource content: <br><br> <code> hs.getResourceFromHydroShare(%s)</code>.' % ( resourceid, resourceid))) return content_files = glob.glob(os.path.join(resdir, '%s/data/contents/*' % resourceid)) display(HTML('<p>Downloaded content is located at: %s</p>' % resdir)) display(HTML('<p>Found %d content file(s). \n</p>' % len(content_files))) content = {} for f in content_files: fname = os.path.basename(f) content[fname] = f display_resource_content_files(content) self.content = content
def test_create_get_delete_resource(self): hs = HydroShare(prompt_auth=False) abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' metadata = json.dumps([{ 'coverage': { 'type': 'period', 'value': { 'start': '01/01/2000', 'end': '12/12/2010' } } }, { 'creator': { 'name': 'John Smith' } }, { 'contributor': { 'name': 'Lisa Miller' } }]) extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'}) with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile( os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'rb') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete hs.deleteResource(newres)
from hs_restclient import HydroShare, HydroShareAuthBasic auth = HydroShareAuthBasic(username="******", password = "******") hs = HydroShare(auth = auth) resource_md = hs.getSystemMetadata('ba3bb556d384478cb11eb7557fce2b8a') print(" ") print("Creator : " + resource_md['creator']) print("Resource ID : " + resource_md['resource_id']) print("Date created : " + resource_md['date_created']) print("Resource Type : " + resource_md['resource_type']) print("Resource Title : " + resource_md['resource_title']) print(" ")