def runSCI(id): if request.method == 'POST': auth = HydroShareAuthBasic(username=request.form['username'], password=request.form['password']) hs = HydroShare(auth=auth) hs.getResource('e987ddcf73a94376a4a70e5db0fb7646', destination='/home/ubuntu/hydroshareLink/', unzip=True) subprocess.Popen( 'sciunit open /home/ubuntu/hydroshareLink/e987ddcf73a94376a4a70e5db0fb7646/e987ddcf73a94376a4a70e5db0fb7646/data/contents/modflow.zip', stdout=subprocess.PIPE, shell=True) os.chdir("/home/ubuntu/test/") hs.getResource(id, destination='/home/ubuntu/test/', unzip=True) proc = subprocess.Popen('sciunit repeat e2 ' + str(id), stdout=subprocess.PIPE, shell=True) output = proc.stdout.read() abstract = '' title = 'MODFLOW_NWT_SCIUNIT_OUTPUT' keywords = ('my keyword 1', 'my keyword 2') rtype = 'MODFLOWModelInstanceResource' output_id = hs.createResource(rtype, title, abstract=abstract, keywords=keywords) for file in os.listdir("/home/ubuntu/test/MODFLOW/"): if file != "mfnwt": hs.addResourceFile(output_id, "/home/ubuntu/test/MODFLOW/" + file) title = 'ModflowNwtCollection' keywords = ('MODFLOW-NWT Input data', 'MODFLOW-NWT Output data', 'MODFLOW-NWT') rtype = 'CollectionResource' resource_id = hs.createResource(rtype, title, abstract=abstract, keywords=keywords) metaData = {'relations': []} newObject = {} newObject['type'] = 'hasPart' newObject['value'] = 'http://www.hydroshare.org/resource/' + str( id) + '/' metaData['relations'].append(newObject) newObject = {} newObject['type'] = 'hasPart' newObject['value'] = 'http://www.hydroshare.org/resource/' + str( output_id) + '/' metaData['relations'].append(newObject) hs.updateScienceMetadata(resource_id, metadata=metaData) return output return render_template('login.html', id=str(id))
def download_executable_lubuntu_hs(save_filepath): resource_id = 'a5dbd5b198c9468387f59f3fefc11e22' hs = HydroShare() hs.getResource(resource_id, destination=save_filepath, unzip=True) resource_name = 'summa-master.zip' executable_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name shutil.unpack_archive(executable_filepath, extract_dir=os.path.dirname(executable_filepath)) executable = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/summa-master/bin' return os.path.dirname(executable)
def install_test_cases_hs(save_filepath): resource_id = 'a0105d479c334764ba84633c5b9c1c01' hs = HydroShare() hs.getResource(resource_id, destination=save_filepath, unzip=True) resource_name = 'summatestcases-2.x.tar.gz' testcase_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name shutil.unpack_archive(testcase_filepath, extract_dir=os.path.dirname(testcase_filepath)) cmd = "cd {}/summaTestCases_2.x/; ./installTestCases_local.sh".format( os.path.dirname(testcase_filepath)) subprocess.run(cmd, shell=True) return os.path.dirname(testcase_filepath)
def get_hs_resource(resource_id, file_path): hs = HydroShare() hs.getResource(resource_id, destination=file_path, unzip=True) # unpack the simulation archive and remove unncessary files hs_resource_dir = os.path.join(file_path, resource_id, resource_id, 'data/contents/') hs_resource = os.listdir(hs_resource_dir) shutil.unpack_archive(hs_resource_dir + hs_resource[0], extract_dir=file_path) cmd = "rm -rf " + os.path.join(file_path, resource_id) subprocess.run(cmd, shell=True)
def get_hs_resource(resource_id, file_path): hs = HydroShare() hs.getResource(resource_id, destination=file_path, unzip=True) # unpack the simulation archive and remove unncessary files hs_resource_dir = os.path.join(file_path, resource_id, resource_id, 'data/contents/') hs_resource = os.listdir(hs_resource_dir) shutil.unpack_archive(hs_resource_dir + hs_resource[0], extract_dir=file_path) shutil.rmtree(os.path.join(file_path, resource_id)) return hs_resource[0].split(".")[0]
def download_model_instance(resource_id): path = (os.getcwd() + '/' + resource_id + '/' + resource_id + '/data/contents/') hs = HydroShare() hs.getResource(resource_id, destination=os.getcwd(), unzip=True) # unpack the simulation archive and remove unncessary files Model_Instance_Name = os.listdir(path)[0] shutil.unpack_archive(path + Model_Instance_Name, extract_dir=os.getcwd()) cmd = "rm -rf " + resource_id subprocess.run(cmd, shell=True) return Model_Instance_Name.split('.')[0]
def test_create_get_delete_resource(self): hs = HydroShare() abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile( os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'r') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def test_create_get_delete_resource(self): hs = HydroShare(prompt_auth=False) abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' metadata = json.dumps([{'coverage': {'type': 'period', 'value': {'start': '01/01/2000', 'end': '12/12/2010'}}}, {'creator': {'name': 'John Smith'}}, {'contributor': {'name': 'Lisa Miller'}}]) extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'}) with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'rb') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def download_hs_res(resquest): # get the value of para my_res_id from the url res_id = resquest.GET['my_res_id'] # download the hydroshare res from hydroshare server to tethys server hs = HydroShare() hs.getResource(res_id, destination='/tmp') # return the res stored on tethys server to frontend/client response = HttpResponse(open("/tmp/{0}.zip".format(res_id), 'rb').read(), content_type='application/zip') #{0} is the place from the .format. #rb is readable binary response['Content-Disposition'] = "attachment; filename={0}.zip".format(res_id) #This tells you what type of files it is response['Content-Length'] = os.path.getsize('/tmp/{0}.zip'.format(res_id)) # So this line helps the browser know how large the file is and calculate time. return response
def download_model_instance_pangeo(resource_id): path = (os.getcwd() + '/' + resource_id + '/' + resource_id + '/data/contents/') hs = HydroShare() hs.getResource(resource_id, destination=os.getcwd(), unzip=True) # unpack the simulation archive and remove unncessary files Model_Instance_Name = os.listdir(path)[1] shutil.unpack_archive(path + Model_Instance_Name, extract_dir=os.getcwd()) cmd = "rm -rf " + resource_id subprocess.run(cmd, shell=True) cmd = ('cd ' + Model_Instance_Name.split('.')[0] + '/; chmod +x ./installTestCases_local.sh') subprocess.run(cmd, shell=True, stderr=subprocess.STDOUT) cmd = ('cd ' + Model_Instance_Name.split('.')[0] + '/; ./installTestCases_local.sh') subprocess.run(cmd, shell=True, stderr=subprocess.STDOUT) return Model_Instance_Name.split('.')[0]
def run_service(user_input): auth = HydroShareAuthBasic(hs_username, hs_password) hs = HydroShare(auth=auth) client = HydroDS(username=ci_username, password=ci_password) # step1: get raster resource files from HydroShare # TODO remember to check if the resource is downloadable resource_id = user_input['input_raster_url_path'] destination = tempfile.mkdtemp() hs.getResource(resource_id, destination, unzip=True) raster_file_folder = '{0}/{1}/{2}/data/contents'.format(destination, resource_id, resource_id) raster_file_path_list = [os.path.join(raster_file_folder, f) for f in os.listdir(raster_file_folder) if '.vrt' not in f] upload_file_url = client.upload_file(file_to_upload=raster_file_path_list[0]) # step2: processing files in HydroDS output_raster_name = 'process_{}'.format(upload_file_url.split('/')[-1]) output_file = client.project_resample_raster(input_raster_url_path=upload_file_url, cell_size_dx=int(user_input['cell_size_dx']), cell_size_dy=int(user_input['cell_size_dy']), output_raster=output_raster_name, resample=user_input['resample'], epsg_code=int(user_input['proj_code']) if user_input['proj_code_type']=='epsg_code' else None, utm_zone=int(user_input['proj_code']) if user_input['proj_code_type']=='utm_zone' else None, ) # step3: download file and create HydroShare resource save_as = '{}/{}'.format(destination, output_file['output_raster'].split('/')[-1]) client.download_file(file_url_path=output_file['output_raster'], save_as=save_as) title = user_input['output_raster'] rtype = 'RasterResource' resource_id = hs.createResource(rtype, title, resource_file=save_as) # step4: clean up the temp in Tethys and HydroDS shutil.rmtree(destination) client.delete_my_file(output_file['output_raster'].split('/')[-1]) client.delete_my_file(upload_file_url.split('/')[-1]) return {'is_success': True, 'message': 'http://www.hydroshare.org/resource/{}/'.format(resource_id)}
def test_create_get_delete_resource(self): hs = HydroShare() abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'r') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def runScript(id): auth = HydroShareAuthBasic(username='', password='') hs = HydroShare(auth=auth) hs.getResource(id, destination='/home/ubuntu/hydroshare_app/', unzip=True) subprocess.call("sudo cp " + str(id) + '/' + str(id) + '/data/contents/* /home/ubuntu/hydroshare_app/Data', shell=True) subprocess.call("sudo rm -r " + str(id), shell=True) process() #Locate the file with the .nam extension os.chdir('MODFLOW') for file in glob.glob("*.nam"): filename = file # Run the model subprocess.call("sudo ./mfnwt " + filename, shell=True) try: hs.deleteResourceFile(id, filename.split(".")[0] + '.list') except: pass #Upload to hydroshare hs.addResourceFile(id, filename.split(".")[0] + '.list') subprocess.call("sudo rm /home/ubuntu/hydroshare_app/MODFLOW/*.*", shell=True) subprocess.call("sudo rm -r /home/ubuntu/hydroshare_app/Scratch", shell=True) subprocess.call("sudo rm -r /home/ubuntu/hydroshare_app/Framework", shell=True) subprocess.call("sudo rm /home/ubuntu/hydroshare_app/Data/*", shell=True) return json.dumps(hs.getScienceMetadata(id))
def test_get_resource(self): hs = HydroShare(hostname=self.url) # ideally, look a system metadata, and if file is small, then load it into memory. sysmeta = hs.getSystemMetadata(self.a_resource_id) self.assertEqual(sysmeta['resource_id'], self.a_resource_id) self.assertEqual(sysmeta['resource_type'], self.a_resource_type) self.assertTrue(sysmeta['public']) # if (sysmeta['size']) < 1000000: stream = hs.getResource(self.a_resource_id) in_memory = StringIO.StringIO() for chunk in stream: in_memory.write(chunk) self.assertTrue(zipfile.is_zipfile(in_memory)) with zipfile.ZipFile(in_memory, 'r') as myzip: filelist = myzip.infolist() success = False for f in filelist: if self.a_resource_filename in f.filename: success = True break self.assertTrue(success, "did not find file")
class hydroshare(): def __init__(self, username=None): self.hs = None self.content = {} # load the HS environment variables # self.load_environment() uname = username if uname is None: uname = os.environ['HS_USR_NAME'] # get a secure connection to hydroshare auth = self.getSecureConnection(uname) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() display(HTML('<b style="color:green;">Successfully established a connection with HydroShare</b>')) except HydroShareHTTPException as e: display(HTML( '<p style="color:red;"><b>Failed to establish a connection with HydroShare. Please check that you provided the correct credentials</b><br>%s </p>' % e)) # remove the cached authentication auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth') os.remove(auth_path) return None def _getResourceFromHydroShare(self, resourceid, destination='.', unzip=True): # download the resource pid = self.hs.getResource(resourceid, destination=destination, unzip=unzip) threadResults.put(pid) def _createHydroShareResource(self, res_type, title, abstract, content_file, keywords=[]): resid = self.hs.createResource(res_type, title, resource_file=content_file, keywords=keywords, abstract=abstract) threadResults.put(resid) def _addContentToExistingResource(self, resid, content_files): for f in content_files: self.hs.addResourceFile(resid, f) def load_environment(self): env_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'env') with open(env_path, 'r') as f: lines = f.readlines() print('Adding the following system variables:') for line in lines: k, v = line.strip().split('=') os.environ[k] = v print(' %s = %s' % (k, v)) print('\nThese can be accessed using the following command: ') print(' os.environ[key]') print('\n (e.g.)\n os.environ["HS_USR_NAME"] => %s' % os.environ['HS_USR_NAME']) def getSecureConnection(self, username): """ Establishes a secure connection with HydroShare. Args: email: email address associated with HydroShare Returns: HydroShare connection """ auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth') if not os.path.exists(auth_path): print('\nThe hs_utils library requires a secure connection to your HydroShare account.') # p = getpass.getpass('Enter the HydroShare password for user \'%s\': ' % username) p = '7jmftUpata' auth = HydroShareAuthBasic(username=username, password=p) with open(auth_path, 'wb') as f: pickle.dump(auth, f, protocol=2) else: with open(auth_path, 'rb') as f: auth = pickle.load(f) return auth def getResourceMetadata(self, resid): science_meta = self.hs.getScienceMetadata(resid) system_meta = self.hs.getSystemMetadata(resid) return ResourceMetadata(system_meta, science_meta) def createHydroShareResource(self, abstract, title, derivedFromId, keywords=[], resource_type='GenericResource', content_files=[], public=False): # query the hydroshare resource types and make sure that resource_type is valid restypes = {r.lower(): r for r in self.hs.getResourceTypes()} try: res_type = restypes[resource_type] except KeyError: display(HTML('<b style="color:red;">[%s] is not a valid HydroShare resource type.</p>' % resource_type)) return None # get the 'derived resource' metadata if derivedFromId is not None: try: # update the abstract and keyword metadata meta = self.getResourceMetadata(derivedFromId) abstract = meta.abstract + '\n\n[Modified in JupyterHub on %s]\n%s' % (dt.now(), abstract) keywords = set(keywords + meta.keywords) except: display(HTML( '<b style="color:red;">[%s] is not a valid HydroShare resource id for setting the "derivedFrom" attribute.</p>' % derivedFromId)) return None else: response = input( 'You have indicated that this resource is NOT derived from any existing HydroShare resource. Are you sure that this is what you intended? [Y/n]') if response == 'n': display(HTML('<b style="color:red;">Resource creation aborted.</p>')) return f = None if len(content_files) == 0 else content_files[0] # create the hs resource (1 content file allowed) t = threading.Thread(target=self._createHydroShareResource, args=(res_type, title, abstract, f), kwargs={'keywords': keywords}) resid = runThreadedFunction(t, msg='Creating HydroShare Resource', success='Resource Creation Successful') # add the remaining content files to the hs resource self.addContentToExistingResource(resid, content_files[1:]) display(HTML('Resource id: %s' % resid)) display(HTML('<a href=%s target="_blank">%s<a>' % ( 'https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare'))) def getResourceFromHydroShare(self, resourceid, destination='.'): """ Downloads the content of HydroShare resource to the JupyterHub userspace Args: resourceid: id of the HydroShare resource to query destination: path relative to /user/[username]/notebooks/data """ default_dl_path = 'C:\\Users\\12672\\Box\\data\\NEON\\lczodata\\' # os.environ['DATA'] dst = os.path.abspath(os.path.join(default_dl_path, destination)) download = True # check if the data should be overwritten dst_res_folder = os.path.join(dst, resourceid) if os.path.exists(dst_res_folder): res = input( 'This resource already exists in your userspace.\nWould you like to overwrite this data [Y/n]? ') if res != 'n': shutil.rmtree(dst_res_folder) else: download = False # re-download the content if desired if download: try: # get some metadata about the resource that will be downloaded res_meta = self.hs.getSystemMetadata(resourceid) header = requests.head(res_meta['bag_url']) # download the resource (threaded) t = threading.Thread(target=self._getResourceFromHydroShare, args=(resourceid,), kwargs={'destination': dst, 'unzip': True}) runThreadedFunction(t, msg='Downloading', success='Download Completed Successfully') except Exception as e: display(HTML('<b style="color:red">Failed to retrieve resource content from HydroShare: %s</b>' % e)) return None # load the resource content outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid)) content_files = glob.glob(os.path.join(outdir, 'data/contents/*')) # display(HTML('Your Content is located at: %s' % outdir)) content = {} for f in content_files: fname = os.path.basename(f) content[fname] = f display_resource_content_files(content) # check_for_ipynb(content_files) # update the content dictionary self.content.update(content) def addContentToExistingResource(self, resid, content): t = threading.Thread(target=self._addContentToExistingResource, args=(resid, content)) runThreadedFunction(t, msg='Adding Content to Resource', success='Successfully Added Content Files') def loadResource(self, resourceid): resdir = find_resource_directory(resourceid) if resdir is None: display(HTML( '<b style="color:red">Could not find any resource matching the id [%s].</b> <br> It is likely that this resource has not yet been downloaded from HydroShare.org, or it was removed from the JupyterHub server. Please use the following command to aquire the resource content: <br><br> <code> hs.getResourceFromHydroShare(%s)</code>.' % ( resourceid, resourceid))) return content_files = glob.glob(os.path.join(resdir, '%s/data/contents/*' % resourceid)) display(HTML('<p>Downloaded content is located at: %s</p>' % resdir)) display(HTML('<p>Found %d content file(s). \n</p>' % len(content_files))) content = {} for f in content_files: fname = os.path.basename(f) content[fname] = f display_resource_content_files(content) self.content = content
def test_create_get_delete_resource(self): hs = HydroShare(prompt_auth=False) abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' metadata = json.dumps([{ 'coverage': { 'type': 'period', 'value': { 'start': '01/01/2000', 'end': '12/12/2010' } } }, { 'creator': { 'name': 'John Smith' } }, { 'contributor': { 'name': 'Lisa Miller' } }]) extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'}) with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile( os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open( '511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'rb') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete hs.deleteResource(newres)
class ResourceManager: """ Class that defines a handler for working with all of a user's resources. This is where they will be able to delete a resource, create a new one, or just get the list of a user's resources in hydroshare or jupyterhub. """ def __init__(self): """Makes an output folder for storing HS files locally, if none exists, and sets up authentication on hydroshare API. """ config = get_config_values(['dataPath', 'hydroShareHostname']) self.hs_hostname = 'www.hydroshare.org' # TODO: Rename to hydroshare_resource_data self.output_folder = Path('local_hs_resources') if config: if 'hydroShareHostname' in config: self.hs_hostname = config['hydroShareHostname'] if 'dataPath' in config: self.output_folder = Path(config['dataPath']) if not self.output_folder.is_dir(): # Let any exceptions that occur bubble up self.output_folder.mkdir(parents=True) self.hs_api_conn = None # Use 'authenticate' to initialize self.username = None def authenticate(self, username=None, password=None, save=False): """ Attempts to authenticate with HydroShare. :param username: the user's HydroShare username :type username: str :param password: the user's HydroShare password :type password: str :param save: whether or not to save the credentials to the config file if the authentication succeeds :type save: bool :return: the user's information (name, email, etc) from HydroShare if successful, None otherwise """ if not username or not password: # Check the config file for a username and password config = get_config_values(['u', 'p']) if not config or 'u' not in config or 'p' not in config: return None # No passed credentials and no saved credentials -- can't authenticate username = config['u'] password = base64.b64decode(config['p']).decode('utf-8') # Try to authenticate auth = HydroShareAuthBasic(username=username, password=password) self.hs_api_conn = HydroShare(auth=auth, hostname=self.hs_hostname) try: user_info = self.hs_api_conn.getUserInfo() self.username = user_info.get('username') except HydroShareHTTPException as e: if e.status_code == 401: # Unauthorized return None # Authentication failed raise e # Some other error -- bubble it up # Authentication succeeded if save: # Save the username and password saved_successfully = set_config_values({ 'u': username, 'p': str( base64.b64encode( password.encode('utf-8')).decode('utf-8')), }) if saved_successfully: logging.info( 'Successfully saved HydroShare credentials to config file.' ) return user_info # Authenticated successfully def is_authenticated(self): if self.hs_api_conn is None: self.authenticate() return self.hs_api_conn is not None def save_resource_locally(self, res_id): """ Downloads a resource to the local filesystem if a copy does not already exist locally :param res_id: the resource ID :type res_id: str """ # Get resource from HS if it doesn't already exist locally if not (self.output_folder / res_id).exists(): logging.info(f"Downloading resource {res_id} from HydroShare...") self.hs_api_conn.getResource(res_id, destination=self.output_folder, unzip=True) def get_user_info(self): """Gets information about the user currently logged into HydroShare """ user_info = None error = None try: user_info = self.hs_api_conn.getUserInfo() except HydroShareHTTPException as e: if e.status_code == 401: # Unauthorized error = HYDROSHARE_AUTHENTICATION_ERROR else: error = { 'type': 'GenericHydroShareHTTPException', 'message': e.status_msg, } return user_info, error def delete_resource_locally(self, res_id): """ Attempts to delete the local copy of the resource files from the disk :param res_id: the ID of the resource to delete :type res_id: str """ resource_path = self.output_folder / res_id if not resource_path.exists(): return { 'type': 'FileNotFoundError', 'message': f'Could not find a local copy of resource {res_id} to delete.', } try: shutil.rmtree(str(resource_path)) except IOError as e: return { 'type': 'IOError', 'message': f'Something went wrong. Could not delete resource {res_id}.', } return None # No error def delete_resource_from_hs(self, res_id): try: self.hs_api_conn.deleteResource(res_id) except HydroShareHTTPException as e: return { 'type': 'GenericHydroShareHTTPException', 'message': e.status_msg, } return None # No error def get_local_resource_ids(self): """ Gets a list of IDs of all the resources saved locally :return a set of strings containing all of the resource IDs :rtype set<str> """ return set(map(lambda p: p.name, self.output_folder.glob('*'))) def get_list_of_user_resources(self): # TODO: speed this up """Gets list of all the resources for the logged in user, including those stored on hydroshare and those stored locally on jupyterhub and information about each one including whether HS ones are stored locally. Assumes there are no local resources that don't exist in HS """ error = None resources = {} # Get local res_ids self.local_res_ids = self.get_local_resource_ids() # Get the user's resources from HydroShare user_hs_resources = self.hs_api_conn.resources(owner=self.username) try: hs_resources = list( user_hs_resources) # Resources can't be listed if auth fails except AttributeError: return None, HYDROSHARE_AUTHENTICATION_ERROR for res in hs_resources: res_id = res['resource_id'] resources[res_id] = { 'abstract': res.get('abstract'), 'authors': res.get('authors'), 'creator': res.get('creator'), 'created': res.get('date_created'), 'lastUpdated': res.get('date_last_updated'), 'localCopyExists': res_id in self.local_res_ids, 'localFiles': res.get('localFiles'), 'id': res_id, 'isPublic': res.get('public'), 'published': res.get('published'), 'status': res.get('status'), 'title': res.get('resource_title'), 'url': res.get('resource_url'), } return list(resources.values()), error def create_HS_resource(self, resource_title, creators, abstract="", privacy="Private"): """ Creates a hydroshare resource from the metadata specified in a dict using the given resource_title and specified creators """ error = None resource_id = None # Type errors for resource_title and creators if not isinstance(resource_title, str): error = { 'type': 'IncorrectType', 'message': 'Resource title should be a string.' } return resource_id, error if not isinstance(creators, list) or not all( isinstance(creator, str) for creator in creators): error = { 'type': 'IncorrectType', 'message': '"Creators" object should be a list of strings.' } return resource_id, error if abstract is None: abstract = "" if privacy == "Public": public = True else: public = False meta_metadata = [] for creator in creators: meta_metadata.append({"creator": {"name": creator}}) meta_metadata = json.dumps(meta_metadata) metadata = { 'abstract': abstract, 'title': resource_title, 'keywords': (), 'rtype': 'GenericResource', 'fpath': '', 'metadata': meta_metadata, 'extra_metadata': '' } resource_id = '' try: resource_id = self.hs_api_conn.createResource( metadata['rtype'], metadata['title'], resource_file=metadata['fpath'], keywords=metadata['keywords'], abstract=metadata['abstract'], metadata=metadata['metadata'], extra_metadata=metadata['extra_metadata']) self.hs_api_conn.setAccessRules(resource_id, public=public) except: error = { 'type': 'UnknownError', 'message': 'Unable to create resource.' } return resource_id, error def create_copy_of_resource_in_hs(self, src_res_id): """ Makes a copy of a resource in HydroShare and updates the local copy of the old resource to point to then ew copy. :param src_res_id: the ID of the resource in HydroShare to duplicate :type src_res_id: str :returns the new resource ID """ response = self.hs_api_conn.resource(src_res_id).copy() new_id = response.content.decode("utf-8") # b'id' is_local = src_res_id in self.local_res_ids # if a local version exists under the old resource ID, rename it to point to the new one if is_local: og_path = self.output_folder / src_res_id / src_res_id new_path = self.output_folder / new_id / new_id shutil.move(str(og_path), str(new_path)) # Change name to 'Copy of []' title = self.hs_api_conn.getScienceMetadata(new_id)['title'] new_title = "Copy of " + title self.hs_api_conn.updateScienceMetadata(new_id, {"title": new_title}) return new_id def get_archive_message(self): """ Gets the message to display on the resource page prompting the user to archive their resources to HydroShare """ loaded_archive_message = get_config_values(['archiveMessage']) if loaded_archive_message and loaded_archive_message['archiveMessage']: archive_message = loaded_archive_message['archiveMessage'] else: archive_message = input("Please enter the archive message: ") #Save the archive message saved_successfully = set_config_values({ 'archiveMessage': archive_message, }) if saved_successfully: logging.info( 'Successfully saved archive message to config file.') return archive_message
from hs_restclient import HydroShare, HydroShareAuthBasic # Download LCZO sesnor database from Hydroshare # link to the Hydroshare resource https://www.hydroshare.org/resource/b38bc00887ec45ac9499f9dea45eb8d5/ auth = HydroShareAuthBasic(username="******", password="******") hs = HydroShare(auth=auth) hs.getResource('b38bc00887ec45ac9499f9dea45eb8d5', destination='./lczodata', unzip=True)