def test_update_scimeta(self): hs = HydroShare(prompt_auth=False) metadata_to_update = {'title': 'Updated resource title'} scimeta = hs.updateScienceMetadata('511debf8858a4ea081f78d66870da76c', metadata=metadata_to_update) self.assertEqual(scimeta['title'], 'Updated resource title') self.assertEqual(len(scimeta['creators']), 2) self.assertEqual(len(scimeta['contributors']), 1) self.assertEqual(len(scimeta['coverages']), 1) self.assertEqual(len(scimeta['dates']), 2) self.assertEqual( scimeta['description'], 'Time series of level, area and volume in the Great Salt Lake. Volume and area of the Great Salt Lake are derived from recorded levels' ) self.assertEqual(scimeta['formats'][0]['value'], 'image/tiff') self.assertEqual(len(scimeta['funding_agencies']), 1) self.assertEqual(len(scimeta['identifiers']), 1) self.assertEqual(scimeta['language'], 'eng') self.assertEqual( scimeta['rights'], 'This resource is shared under the Creative Commons Attribution CC BY. http://creativecommons.org/licenses/by/4.0/' ) self.assertEqual(scimeta['type'], 'http://www.hydroshare.org/terms/GenericResource') self.assertEqual(scimeta['publisher'], None) self.assertEqual(len(scimeta['sources']), 0) self.assertEqual(len(scimeta['relations']), 0) self.assertEqual(len(scimeta['subjects']), 2)
def get_epanet_model_metadata(request): return_obj = { 'success': False, 'message': None, 'results': "", } if request.is_ajax() and request.method == 'GET': if not request.GET.get('model_id'): return_obj['message'] = message_template_param_unfilled.format( param='model_id') else: model_id = request.GET['model_id'] try: hs = get_oauth_hs(request) except: hs = HydroShare() metadata_json = hs.getScienceMetadata(model_id) return_obj['results'] = metadata_json return_obj['success'] = True else: return_obj['message'] = message_template_wrong_req_method.format( method="GET") return JsonResponse(return_obj)
def download_epanet_model(request): return_obj = {'success': False, 'message': None, 'results': "", 'name': ""} if request.is_ajax() and request.method == 'GET': if not request.GET.get('model_id'): return_obj['message'] = message_template_param_unfilled.format( param='model_id') else: model_id = request.GET['model_id'] try: hs = get_oauth_hs(request) except: hs = HydroShare() for model_file in hs.getResourceFileList(model_id): model_url = model_file['url'] model_name = model_url[model_url.find('contents/') + 9:] model = "" for line in hs.getResourceFile(model_id, model_name): model += line.decode("utf-8") return_obj['name'] = model_name return_obj['results'] = model return_obj['success'] = True else: return_obj['message'] = message_template_wrong_req_method.format( method="POST") return JsonResponse(return_obj)
def test_get_resourcemap(self): hs = HydroShare(prompt_auth=False) resourcemap = hs.getResourceMap('6dbb0dfb8f3a498881e4de428cb1587c') self.assertTrue( resourcemap.find( """<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">""" ) != -1)
def test_get_resource_list_filter_date(self): hs = HydroShare(hostname=self.url) from_date = date(2015, 5, 20) res_list = hs.getResourceList(from_date=from_date) for (i, r) in enumerate(res_list): self.assertTrue( datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date) to_date = date(2015, 5, 21) # up to and including 5/21/2015 res_list = hs.getResourceList(to_date=to_date) for (i, r) in enumerate(res_list): self.assertTrue( datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date) from_date = date(2015, 5, 19) to_date = date(2015, 5, 22) # up to and including 5/21/2015 res_list = hs.getResourceList(from_date=from_date, to_date=to_date) for (i, r) in enumerate(res_list): self.assertTrue( datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date) self.assertTrue( datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date)
def test_get_scimeta_xml(self): hs = HydroShare(prompt_auth=False) scimeta = hs.getScienceMetadataRDF('6dbb0dfb8f3a498881e4de428cb1587c') self.assertTrue( scimeta.find( """<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">""" ) != -1)
def authenticate(self, username, password, client_id=None, client_secret=None): """ Authenticates access to allow read/write access to privileged resource_cache :param username: username for HydroShare.org :param password: password associated with username :param client_id: Client ID obtained from HydroShare :param client_secret: Client Secret provided by HydroShare :return: Returns true if authentication was successful, false otherwise """ if not all([username, password]): self.auth = None return False if client_id is not None and client_secret is not None: self.auth = HydroShareAuthOAuth2(client_id, client_secret, username=username, password=password) else: self.auth = HydroShareAuthBasic(username, password) try: self.client = HydroShare(auth=self.auth) # , verify=False) self.user_info = self.client.getUserInfo() return True except HydroShareException as e: # for incorrect username/password combinations print('Authentication failed: {}'.format(e)) except InvalidGrantError as e: # for failures when attempting to use OAuth2 print('Credentials could not be validated: {}'.format(e)) except InvalidClientError as e: print('Invalid client ID and/or client secret: {}'.format(e)) except Exception as e: print(e) self.auth = None return False
def __init__(self, username=None): self.hs = None self.content = {} # load the HS environment variables # self.load_environment() uname = username if uname is None: uname = os.environ['HS_USR_NAME'] # get a secure connection to hydroshare auth = self.getSecureConnection(uname) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() display(HTML('<b style="color:green;">Successfully established a connection with HydroShare</b>')) except HydroShareHTTPException as e: display(HTML( '<p style="color:red;"><b>Failed to establish a connection with HydroShare. Please check that you provided the correct credentials</b><br>%s </p>' % e)) # remove the cached authentication auth_path = os.path.join(os.path.dirname(__file__), '../../../.auth') os.remove(auth_path) return None
def test_resource_scimeta_custom(self): hs = HydroShare(prompt_auth=False) response = hs.resource('511debf8858a4ea081f78d66870da76c').scimeta.custom({ "foo": "bar", "foo2": "bar2" }) self.assertEqual(response.status_code, 200)
def home(request): """ Controller for the app home page. """ logger = logging.getLogger('django') logger.error('Entering controller.py...') res_output = "" client_id = getattr(settings, "SOCIAL_AUTH_HYDROSHARE_KEY", "None") client_secret = getattr(settings, "SOCIAL_AUTH_HYDROSHARE_SECRET", "None") token = request.user.social_auth.get(provider='hydroshare').extra_data['token_dict'] token_dict_str = "Token: {0}".format(str(token)) logger.error(token_dict_str) auth = HydroShareAuthOAuth2(client_id, client_secret, token=token) try: logger.error("Fetching resource list from playground.hydroshare.org") hs = HydroShare(auth=auth, hostname='playground.hydroshare.org') for resource in hs.getResourceList(): res_output += str(resource) context = {"res_output": res_output, "token_dict_str": token_dict_str} return render(request, 'oauth_user_demo/home.html', context) except TokenExpiredError as e: # TODO: redirect back to login view, giving this view as the view to return to logger.error("TokenExpiredError: TODO: redirect to login view") raise e
def test_referenced_update(self): hs = HydroShare(prompt_auth=False) response = hs.updateReferencedFile(pid='511debf8858a4ea081f78d66870da76c', path='data/contents', name='file.url', ref_url='https://www.cuahsi.org') self.assertEqual(response['status'], 'success')
def test_get_resource_list(self): hs = HydroShare() res_list = hs.getResourceList() for (i, r) in enumerate(res_list): self.assertEquals(r['resource_title'], self.resource_titles[i]) self.assertEquals(r['resource_id'], self.resource_ids[i])
def __init__(self, username=None, password=None, cache=True): self.hs = None self.content = {} # load the HS environment variables # todo: this should be set as a path variable somehow. # possibly add JPY_TMP to Dockerfile self.cache = cache if cache: utilities.load_environment() self.auth_path = '/home/jovyan/.auth' if password is None: # try for secure connection auth = self.getSecureConnection(username) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=username, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path)
def test_resource_move_or_rename(self): hs = HydroShare(prompt_auth=False) response = hs.resource('511debf8858a4ea081f78d66870da76c').functions.move_or_rename({ "source_path": "/source/path", "target_path": "/target/path" }) self.assertEqual(response.status_code, 200)
def test_get_resource_list(self): hs = HydroShare(prompt_auth=False) res_list = hs.getResourceList() for (i, r) in enumerate(res_list): self.assertEqual(r['resource_title'], self.resource_titles[i]) self.assertEqual(r['resource_id'], self.resource_ids[i])
def test_get_user_info(self): hs = HydroShare(prompt_auth=False) user_info = hs.getUserInfo() self.assertEqual(user_info['username'], 'username') self.assertEqual(user_info['first_name'], 'First') self.assertEqual(user_info['last_name'], 'Last') self.assertEqual(user_info['email'], '*****@*****.**')
def test_resource_zip_folder(self): hs = HydroShare(prompt_auth=False) response = hs.resource('511debf8858a4ea081f78d66870da76c').functions.zip({ "input_coll_path": "/source/path", "output_zip_fname": "filename.zip", "remove_original_after_zip": True }) self.assertEqual(response.status_code, 200)
def get_resource_list(): hs = HydroShare() resource_list = [] for resource in hs.getResourceList(types=['RasterResource']): res_name = '{} [creator: {}]'.format(resource['resource_title'], resource['creator']) resource_list.append((res_name, resource['resource_id'])) return resource_list
def test_get_user_info(self): hs = HydroShare() user_info = hs.getUserInfo() self.assertEquals(user_info['username'], 'username') self.assertEquals(user_info['first_name'], 'First') self.assertEquals(user_info['last_name'], 'Last') self.assertEquals(user_info['email'], '*****@*****.**')
def test_get_folder_contents(self): hs = HydroShare(prompt_auth=False) folder_contents = hs.getResourceFolderContents('511debf8858a4ea081f78d66870da76c', pathname='model/initial/') self.assertEqual(folder_contents['resource_id'], '511debf8858a4ea081f78d66870da76c') self.assertEqual(folder_contents['path'], 'model/initial') self.assertEqual(folder_contents['files'], ["model.exe", "param.txt"]) self.assertEqual(folder_contents['folders'], ["run/1", "run/2"])
def test_resource_scimeta_custom(self): hs = HydroShare(prompt_auth=False) response = hs.resource( '511debf8858a4ea081f78d66870da76c').scimeta.custom({ "foo": "bar", "foo2": "bar2" }) self.assertEqual(response.status_code, 200)
def test_folder_delete(self): hs = HydroShare(prompt_auth=False) response = hs.deleteResourceFolder('511debf8858a4ea081f78d66870da76c', pathname='model/initial/') self.assertEqual(response['resource_id'], '511debf8858a4ea081f78d66870da76c') self.assertEqual(response['path'], 'model/initial')
def test_get_resource_file_list(self): hs = HydroShare() res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c') for (i, r) in enumerate(res_list): self.assertEquals(r['url'], self.urls[i]) self.assertEquals(r['size'], self.sizes[i]) self.assertEquals(r['content_type'], self.content_types[i])
def test_get_resource_file_list(self): hs = HydroShare(prompt_auth=False) res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c') for (i, r) in enumerate(res_list): self.assertEqual(r['url'], self.urls[i]) self.assertEqual(r['size'], self.sizes[i]) self.assertEqual(r['content_type'], self.content_types[i])
def test_resource_list_by_bounding_box(self): hs = HydroShare(prompt_auth=False) res_list = hs.resources(coverage_type="box", north="50", south="30", east="40", west="20") for (i, r) in enumerate(res_list): self.assertEquals(True, True)
def test_set_file_type(self): hs = HydroShare(prompt_auth=False) response = hs.resource( '511debf8858a4ea081f78d66870da76c').functions.set_file_type({ "file_path": "file_path", "hs_file_type": "NetCDF" }) self.assertEqual(response.status_code, 202)
def test_resource_move_or_rename(self): hs = HydroShare(prompt_auth=False) response = hs.resource( '511debf8858a4ea081f78d66870da76c').functions.move_or_rename({ "source_path": "/source/path", "target_path": "/target/path" }) self.assertEqual(response.status_code, 200)
def test_resource_unzip_file(self): hs = HydroShare() response = hs.resource( '511debf8858a4ea081f78d66870da76c').functions.unzip({ "zip_with_rel_path": "/path/to/zip", "remove_original_zip": True }) self.assertEqual(response.status_code, 200)
def test_referenced_update(self): hs = HydroShare(prompt_auth=False) response = hs.updateReferencedFile( pid='511debf8858a4ea081f78d66870da76c', path='data/contents', name='file.url', ref_url='https://www.cuahsi.org') self.assertEqual(response['status'], 'success')
def test_resource_upload_file(self): hs = HydroShare() response = hs.resource('511debf8858a4ea081f78d66870da76c').files({ "file": 'mocks/data/another_resource_file.txt', "folder": "/target/folder" }) self.assertEqual(response.status_code, 200)
def test_get_resource_types(self): hs = HydroShare(prompt_auth=False) res_type_proto = { 'GenericResource', 'ModelInstanceResource', 'ModelProgramResource', 'NetcdfResource', 'RasterResource', 'RefTimeSeries', 'SWATModelInstanceResource', 'TimeSeriesResource', 'ToolResource' } res_types = hs.getResourceTypes() self.assertSetEqual(res_type_proto, res_types)
def download_executable_lubuntu_hs(save_filepath): resource_id = 'a5dbd5b198c9468387f59f3fefc11e22' hs = HydroShare() hs.getResource(resource_id, destination=save_filepath, unzip=True) resource_name = 'summa-master.zip' executable_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name shutil.unpack_archive(executable_filepath, extract_dir=os.path.dirname(executable_filepath)) executable = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/summa-master/bin' return os.path.dirname(executable)
def test_get_folder_contents(self): hs = HydroShare(prompt_auth=False) folder_contents = hs.getResourceFolderContents( '511debf8858a4ea081f78d66870da76c', pathname='model/initial/') self.assertEqual(folder_contents['resource_id'], '511debf8858a4ea081f78d66870da76c') self.assertEqual(folder_contents['path'], 'model/initial') self.assertEqual(folder_contents['files'], ["model.exe", "param.txt"]) self.assertEqual(folder_contents['folders'], ["run/1", "run/2"])
def start_file_download(request): global temp_dir, prediction_data, rp_data, total_prediction_comids, total_rp_comids, sorted_prediction_comids, \ sorted_rp_comids, time if request.method == 'GET': get_data = request.GET this_script_path = inspect.getfile(inspect.currentframe()) try: if get_data['res_id'] is not None: temp_dir = tempfile.mkdtemp() file_path = get_data['res_id'] if get_data['src'] == 'iRODS': download = requests.get(file_path, stream=True) filename = os.path.basename(file_path) local_file_path = os.path.join(temp_dir, filename) with open(local_file_path, 'wb') as fd: for chunk in download.iter_content(1024): fd.write(chunk) prediction_data = nc.Dataset(local_file_path, mode="r") elif get_data['src'] == 'hs': auth = HydroShareAuthBasic(username='******', password='******') hs = HydroShare(auth=auth, hostname="playground.hydroshare.org", use_https=False) resource_data = hs.getSystemMetadata(file_path) filename = resource_data['resource_title'] # this will only work if there is only one file in the resource and if # the resource title is the same as filename hs.getResourceFile(file_path, filename, destination=temp_dir) local_file_path = temp_dir + "/" + filename prediction_data = nc.Dataset(local_file_path, mode="r") else: testfile_path = this_script_path.replace('controllers.py', 'public/data/test.nc') prediction_data = nc.Dataset(testfile_path, mode="r") # Sort the RAPID netCDF file by COMID qout_dimensions = prediction_data.variables['Qout'].dimensions if qout_dimensions[0].lower() == 'comid' and qout_dimensions[1].lower() == 'time': sorted_prediction_comids = sorted(enumerate(prediction_data.variables['COMID'][:]), key=lambda comid: comid[1]) total_prediction_comids = len(sorted_prediction_comids) else: return JsonResponse({'error': "Invalid netCDF file"}) variables = prediction_data.variables.keys() if 'time' in variables: time = [t * 1000 for t in prediction_data.variables['time'][:]] else: return JsonResponse({'error': "Invalid netCDF file"}) rp_data_path = this_script_path.replace('controllers.py', 'public/data/return_period_data.nc') rp_data = nc.Dataset(rp_data_path, mode="r") sorted_rp_comids = sorted(enumerate(rp_data.variables['COMID'][:]), key=lambda comid: comid[1]) total_rp_comids = len(sorted_rp_comids) return JsonResponse({'success': "The file is ready to go."}) except Exception, err: return JsonResponse({'error': err})
def map(request): """ Controller for the app home page. """ hs = HydroShare() res_info_list = [] i =0 for resource in hs.getResourceList(types=['MODFLOWModelInstanceResource']): xml = hs.getScienceMetadata(resource['resource_id']) print "iteration number" print i dict = parse(xml) #takes xml turns into string then json dictionary object ^ try: box = dict['rdf:RDF']['rdf:Description'][0]['dc:coverage'][1]['dcterms:box']['rdf:value'] #['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value'] # ^From the dictionary I pull out the values I need or the data from the "coverage" part of the model list_points = box.split('; ') # ^ splits up box to produce a list containing: northlimit, eastlimit, southlimit, westlimit, units # and porjection. temp_dict = {} #creates an empty dictionary for l in list_points: temp_list = l.split('=') # splits up the list even further into name and value ex: northlimit, 40.5033 # print temp_list temp_dict[temp_list[0]] = temp_list[1] res_info = {} res_info["resource_id"]=resource['resource_id'] res_info["box"]=temp_dict res_info_list.append(res_info) print res_info except Exception, e: pass try: box = dict['rdf:RDF']['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value'] # ['rdf:Description'][0]['dc:coverage'][0]['dcterms:box']['rdf:value'] # ^From the dictionary I pull out the values I need or the data from the "coverage" part of the model list_points = box.split('; ') # ^ splits up box to produce a list containing: northlimit, eastlimit, southlimit, westlimit, units # and porjection. temp_dict = {} # creates an empty dictionary for l in list_points: temp_list = l.split('=') # splits up the list even further into name and value ex: northlimit, 40.5033 # print temp_list temp_dict[temp_list[0]] = temp_list[1] res_info = {} res_info["resource_id"] = resource['resource_id'] res_info["box"] = temp_dict res_info_list.append(res_info) print res_info except Exception, e: pass
def handle(self, *args, **options): # (f,fileid, databeginson,columnheaderson, cmd): # cmdline = bool(options['cmdline'][0]) username = str(options['username'][0]) password = str(options['password'][0]) hs_client_id = settings.SOCIAL_AUTH_HYDROSHARE_UP_KEY hs_client_secret = settings.SOCIAL_AUTH_HYDROSHARE_UP_SECRET auth = HydroShareAuthOAuth2(hs_client_id, hs_client_secret, username=username, password=password) # print(username) # print(password) export_complete = True resource_link = '' user = str(options['django_user_name'][0]) datasettitle = str(options['datasettitle'][0]) startdate = str(options['startdate'][0]) enddate = str(options['enddate'][0]) datafile = str(options['datafile'][0]) dbfilename = str(options['dbfilename'][0]) # print(request.POST['hydroshareusername']) # hs = get_oauth_hs(request) # userInfo = hs.getUserInfo() # hs = HydroShare(auth=auth) # username = hs.getUserInfo() # print(username) abstracttext = '' title = '' abstracttext += 'ODM2 Admin dataset: ' + str(datasettitle) title += 'ODM2 Admin dataset ' + str(datasettitle) abstract = abstracttext keywords = ['ODM2'] rtype = 'GenericResource' fpath = datafile # str(exportdb.DATABASES['default']['NAME']) # # print(fpath) # #metadata = '[{"coverage":{"type":"period", "value":{"start":"'+entered_start_date +'", "end":"'+ entered_end_date +'"}}}, {"creator":{"name":"Miguel Leon"}}]' metadata = '[{"coverage":{"type":"period", "value":{"start":"' + str(startdate) + '", "end":"' + str( enddate) + '"}}}, ' \ '{"creator":{"name":"' + user + '"}}]' extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}' # #abstract = 'My abstract' # #title = 'My resource' # #keywords = ('my keyword 1', 'my keyword 2') # #rtype = 'GenericResource' # #fpath = 'C:/Users/leonmi/Google Drive/ODM2AdminLT2/ODM2SQliteBlank.db' # #metadata = '[{"coverage":{"type":"period", "value":{"start":"01/01/2000", "end":"12/12/2010"}}}, {"creator":{"name":"John Smith"}}, {"creator":{"name":"Lisa Miller"}}]' # #extra_metadata = '{"key-1": "value-1", "key-2": "value-2"}' # messages.success(request, 'Profile details updated.') resource_id = hs.createResource(rtype, title, resource_file=datafile, resource_filename=dbfilename, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) print('resource created') print(resource_id)
def check_auth(self, auth): hs = HydroShare(auth=auth) self.log.info('hs=%s' % str(hs)) try: info = hs.getUserInfo() self.settings['hydroshare'] = hs self.log.info('info=%s' % info) except: hs = None return hs
def makePublic(self, resource, public=True): """ Makes a resource public or private :param resource: The resource you want to modify :param public: boolean value, True makes the resource public, False makes it private (wowzer) :return: None """ hs = HydroShare(auth=self.auth) res = hs.resource(resource.id).public(public) if res.status_code == 200 or res.status_code == 202: resource.public = public
def install_test_cases_hs(save_filepath): resource_id = 'a0105d479c334764ba84633c5b9c1c01' hs = HydroShare() hs.getResource(resource_id, destination=save_filepath, unzip=True) resource_name = 'summatestcases-2.x.tar.gz' testcase_filepath = save_filepath + '/' + resource_id + '/' + resource_id + '/data/contents/' + resource_name shutil.unpack_archive(testcase_filepath, extract_dir=os.path.dirname(testcase_filepath)) cmd = "cd {}/summaTestCases_2.x/; ./installTestCases_local.sh".format( os.path.dirname(testcase_filepath)) subprocess.run(cmd, shell=True) return os.path.dirname(testcase_filepath)
def download_model_instance(resource_id): path = (os.getcwd() + '/' + resource_id + '/' + resource_id + '/data/contents/') hs = HydroShare() hs.getResource(resource_id, destination=os.getcwd(), unzip=True) # unpack the simulation archive and remove unncessary files Model_Instance_Name = os.listdir(path)[0] shutil.unpack_archive(path + Model_Instance_Name, extract_dir=os.getcwd()) cmd = "rm -rf " + resource_id subprocess.run(cmd, shell=True) return Model_Instance_Name.split('.')[0]
def get_hs_resource(resource_id, file_path): hs = HydroShare() hs.getResource(resource_id, destination=file_path, unzip=True) # unpack the simulation archive and remove unncessary files hs_resource_dir = os.path.join(file_path, resource_id, resource_id, 'data/contents/') hs_resource = os.listdir(hs_resource_dir) shutil.unpack_archive(hs_resource_dir + hs_resource[0], extract_dir=file_path) shutil.rmtree(os.path.join(file_path, resource_id)) return hs_resource[0].split(".")[0]
def test_resource_zip_folder(self): hs = HydroShare(prompt_auth=False) response = hs.resource( '511debf8858a4ea081f78d66870da76c').functions.zip({ "input_coll_path": "/source/path", "output_zip_fname": "filename.zip", "remove_original_after_zip": True }) self.assertEqual(response.status_code, 200)
def upload_to_hydroshare(request): try: if request.method == 'GET': get_data = request.GET download_path = str(get_data['download_path']) hs_username = str(get_data['hs_username']) hs_password = str(get_data['hs_password']) r_title = str(get_data['r_title']) r_type = str(get_data['r_type']) r_abstract = str(get_data['r_abstract']) r_keywords_raw = str(get_data['r_keywords']) r_keywords = r_keywords_raw.split(',') #startup a Hydroshare instance with user's credentials auth = HydroShareAuthBasic(username=hs_username, password=hs_password) hs = HydroShare(auth=auth, hostname="dev.hydroshare.org", use_https=True) # try to download a tiny file simply to test the user's credentials # test_id = '49d01b5b0d0a41b6a5a31d8aace0a36e' # hs.getResource(test_id, destination=None, unzip=False) #download the iRODS file to a temp directory temp_dir = tempfile.mkdtemp() filename = os.path.basename(download_path) download_file_path = os.path.join(temp_dir, filename) download = requests.get(download_path, stream=True) with open(download_file_path, 'wb') as fd: for chunk in download.iter_content(1024): fd.write(chunk) #upload the temp file to HydroShare if os.path.exists(download_file_path): resource_id = hs.createResource(r_type, r_title, resource_file=download_file_path, keywords=r_keywords, abstract=r_abstract) else: if temp_dir: # remove the temp directory/file shutil.rmtree(temp_dir) return JsonResponse({'error': 'An error occurred with the file upload.'}) except Exception, err: if temp_dir: # remove the temp directory/file shutil.rmtree(temp_dir) if "401 Unauthorized" in str(err): return JsonResponse({'error': 'Username or password invalid.'}) elif "400 Bad Request" in str(err): return JsonResponse({'error': 'File uploaded successfully despite 400 Bad Request Error.'}) else: traceback.print_exc() return JsonResponse({'error': 'HydroShare rejected the upload for some reason.'})
def test_get_resource_types(self): hs = HydroShare() res_type_proto = {'GenericResource', 'ModelInstanceResource', 'ModelProgramResource', 'NetcdfResource', 'RasterResource', 'RefTimeSeries', 'SWATModelInstanceResource', 'TimeSeriesResource', 'ToolResource'} res_types = hs.getResourceTypes() self.assertSetEqual(res_type_proto, res_types)
def test_create_get_delete_resource_file(self): hs = HydroShare() # Add res_id = '511debf8858a4ea081f78d66870da76c' fpath = 'mocks/data/another_resource_file.txt' fname = os.path.basename(fpath) resp = hs.addResourceFile(res_id, fpath) self.assertEqual(resp, res_id) # Get tmpdir = tempfile.mkdtemp() res_file = hs.getResourceFile(res_id, fname, destination=tmpdir) self.assertTrue(filecmp.cmp(res_file, fpath, shallow=False)) shutil.rmtree(tmpdir) # Delete delres = hs.deleteResourceFile(res_id, fname) self.assertEqual(delres, res_id)
def test_get_resource_list_filter_date(self): hs = HydroShare() from_date = date(2015, 5, 20) res_list = hs.getResourceList(from_date=from_date) for (i, r) in enumerate(res_list): self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date) to_date = date(2015, 5, 21) # up to and including 5/21/2015 res_list = hs.getResourceList(to_date=to_date) for (i, r) in enumerate(res_list): self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date) from_date = date(2015, 5, 19) to_date = date(2015, 5, 22) # up to and including 5/21/2015 res_list = hs.getResourceList(from_date=from_date, to_date=to_date) for (i, r) in enumerate(res_list): self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() >= from_date) self.assertTrue(datetime.strptime(r['date_created'], '%m-%d-%Y').date() < to_date)
def test_create_get_delete_resource(self): hs = HydroShare(prompt_auth=False) abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' metadata = json.dumps([{'coverage': {'type': 'period', 'value': {'start': '01/01/2000', 'end': '12/12/2010'}}}, {'creator': {'name': 'John Smith'}}, {'contributor': {'name': 'Lisa Miller'}}]) extra_metadata = json.dumps({'latitude': '40', 'longitude': '-111'}) with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract, metadata=metadata, extra_metadata=extra_metadata) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'rb') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def test_get_scimeta_json(self): hs = HydroShare(prompt_auth=False) scimeta = hs.getScienceMetadata('511debf8858a4ea081f78d66870da76c') self.assertEqual(scimeta['title'], 'Great Salt Lake Level and Volume') self.assertEqual(len(scimeta['creators']), 2) self.assertEqual(len(scimeta['contributors']), 1) self.assertEqual(len(scimeta['coverages']), 1) self.assertEqual(len(scimeta['dates']), 2) self.assertEqual(scimeta['description'], 'Time series of level, area and volume in the Great Salt Lake. Volume and area of the Great Salt Lake are derived from recorded levels') self.assertEqual(scimeta['formats'][0]['value'], 'image/tiff') self.assertEqual(len(scimeta['funding_agencies']), 1) self.assertEqual(len(scimeta['identifiers']), 1) self.assertEqual(scimeta['language'], 'eng') self.assertEqual(scimeta['rights'], 'This resource is shared under the Creative Commons Attribution CC BY. http://creativecommons.org/licenses/by/4.0/') self.assertEqual(scimeta['type'], 'http://www.hydroshare.org/terms/GenericResource') self.assertEqual(scimeta['publisher'], None) self.assertEqual(len(scimeta['sources']), 0) self.assertEqual(len(scimeta['relations']), 0) self.assertEqual(len(scimeta['subjects']), 2)
def __init__(self, username=None, password=None, cache=True): self.hs = None self.content = {} # connect to hydroshare using OAUTH2 authfile = os.path.expanduser("~/.hs_auth") if os.path.exists(authfile): with open(authfile, 'rb') as f: token, cid = pickle.load(f) auth = HydroShareAuthOAuth2(cid, '', token=token) else: # connect to hydroshare using Basic Authentication self.cache = cache if cache: utilities.load_environment(os.path.join( os.environ['NOTEBOOK_HOME'], '.env')) self.auth_path = os.environ.get('NOTEBOOK_HOME', '/home/jovyan/.auth') uname = username if uname is None: if 'HS_USR_NAME' in os.environ.keys(): uname = os.environ['HS_USR_NAME'] if password is None: # get a secure connection to hydroshare auth = self.getSecureConnection(uname) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=uname, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path) return None # set the HS resource download directory download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'Downloads') if not os.path.isdir(download_dir): os.makedirs(download_dir) self.download_dir = download_dir
def run_service(user_input): auth = HydroShareAuthBasic(hs_username, hs_password) hs = HydroShare(auth=auth) client = HydroDS(username=ci_username, password=ci_password) # step1: get raster resource files from HydroShare # TODO remember to check if the resource is downloadable resource_id = user_input['input_raster_url_path'] destination = tempfile.mkdtemp() hs.getResource(resource_id, destination, unzip=True) raster_file_folder = '{0}/{1}/{2}/data/contents'.format(destination, resource_id, resource_id) raster_file_path_list = [os.path.join(raster_file_folder, f) for f in os.listdir(raster_file_folder) if '.vrt' not in f] upload_file_url = client.upload_file(file_to_upload=raster_file_path_list[0]) # step2: processing files in HydroDS output_raster_name = 'process_{}'.format(upload_file_url.split('/')[-1]) output_file = client.project_resample_raster(input_raster_url_path=upload_file_url, cell_size_dx=int(user_input['cell_size_dx']), cell_size_dy=int(user_input['cell_size_dy']), output_raster=output_raster_name, resample=user_input['resample'], epsg_code=int(user_input['proj_code']) if user_input['proj_code_type']=='epsg_code' else None, utm_zone=int(user_input['proj_code']) if user_input['proj_code_type']=='utm_zone' else None, ) # step3: download file and create HydroShare resource save_as = '{}/{}'.format(destination, output_file['output_raster'].split('/')[-1]) client.download_file(file_url_path=output_file['output_raster'], save_as=save_as) title = user_input['output_raster'] rtype = 'RasterResource' resource_id = hs.createResource(rtype, title, resource_file=save_as) # step4: clean up the temp in Tethys and HydroDS shutil.rmtree(destination) client.delete_my_file(output_file['output_raster'].split('/')[-1]) client.delete_my_file(upload_file_url.split('/')[-1]) return {'is_success': True, 'message': 'http://www.hydroshare.org/resource/{}/'.format(resource_id)}
def test_create_get_delete_resource(self): hs = HydroShare() abstract = 'Abstract for hello world resource' title = 'Minimal hello world resource' keywords = ('hello', 'world') rtype = 'GenericResource' fname = 'mocks/data/minimal_resource_file.txt' with HTTMock(mocks.hydroshare.createResourceCRUD): # Create newres = hs.createResource(rtype, title, resource_file=fname, keywords=keywords, abstract=abstract) self.assertIsNotNone(newres) sysmeta = hs.getSystemMetadata(newres) self.assertEqual(sysmeta['resource_id'], newres) self.assertEqual(sysmeta['resource_type'], rtype) self.assertFalse(sysmeta['public']) with HTTMock(mocks.hydroshare.accessRules_put): # Make resource public hs.setAccessRules(newres, public=True) sysmeta = hs.getSystemMetadata(newres) self.assertTrue(sysmeta['public']) with HTTMock(mocks.hydroshare.createResourceCRUD): # Get tmpdir = tempfile.mkdtemp() hs.getResource(newres, destination=tmpdir) with ZipFile(os.path.join(tmpdir, '511debf8858a4ea081f78d66870da76c.zip'), 'r') as zfile: self.assertTrue('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt' in zfile.namelist()) downloaded = zfile.open('511debf8858a4ea081f78d66870da76c/data/contents/minimal_resource_file.txt', 'r') original = open('mocks/data/minimal_resource_file.txt', 'r') self.assertEqual(downloaded.read(), original.read()) downloaded.close() original.close() shutil.rmtree(tmpdir) # Delete delres = hs.deleteResource(newres) self.assertEqual(delres, newres)
def __init__(self, username=None, password=None, cache=True): self.hs = None self.content = {} # load the HS environment variables # todo: this should be set as a path variable somehow. # possibly add JPY_TMP to Dockerfile self.cache = cache if cache: utilities.load_environment(os.path.join( os.environ['NOTEBOOK_HOME'], '.env')) self.auth_path = '/home/jovyan/.auth' # todo: either use JPY_USR or ask them to # enter their hydroshare username uname = username if uname is None: if 'HS_USR_NAME' in os.environ.keys(): uname = os.environ['HS_USR_NAME'] if password is None: # get a secure connection to hydroshare auth = self.getSecureConnection(uname) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=uname, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path) return None
def test_get_resource_list_filter_type(self): hs = HydroShare() res_list = hs.getResourceList(types=('RasterResource',)) for (i, r) in enumerate(res_list): self.assertEquals(r['resource_type'], 'RasterResource')
def Test_All_Resources(request): try: start = time() res_count = 0 num_success = 0 num_errors = 0 error_resource_list = [] ACCEPTABLE_ERRORS_LIST = ['This resource is too large to open in HydroShare GIS', 'There is no projection information associated with this resource', 'Resource contains insufficient geospatial information'] set_currently_testing(True) auth = HydroShareAuthBasic(username='******', password='******') hs = HydroShare(auth=auth) valid_res_types = ['GeographicFeatureResource', 'RasterResource', 'RefTimeSeriesResource', 'GenericResource', 'ScriptResource'] resource_list = hs.getResourceList(types=valid_res_types) num_resources = 0 for res in hs.getResourceList(types=valid_res_types): if res['public']: num_resources += 1 for res in resource_list: res_id = None try: if res['public']: res_count += 1 res_id = res['resource_id'] print "Currently testing resource %s of %s: %s" % (res_count, num_resources, res_id) if res['resource_type'] == 'GenericResource': response = get_res_files_list(hs, res_id) if response['success']: res_files_list = response['results']['generic_res_files_list'] num_files_failed = 0 for i, res_file in enumerate(res_files_list): response = process_generic_res_file(hs, res_id, res_file, request.user.username, i) if response['success']: pass else: error_acceptable = False for error in ACCEPTABLE_ERRORS_LIST: if error in response['message']: error_acceptable = True break if error_acceptable: pass else: num_files_failed += 1 error_resource_list.append('https://www.hydroshare.org/resource/%s on file %s' % (res_id, res_file)) print 'ERROR ENCOUNTERED:' print 'RES_ID: %s' % res_id print 'MESSAGE: %s' % response['message'] if num_files_failed == 0: num_success += 1 else: num_errors += 1 else: response = process_nongeneric_res(hs, res_id) if response['success']: num_success += 1 else: error_acceptable = False for error in ACCEPTABLE_ERRORS_LIST: if error in response['message']: error_acceptable = True break if error_acceptable: num_success += 1 else: num_errors += 1 error_resource_list.append('https://www.hydroshare.org/resource/%s' % res_id) print 'ERROR ENCOUNTERED:' print 'RES_ID: %s' % res_id print 'MESSAGE: %s' % response['message'] except Exception as e: num_errors += 1 error_resource_list.append('https://www.hydroshare.org/resource/%s' % res_id) print 'ERROR ENCOUNTERED:' print 'RES_ID: %s' % res_id print 'MESSAGE: %s' % str(e) print "%d%% complete" % (res_count * 100 / num_resources) elapsed = str(timedelta(seconds=time()-start)) results = ''' ALL TESTS COMPLETE! SUMMARY TIME ELAPSED: {0} TOTAL RESOURCES TESTED: {1} TOTAL FAILED: {2} PERCENT FAILED: {3} TOTAL SUCCESSFUL: {4} PERCENT SUCCESSFUL: {5} {6} {7} '''.format( elapsed, res_count, num_errors, '%d%%' % (num_errors * 100 / res_count), num_success, '%d%%' % (num_success * 100 / res_count), 'LIST OF FAILED RESOURCES:' if len(error_resource_list) != 0 else 'ALL TESTS PASSED!', '\n'.join(error_resource_list) ) print results email_admin('Test Results', custom_msg=results) context = {'results': '<br>'.join(results.split('\n'))} set_currently_testing(False) return render(request, 'hydroshare_gis/test-results.html', context) finally: set_currently_testing(False)
def test_get_resource_list_filter_creator(self): hs = HydroShare() res_list = hs.getResourceList(creator='bmiles') for (i, r) in enumerate(res_list): self.assertEquals(r['creator'], 'bmiles')
def upload_to_hs(id,file): auth = HydroShareAuthBasic(username='******', password='******') hs = HydroShare(auth=auth) fpath = '/path/to/somefile.txt' resource_id = hs.addResourceFile('id', file)
def test_get_scimeta(self): hs = HydroShare() scimeta = hs.getScienceMetadata('6dbb0dfb8f3a498881e4de428cb1587c') self.assertTrue(scimeta.find("""<rdf:Description rdf:about="http://www.hydroshare.org/resource/6dbb0dfb8f3a498881e4de428cb1587c">""") != -1)
class hydroshare(): def __init__(self, username=None, password=None, cache=True): self.hs = None self.content = {} # connect to hydroshare using OAUTH2 authfile = os.path.expanduser("~/.hs_auth") if os.path.exists(authfile): with open(authfile, 'rb') as f: token, cid = pickle.load(f) auth = HydroShareAuthOAuth2(cid, '', token=token) else: # connect to hydroshare using Basic Authentication self.cache = cache if cache: utilities.load_environment(os.path.join( os.environ['NOTEBOOK_HOME'], '.env')) self.auth_path = os.environ.get('NOTEBOOK_HOME', '/home/jovyan/.auth') uname = username if uname is None: if 'HS_USR_NAME' in os.environ.keys(): uname = os.environ['HS_USR_NAME'] if password is None: # get a secure connection to hydroshare auth = self.getSecureConnection(uname) else: print('WARNING: THIS IS NOT A SECURE METHOD OF CONNECTING TO ' 'HYDROSHARE...AVOID TYPING CREDENTIALS AS PLAIN TEXT') auth = HydroShareAuthBasic(username=uname, password=password) try: self.hs = HydroShare(auth=auth) self.hs.getUserInfo() print('Successfully established a connection with HydroShare') except HydroShareHTTPException as e: print('Failed to establish a connection with HydroShare.\n ' 'Please check that you provided the correct credentials.\n' '%s' % e) # remove the cached authentication if os.path.exists(self.auth_path): os.remove(self.auth_path) return None # set the HS resource download directory download_dir = os.environ.get('JUPYTER_DOWNLOADS', 'Downloads') if not os.path.isdir(download_dir): os.makedirs(download_dir) self.download_dir = download_dir def _addContentToExistingResource(self, resid, content_files): for f in content_files: self.hs.addResourceFile(resid, f) def getSecureConnection(self, username=None): """Establishes a secure connection with hydroshare. args: -- email: email address associated with hydroshare returns: -- hydroshare api connection """ if not os.path.exists(self.auth_path): print('\nThe hs_utils library requires a secure connection to ' 'your HydroShare account.') if username is None: username = input('Please enter your HydroShare username: '******'Enter the HydroShare password for user ' '\'%s\': ' % username) auth = HydroShareAuthBasic(username=username, password=p) if self.cache: with open(self.auth_path, 'wb') as f: pickle.dump(auth, f, protocol=2) else: with open(self.auth_path, 'rb') as f: auth = pickle.load(f) return auth def getResourceMetadata(self, resid): """Gets metadata for a specified resource. args: -- resid: hydroshare resource id returns: -- resource metadata object """ science_meta = self.hs.getScienceMetadata(resid) system_meta = self.hs.getSystemMetadata(resid) return resource.ResourceMetadata(system_meta, science_meta) def createHydroShareResource(self, abstract, title, derivedFromId=None, keywords=[], resource_type='GenericResource', content_files=[], public=False): """Creates a hydroshare resource. args: -- abstract: abstract for resource (str, required) -- title: title of resource (str, required) -- derivedFromId: id of parent hydroshare resource (str, default=>None) -- keywords: list of subject keywords (list, default=>[]) -- resource_type: type of resource to create (str, default=> 'GenericResource') -- content_files: data to save as resource content (list, default=>[]) -- public: resource sharing status (bool, default=>False) returns: -- None """ # query the hydroshare resource types and make sure that # resource_type is valid restypes = {r.lower(): r for r in self.hs.getResourceTypes()} try: res_type = restypes[resource_type] except KeyError: display(HTML('<b style="color:red;">[%s] is not a valid ' 'HydroShare resource type.</p>' % resource_type)) return None # get the 'derived resource' metadata if derivedFromId is not None: try: # update the abstract and keyword metadata meta = self.getResourceMetadata(derivedFromId) abstract = meta.abstract \ + '\n\n[Modified in JupyterHub on %s]\n%s' \ % (dt.now(), abstract) keywords = set(keywords + meta.keywords) except: display(HTML('<b style="color:red;">Encountered an error ' ' while setting the derivedFrom relationship ' ' using id=%s. Make sure this resource is ' ' is accessible to your account. ' % derivedFromId)) display(HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % derivedFromId, 'View the "DerivedFrom" ' 'Resource'))) return None f = None if len(content_files) == 0 else content_files[0] # create the hs resource (1 content file allowed) resid = threads.runThreadedFunction('Creating HydroShare Resource', 'Resource Created Successfully', self.hs.createResource, resource_type=res_type, title=title, abstract=abstract, resource_file=f, keywords=keywords) # add the remaining content files to the hs resource try: if len(content_files) > 1: self.addContentToExistingResource(resid, content_files[1:]) except Exception as e: print(e) display(HTML('Resource id: %s' % resid)) display(HTML('<a href=%s target="_blank">%s<a>' % ('https://www.hydroshare.org/resource/%s' % resid, 'Open Resource in HydroShare'))) def getResourceFromHydroShare(self, resourceid, destination='.'): """Downloads content of a hydroshare resource. args: -- resourceid: id of the hydroshare resource (str) -- destination: path to save resource, default /user/[username]/notebooks/data (str) returns: -- None """ default_dl_path = self.download_dir dst = os.path.abspath(os.path.join(default_dl_path, destination)) download = True # check if the data should be overwritten dst_res_folder = os.path.join(dst, resourceid) if os.path.exists(dst_res_folder): print('This resource already exists in your userspace.') utils.tree(dst_res_folder) res = input('\nDo you want to overwrite these data [Y/n]? ') if res != 'n': shutil.rmtree(dst_res_folder) else: download = False # re-download the content if desired if download: try: # download the resource (threaded) threads.runThreadedFunction('Downloading Resource', 'Download Finished', self.hs.getResource, resourceid, destination=dst, unzip=True) print('Successfully downloaded resource %s' % resourceid) except Exception as e: display(HTML('<b style="color:red">Failed to retrieve ' 'resource content from HydroShare: %s</b>' % e)) return None # load the resource content outdir = os.path.join(dst, '%s/%s' % (resourceid, resourceid)) content_files = glob.glob(os.path.join(outdir, 'data/contents/*')) content = {} for f in content_files: fname = os.path.basename(f) # trim the base name relative to the data directory dest_folder_name = os.path.dirname(destination).split('/')[-1] f = os.path.join(dest_folder_name, os.path.relpath(f, dest_folder_name)) content[fname] = f # show the resource content files utilities.display_resource_content_files(content) # update the content dictionary self.content.update(content) def addContentToExistingResource(self, resid, content): """Adds content files to an existing hydroshare resource. args: -- resid: id of an existing hydroshare resource (str) -- content: files paths to be added to resource (list) returns: -- None """ threads.runThreadedFunction('Adding Content to Resource', 'Successfully Added Content Files', self._addContentToExistingResource, resid, content) def loadResource(self, resourceid): """Loads the contents of a previously downloaded resource. args: -- resourceid: the id of the resource that has been downloaded (str) returns: -- {content file name: path} """ resdir = utilities.find_resource_directory(resourceid) if resdir is None: display(HTML('<b style="color:red">Could not find any resource ' 'matching the id [%s].</b> <br> It is likely that ' 'this resource has not yet been downloaded from ' 'HydroShare.org, or it was removed from the ' 'JupyterHub server. Please use the following ' 'command to aquire the resource content: ' '<br><br><code>hs.getResourceFromHydroShare(%s)' '</code>.' % (resourceid, resourceid))) return # create search paths. Need to check 2 paths due to hs_restclient bug #63. search_paths = [os.path.join(resdir, '%s/data/contents/*' % resourceid), os.path.join(resdir, 'data/contents/*')] content = {} found_content = False for p in search_paths: content_files = glob.glob(p) if len(content_files) > 0: found_content = True display(HTML('<p>Downloaded content is located at: %s</p>' % resdir)) display(HTML('<p>Found %d content file(s). \n</p>' % len(content_files))) for f in content_files: fname = os.path.basename(f) content[fname] = f if len(content.keys()) == 0: display(HTML('<p>Did not find any content files for resource id: %s</p>' % resourceid)) utilities.display_resource_content_files(content) self.content = content def getContentFiles(self, resourceid): """Gets the content files for a resource that exists on the Jupyter Server args: -- resourceid: the id of the hydroshare resource returns: -- {content file name: path} """ content = utilities.get_hs_content(resourceid) return content def getContentPath(self, resourceid): """Gets the server path of a resources content files. args: -- resourceid: the id of the hydroshare resource returns: -- server path the the resource content files """ path = utilities.find_resource_directory(resourceid) if path is not None: return os.path.join(path, resourceid, 'data/contents')
def create_hydroshare_resource(context, auth, title, hydroshare_host='www.hydroshare.org', hydroshare_port=None, use_https=False, resource_type='GenericResource', abstract=None, keywords=None, create_callback=None, verbose=False, outfp=sys.stderr): """ Create HydroShare resource of an EcohydroLib project @param context ecohydrolib.Context object @param auth hs_restclient.HydroShareAuth object @param title string representing the title of the resource @param hydroshare_host string representing DNS name of the HydroShare server in which to create the resource @param hydroshare_port int representing the TCP port of the HydroShare server @param use_https True if HTTPS should be used. Default: False @param resource_type string representing the HydroShare resource type that should be used to create the resource @param abstract string representing the abstract of the resource @param keywords list of strings representing the keywords to assign to the resource @param create_callback user-defined callable that takes as input a file size in bytes, and generates a callable to provide feedback to the user about the progress of the upload of resource_file. For more information, see: http://toolbelt.readthedocs.org/en/latest/uploading-data.html#monitoring-your-streaming-multipart-upload @param verbose Boolean True if detailed output information should be printed to outfp @param outfp File-like object to which verbose output should be printed @return string representing the ID of the newly created resource """ temp_dir = tempfile.mkdtemp() zip_filename = "{0}.zip".format(os.path.basename(context.projectDir)) zip_filepath = os.path.join(temp_dir, zip_filename) # Zip up the project for upload ... if verbose: outfp.write("Creating archive...") outfp.flush() zfile = zipfile.ZipFile(zip_filepath, 'w', zipfile.ZIP_DEFLATED, True) os.path.walk(context.projectDir, _addToZip, (context.projectDir, zfile)) zfile.close() if verbose: outfp.write("done\n") outfp.flush() # Make upload progress callback progress_callback = None if create_callback: s = os.stat(zip_filepath) progress_callback = create_callback(s.st_size) if verbose: outfp.write("Uploading to HydroShare...") outfp.flush() # This next line is needed for some text-based progress indicators # to properly render. outfp.write(" \nThis is a hack\r") outfp.flush() if hydroshare_host: hs = HydroShare(hostname=hydroshare_host, port=hydroshare_port, use_https=use_https, auth=auth) else: hs = HydroShare(port=hydroshare_port, use_https=use_https, auth=auth) resource_id = None try: resource_id = hs.createResource(resource_type, title, resource_file=zip_filepath, resource_filename=zip_filename, abstract=abstract, keywords=keywords, progress_callback=progress_callback) if verbose: url = _getResourceURL(resource_id, hydroshare_host, hs_port=hydroshare_port, use_https=use_https) outfp.write("\nNew resource created at:\n{0}\n".format(url)) outfp.flush() except Exception as e: raise e finally: shutil.rmtree(temp_dir) return resource_id