def runScript(id): auth = HydroShareAuthBasic(username='', password='') hs = HydroShare(auth=auth) hs.getResource(id, destination='/home/ubuntu/hydroshare_app/', unzip=True) subprocess.call("sudo cp " + str(id) + '/' + str(id) + '/data/contents/* /home/ubuntu/hydroshare_app/Data', shell=True) subprocess.call("sudo rm -r " + str(id), shell=True) process() #Locate the file with the .nam extension os.chdir('MODFLOW') for file in glob.glob("*.nam"): filename = file # Run the model subprocess.call("sudo ./mfnwt " + filename, shell=True) try: hs.deleteResourceFile(id, filename.split(".")[0] + '.list') except: pass #Upload to hydroshare hs.addResourceFile(id, filename.split(".")[0] + '.list') subprocess.call("sudo rm /home/ubuntu/hydroshare_app/MODFLOW/*.*", shell=True) subprocess.call("sudo rm -r /home/ubuntu/hydroshare_app/Scratch", shell=True) subprocess.call("sudo rm -r /home/ubuntu/hydroshare_app/Framework", shell=True) subprocess.call("sudo rm /home/ubuntu/hydroshare_app/Data/*", shell=True) return json.dumps(hs.getScienceMetadata(id))
def test_create_get_delete_resource_file(self): hs = HydroShare(prompt_auth=False) # Add res_id = '511debf8858a4ea081f78d66870da76c' fpath = 'mocks/data/another_resource_file.txt' fname = os.path.basename(fpath) resp = hs.addResourceFile(res_id, fpath) self.assertEqual(resp, res_id) # Get tmpdir = tempfile.mkdtemp() res_file = hs.getResourceFile(res_id, fname, destination=tmpdir) self.assertTrue(filecmp.cmp(res_file, fpath, shallow=False)) shutil.rmtree(tmpdir) # Delete delres = hs.deleteResourceFile(res_id, fname) self.assertEqual(delres, res_id)
def test_create_get_delete_resource_file(self): hs = HydroShare() # Add res_id = '511debf8858a4ea081f78d66870da76c' fpath = 'mocks/data/another_resource_file.txt' fname = os.path.basename(fpath) resp = hs.addResourceFile(res_id, fpath) self.assertEqual(resp, res_id) # Get tmpdir = tempfile.mkdtemp() res_file = hs.getResourceFile(res_id, fname, destination=tmpdir) self.assertTrue(filecmp.cmp(res_file, fpath, shallow=False)) shutil.rmtree(tmpdir) # Delete delres = hs.deleteResourceFile(res_id, fname) self.assertEqual(delres, res_id)
def upload_to_hs(uploadtype, modelname, resource_name, resource_abstract, resource_key): dbs = { 'zone': zone, 'mult': mult, 'pval': pval, 'bas6': bas6, 'dis': dis, 'disu': disu, 'bcf6': bcf6, 'lpf': lpf, 'hfb6': hfb6, 'chd': chd, 'fhb': fhb, 'wel': wel, 'mnw1': mnw1, 'mnw2': mnw2, 'mnwi': mnwi, 'drn': drn, 'rch': rch, 'evt': evt, 'ghb': ghb, 'gmg': gmg, 'lmt6': lmt6, 'lmt7': lmt7, 'riv': riv, 'str': str, 'swi2': swi2, 'pcg': pcg, 'pcgn': pcgn, 'nwt': nwt, 'pks': pks, 'sms': sms, 'sfr': sfr, 'lak': lak, 'gage': gage, 'sip': sip, 'sor': sor, 'de4': de4, 'oc': oc, 'uzf': uzf, 'upw': upw, 'sub': sub, 'swt': swt, 'hyd': hyd, 'hob': hob, 'vdf': vdf, 'vsc': vsc, 'drt': drt, 'pvl': pvl, 'ets': ets, 'bas': bas, 'nam': nam } hs = HydroShare() Session = app.get_persistent_store_database('primary_db', as_sessionmaker=True) session = Session() fileliststr = session.query(Model).filter( Model.displayname == modelname).first() filelist = [i for i in fileliststr.modelfiles.strip('{}').split(',')] mainid = fileliststr.id resourceid = fileliststr.resourceid if uploadtype == 'new': abstract = resource_abstract title = resource_name keywords = (i for i in resource_key.split(',')) rtype = 'ModelInstanceResource' new_resource_id = hs.createResource(rtype, title, keywords=keywords, abstract=abstract) for fi in filelist: parts = fi.split(".") ext_data = session.query( dbs[parts[1]]).filter(dbs[parts[1]].id == mainid).first().data if uploadtype == 'add': date = dt.now().strftime("%m-%d-%Y-%X") filename = "{}_{}.{}".format(parts[0], date, parts[1]) else: filename = fi if uploadtype == 'new': hs.addResourceFile(new_resource_id, ext_data, resource_filename=filename) elif uploadtype == 'overwrite': hs.deleteResourceFile(resourceid, filename) hs.addResourceFile(resourceid, ext_data, resource_filename=filename) else: hs.addResourceFile(resourceid, ext_data, resource_filename=filename) session.close() return_obj = {'success': True} return JsonResponse(return_obj)
class HydroShareUtility: def __init__(self): self.client = None # type: HydroShare self.auth = None self.user_info = None self.re_period = re.compile(r'(?P<tag_start>^start=)(?P<start>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_end>end=)' r'(?P<end>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_scheme>scheme=)(?P<scheme>.+$)', re.I) self.xml_ns = { 'dc': "http://purl.org/dc/elements/1.1/", 'dcterms': "http://purl.org/dc/terms/", 'hsterms': "http://hydroshare.org/terms/", 'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 'rdfs1': "http://www.w3.org/2001/01/rdf-schema#"} self.time_format = '%Y-%m-%dT%H:%M:%S' self.xml_coverage = 'start={start}; end={end}; scheme=W3C-DTF' def authenticate(self, username, password, client_id=None, client_secret=None): """ Authenticates access to allow read/write access to privileged resource_cache :param username: username for HydroShare.org :param password: password associated with username :param client_id: Client ID obtained from HydroShare :param client_secret: Client Secret provided by HydroShare :return: Returns true if authentication was successful, false otherwise """ if not all([username, password]): self.auth = None return False if client_id is not None and client_secret is not None: self.auth = HydroShareAuthOAuth2(client_id, client_secret, username=username, password=password) else: self.auth = HydroShareAuthBasic(username, password) try: self.client = HydroShare(auth=self.auth) # , verify=False) self.user_info = self.client.getUserInfo() return True except HydroShareException as e: # for incorrect username/password combinations print('Authentication failed: {}'.format(e)) except InvalidGrantError as e: # for failures when attempting to use OAuth2 print('Credentials could not be validated: {}'.format(e)) except InvalidClientError as e: print('Invalid client ID and/or client secret: {}'.format(e)) except Exception as e: print(e) self.auth = None return False def purgeDuplicateGamutFiles(self, resource_id, regex, confirm_delete=False): """ Removes all files that have a duplicate-style naming pattern (e.g. ' (1).csv', '_ASDFGJK9.csv' :param resource_id: Resource to inspect for duplicates :type resource_id: Resource object received from the HydroShare API client :param confirm_delete: If true, requires input that confirm file should be deleted :type confirm_delete: bool """ from collections import defaultdict re_breakdown = re.compile(regex, re.I) resource_files = self.getResourceFileList(resource_id) # type: [dict] duplicates_list = [] for remote_file in resource_files: url = remote_file.get('url', '') results = re_breakdown.match(url) # Check the file URL for expected patterns temp_dict = defaultdict(lambda: '') # type: dict if results: temp_dict['duplicated'] = results.groupdict() if len(temp_dict.get('duplicated', '')): # A non-duplicated file will match with length 0 duplicates_list.append(temp_dict) # Save the file name so we can remove it later for file_detail in duplicates_list: if not confirm_delete: delete_me = True else: user_answer = raw_input("Delete file {} [Y/n]: ".format(file_detail['name'])) if user_answer != 'N' and user_answer != 'n': delete_me = True else: delete_me = False if delete_me: self.client.deleteResourceFile(resource_id, file_detail['name']) print('Deleting file {}...'.format(file_detail['name'])) else: print('Skipping duplicate file {}...'.format(file_detail['name'])) def getResourceFileList(self, resource_id): """ :param resource_id: ID of resource for which to retrieve a file list :type resource_id: str :return: List of files in resource :rtype: list of str """ try: return list(self.client.getResourceFileList(resource_id)) except Exception as e: print('Error while fetching resource files {}'.format(e)) return [] def getAllResources(self): filtered_resources = {} owner = self.user_info['username'] if self.auth is None: raise HydroShareUtilityException("Cannot query resources without authentication") all_resources = self.client.resources(owner=owner) for resource in all_resources: resource_object = HydroShareResource(resource) filtered_resources[resource_object.id] = resource_object return filtered_resources def getMetadataForResource(self, resource): """ :type resource: HydroShareResource """ metadata = self.client.getScienceMetadata(resource.id) resource.title = metadata.get('title', '') resource.subjects = [item['value'] for item in metadata.get('subjects', [])] resource.abstract = metadata.get('description', '') if resource.abstract is None: resource.abstract = '' if 'funding_agencies' in metadata and len(metadata['funding_agencies']) > 0: funding_agency = metadata['funding_agencies'][0] resource.agency_url = funding_agency['agency_url'] if 'agency_url' in funding_agency else '' resource.funding_agency = funding_agency['agency_name'] if 'agency_name' in funding_agency else '' resource.award_number = funding_agency['award_number'] if 'award_number' in funding_agency else '' resource.award_title = funding_agency['award_title'] if 'award_title' in funding_agency else '' def updateResourceMetadata(self, resource): """ :type resource: HydroShareResource """ return self.client.updateScienceMetadata(resource.id, resource.get_metadata()) def _request(self, method, url, params=None, data=None, files=None, headers=None, stream=False): request = self.client.session.request(method, url, params=params, data=data, files=files, headers=headers, stream=stream) return request def requestAccessRules(self, resource): """ Get access rule for a resource. """ url = "{url_base}/resource/{pid}/sysmeta/".format(url_base=self.client.url_base, pid=resource.id) r = self._request('GET', url) if r.status_code != 200: raise Exception("Failed to get system metadata for resource: {}".format(resource.id)) data = r.json() resource.public = data.get('public', False) resource.shareable = data.get('shareable', False) def makePublic(self, resource, public=True): """ Makes a resource public or private :param resource: The resource you want to modify :param public: boolean value, True makes the resource public, False makes it private (wowzer) :return: None """ hs = HydroShare(auth=self.auth) res = hs.resource(resource.id).public(public) if res.status_code == 200 or res.status_code == 202: resource.public = public def updateKeywords(self, resource, keywords=None): if keywords is None: keywords = resource.keywords # remove leading/trailing whitespaces from keywords keywords = map(lambda x: x.strip(), keywords) url = "{url_base}/resource/{id}/scimeta/elements/".format(url_base=self.client.url_base, id=resource.id) subjects = [] for keyword in keywords: subjects.append({"value": keyword}) r = self.client.session.request('PUT', url, json={"subjects": subjects}) if r.status_code != 202: raise HydroShareException((url, 'PUT', r.status_code, keywords)) return r.json() def getFileListForResource(self, resource): resource.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource.id)] return resource.files def getFilesByResourceId(self, resource_id): return [os.path.basename(f['url']) for f in self.getResourceFileList(resource_id)] def filterResourcesByRegex(self, regex_string=None, owner=None, regex_flags=re.IGNORECASE): """ Apply a regex filter to all available resource_cache. Useful for finding GAMUT resource_cache :param owner: username of the owner of the resource :type owner: string :param regex_string: String to be used as the regex filter :param regex_flags: Flags to be passed to the regex search :return: A list of resource_cache that matched the filter """ filtered_resources = [] if owner is None: owner = self.user_info['username'] if self.auth is None: raise HydroShareUtilityException("Cannot query resources without authentication") all_resources = self.client.resources(owner=owner) regex_filter = re.compile(regex_string, regex_flags) for resource in all_resources: if regex_string is not None and regex_filter.search(resource['resource_title']) is None: continue resource_object = HydroShareResource(resource) resource_object.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource_object.id)] filtered_resources.append(resource_object) return filtered_resources def UploadFiles(self, files, resource): # type: ([str], HydroShareResource) -> bool if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: for csv_file in files: try: self.client.deleteResourceFile(resource.id, os.path.basename(csv_file)) except HydroShareNotFound: pass # except Exception as e: # if APP_SETTINGS.H2O_DEBUG and APP_SETTINGS.VERBOSE: # print 'File did not exist in remote: {}, {}'.format(type(e), e) if type(csv_file) != str: csv_file = str(csv_file) self.client.addResourceFile(resource.id, csv_file) msg = "File {} uploaded to remote {}".format(os.path.basename(csv_file), repr(resource)) print(msg) pub.sendMessage('logger', message=msg) except HydroShareException as e: print("Upload failed - could not complete upload to HydroShare due to exception: {}".format(e)) return False except KeyError as e: print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e)) return False return True def setResourcesAsPublic(self, resource_ids): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") for resource_id in resource_ids: try: print('Setting resource {} as public'.format(resource_id)) self.client.setAccessRules(resource_id, public=True) except HydroShareException as e: print("Access rule edit failed - could not set to public due to exception: {}".format(e)) except KeyError as e: print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e)) def deleteFilesInResource(self, resource): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: file_list = self.getResourceFileList(resource.id) for file_info in file_list: msg = 'Deleting resource file: {}'.format(os.path.basename(file_info['url'])) print(msg) pub.sendMessage('logger', message=msg) self.client.deleteResourceFile(resource.id, os.path.basename(file_info['url'])) except Exception as e: print('Could not delete files in resource {}\n{}'.format(resource.id, e)) def getResourceCoveragePeriod(self, resource_id): metadata = self.client.getScienceMetadata(resource_id) period_start = None period_end = None try: xml_tree = ElementTree.fromstring(metadata) description_node = xml_tree.find('rdf:Description', namespaces=self.xml_ns) coverage_node = description_node.find('dc:coverage', namespaces=self.xml_ns) period_node = coverage_node.find('dcterms:period', namespaces=self.xml_ns) value_node = period_node.find('rdf:value', namespaces=self.xml_ns) match = self.re_period.match(value_node.text) if match is not None: period_start = dateutil.parser.parse(match.group('start')) period_end = dateutil.parser.parse(match.group('end')) except Exception as e: print("Unable to find coverage data - encountered exception {}".format(e)) return period_start, period_end def deleteResource(self, resource_id, confirm=True): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: if confirm: user_input = raw_input('Are you sure you want to delete the resource {}? (y/N): '.format(resource_id)) if user_input.lower() != 'y': return print('Deleting resource {}'.format(resource_id)) self.client.deleteResource(resource_id) except Exception as e: print('Exception encountered while deleting resource {}: {}'.format(resource_id, e)) def createNewResource(self, resource): # type: (ResourceTemplate) -> HydroShareResource """ :param resource: :type resource: ResourceTemplate :return: :rtype: str """ if self.auth is None: raise HydroShareUtilityException("Cannot create resource without authentication") # http://hs-restclient.readthedocs.io/en/latest/ if resource is not None: metadata = [] fundingagency = {} fundingagency_attr_map = ( ('funding_agency', 'agency_name'), ('award_title', 'award_title'), ('award_number', 'award_number'), ('agency_url', 'agency_url') ) for reskey, fakey in fundingagency_attr_map: # reskey - resource attribute name # fakey - funding agency attribute name value = getattr(resource, reskey).strip() if len(value): fundingagency[fakey] = value if len(fundingagency.keys()): metadata.append({'fundingagency': fundingagency}) resource_id = self.client.createResource(resource_type='CompositeResource', title=resource.title, abstract=resource.abstract, keywords=resource.keywords, metadata=json.dumps(metadata, encoding='ascii')) hs_resource = HydroShareResource({'resource_id': resource_id}) self.getMetadataForResource(hs_resource) return hs_resource return None