def download_epanet_model(request): return_obj = {'success': False, 'message': None, 'results': "", 'name': ""} if request.is_ajax() and request.method == 'GET': if not request.GET.get('model_id'): return_obj['message'] = message_template_param_unfilled.format( param='model_id') else: model_id = request.GET['model_id'] try: hs = get_oauth_hs(request) except: hs = HydroShare() for model_file in hs.getResourceFileList(model_id): model_url = model_file['url'] model_name = model_url[model_url.find('contents/') + 9:] model = "" for line in hs.getResourceFile(model_id, model_name): model += line.decode("utf-8") return_obj['name'] = model_name return_obj['results'] = model return_obj['success'] = True else: return_obj['message'] = message_template_wrong_req_method.format( method="POST") return JsonResponse(return_obj)
def test_get_resource_file_list(self): hs = HydroShare(prompt_auth=False) res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c') for (i, r) in enumerate(res_list): self.assertEqual(r['url'], self.urls[i]) self.assertEqual(r['size'], self.sizes[i]) self.assertEqual(r['content_type'], self.content_types[i])
def test_get_resource_file_list(self): hs = HydroShare() res_list = hs.getResourceFileList('511debf8858a4ea081f78d66870da76c') for (i, r) in enumerate(res_list): self.assertEquals(r['url'], self.urls[i]) self.assertEquals(r['size'], self.sizes[i]) self.assertEquals(r['content_type'], self.content_types[i])
def loadresource(request): res_id = request.GET['res_id'] hs = HydroShare() resource_md = hs.getSystemMetadata(res_id) resource_files = hs.getResourceFileList(res_id).list[0] context = { 'path': request.path, 'method': request.method, 'query': request.META['QUERY_STRING'], 'res_id': res_id, 'resource_title': resource_md["resource_title"], 'resource_files': resource_files, } return render(request, 'colab_launcher/loadresource.html', context)
def save_hs_to_favorites(resourceid, displayname, modeltype): dbs = { 'zone': zone, 'mult': mult, 'pval': pval, 'bas6': bas6, 'dis': dis, 'disu': disu, 'bcf6': bcf6, 'lpf': lpf, 'hfb6': hfb6, 'chd': chd, 'fhb': fhb, 'wel': wel, 'mnw1': mnw1, 'mnw2': mnw2, 'mnwi': mnwi, 'drn': drn, 'rch': rch, 'evt': evt, 'ghb': ghb, 'gmg': gmg, 'lmt6': lmt6, 'lmt7': lmt7, 'riv': riv, 'str': str, 'swi2': swi2, 'pcg': pcg, 'pcgn': pcgn, 'nwt': nwt, 'pks': pks, 'sms': sms, 'sfr': sfr, 'lak': lak, 'gage': gage, 'sip': sip, 'sor': sor, 'de4': de4, 'oc': oc, 'uzf': uzf, 'upw': upw, 'sub': sub, 'swt': swt, 'hyd': hyd, 'hob': hob, 'vdf': vdf, 'vsc': vsc, 'drt': drt, 'pvl': pvl, 'ets': ets, 'bas': bas, 'nam': nam, } Session = app.get_persistent_store_database('primary_db', as_sessionmaker=True) session = Session() hs = HydroShare() app_dir = app.get_app_workspace().path resourcelist = hs.getResourceFileList(resourceid) filelist = [] for resource in resourcelist: url = resource['url'].split("/") fname = url[-1] hs.getResourceFile(resourceid, fname, destination=app_dir) filelist.append(fname) json.dumps(filelist) fav = Model(resourceid=resourceid, displayname=displayname, modeltype=modeltype, modelfiles=filelist) # Add the model to the session, commit, and close session.add(fav) model = session.query(Model).filter( Model.displayname == displayname).first() mainid = model.id for fi in filelist: ext = fi.split(".")[1] filepath = os.path.join(app.get_app_workspace().path, fi) with open(filepath, 'r') as myfile: data = myfile.read() json.dumps(data) tbl = dbs[ext](data=data, ) # Add the model to the session, commit, and close session.add(tbl) session.commit() setattr(model, ext + 'id', tbl.id) session.commit() os.remove(filepath) session.close() return
class HydroShareUtility: def __init__(self): self.client = None # type: HydroShare self.auth = None self.user_info = None self.re_period = re.compile(r'(?P<tag_start>^start=)(?P<start>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_end>end=)' r'(?P<end>[0-9-]{10}T[0-9:]{8}).{2}(?P<tag_scheme>scheme=)(?P<scheme>.+$)', re.I) self.xml_ns = { 'dc': "http://purl.org/dc/elements/1.1/", 'dcterms': "http://purl.org/dc/terms/", 'hsterms': "http://hydroshare.org/terms/", 'rdf': "http://www.w3.org/1999/02/22-rdf-syntax-ns#", 'rdfs1': "http://www.w3.org/2001/01/rdf-schema#"} self.time_format = '%Y-%m-%dT%H:%M:%S' self.xml_coverage = 'start={start}; end={end}; scheme=W3C-DTF' def authenticate(self, username, password, client_id=None, client_secret=None): """ Authenticates access to allow read/write access to privileged resource_cache :param username: username for HydroShare.org :param password: password associated with username :param client_id: Client ID obtained from HydroShare :param client_secret: Client Secret provided by HydroShare :return: Returns true if authentication was successful, false otherwise """ if not all([username, password]): self.auth = None return False if client_id is not None and client_secret is not None: self.auth = HydroShareAuthOAuth2(client_id, client_secret, username=username, password=password) else: self.auth = HydroShareAuthBasic(username, password) try: self.client = HydroShare(auth=self.auth) # , verify=False) self.user_info = self.client.getUserInfo() return True except HydroShareException as e: # for incorrect username/password combinations print('Authentication failed: {}'.format(e)) except InvalidGrantError as e: # for failures when attempting to use OAuth2 print('Credentials could not be validated: {}'.format(e)) except InvalidClientError as e: print('Invalid client ID and/or client secret: {}'.format(e)) except Exception as e: print(e) self.auth = None return False def purgeDuplicateGamutFiles(self, resource_id, regex, confirm_delete=False): """ Removes all files that have a duplicate-style naming pattern (e.g. ' (1).csv', '_ASDFGJK9.csv' :param resource_id: Resource to inspect for duplicates :type resource_id: Resource object received from the HydroShare API client :param confirm_delete: If true, requires input that confirm file should be deleted :type confirm_delete: bool """ from collections import defaultdict re_breakdown = re.compile(regex, re.I) resource_files = self.getResourceFileList(resource_id) # type: [dict] duplicates_list = [] for remote_file in resource_files: url = remote_file.get('url', '') results = re_breakdown.match(url) # Check the file URL for expected patterns temp_dict = defaultdict(lambda: '') # type: dict if results: temp_dict['duplicated'] = results.groupdict() if len(temp_dict.get('duplicated', '')): # A non-duplicated file will match with length 0 duplicates_list.append(temp_dict) # Save the file name so we can remove it later for file_detail in duplicates_list: if not confirm_delete: delete_me = True else: user_answer = raw_input("Delete file {} [Y/n]: ".format(file_detail['name'])) if user_answer != 'N' and user_answer != 'n': delete_me = True else: delete_me = False if delete_me: self.client.deleteResourceFile(resource_id, file_detail['name']) print('Deleting file {}...'.format(file_detail['name'])) else: print('Skipping duplicate file {}...'.format(file_detail['name'])) def getResourceFileList(self, resource_id): """ :param resource_id: ID of resource for which to retrieve a file list :type resource_id: str :return: List of files in resource :rtype: list of str """ try: return list(self.client.getResourceFileList(resource_id)) except Exception as e: print('Error while fetching resource files {}'.format(e)) return [] def getAllResources(self): filtered_resources = {} owner = self.user_info['username'] if self.auth is None: raise HydroShareUtilityException("Cannot query resources without authentication") all_resources = self.client.resources(owner=owner) for resource in all_resources: resource_object = HydroShareResource(resource) filtered_resources[resource_object.id] = resource_object return filtered_resources def getMetadataForResource(self, resource): """ :type resource: HydroShareResource """ metadata = self.client.getScienceMetadata(resource.id) resource.title = metadata.get('title', '') resource.subjects = [item['value'] for item in metadata.get('subjects', [])] resource.abstract = metadata.get('description', '') if resource.abstract is None: resource.abstract = '' if 'funding_agencies' in metadata and len(metadata['funding_agencies']) > 0: funding_agency = metadata['funding_agencies'][0] resource.agency_url = funding_agency['agency_url'] if 'agency_url' in funding_agency else '' resource.funding_agency = funding_agency['agency_name'] if 'agency_name' in funding_agency else '' resource.award_number = funding_agency['award_number'] if 'award_number' in funding_agency else '' resource.award_title = funding_agency['award_title'] if 'award_title' in funding_agency else '' def updateResourceMetadata(self, resource): """ :type resource: HydroShareResource """ return self.client.updateScienceMetadata(resource.id, resource.get_metadata()) def _request(self, method, url, params=None, data=None, files=None, headers=None, stream=False): request = self.client.session.request(method, url, params=params, data=data, files=files, headers=headers, stream=stream) return request def requestAccessRules(self, resource): """ Get access rule for a resource. """ url = "{url_base}/resource/{pid}/sysmeta/".format(url_base=self.client.url_base, pid=resource.id) r = self._request('GET', url) if r.status_code != 200: raise Exception("Failed to get system metadata for resource: {}".format(resource.id)) data = r.json() resource.public = data.get('public', False) resource.shareable = data.get('shareable', False) def makePublic(self, resource, public=True): """ Makes a resource public or private :param resource: The resource you want to modify :param public: boolean value, True makes the resource public, False makes it private (wowzer) :return: None """ hs = HydroShare(auth=self.auth) res = hs.resource(resource.id).public(public) if res.status_code == 200 or res.status_code == 202: resource.public = public def updateKeywords(self, resource, keywords=None): if keywords is None: keywords = resource.keywords # remove leading/trailing whitespaces from keywords keywords = map(lambda x: x.strip(), keywords) url = "{url_base}/resource/{id}/scimeta/elements/".format(url_base=self.client.url_base, id=resource.id) subjects = [] for keyword in keywords: subjects.append({"value": keyword}) r = self.client.session.request('PUT', url, json={"subjects": subjects}) if r.status_code != 202: raise HydroShareException((url, 'PUT', r.status_code, keywords)) return r.json() def getFileListForResource(self, resource): resource.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource.id)] return resource.files def getFilesByResourceId(self, resource_id): return [os.path.basename(f['url']) for f in self.getResourceFileList(resource_id)] def filterResourcesByRegex(self, regex_string=None, owner=None, regex_flags=re.IGNORECASE): """ Apply a regex filter to all available resource_cache. Useful for finding GAMUT resource_cache :param owner: username of the owner of the resource :type owner: string :param regex_string: String to be used as the regex filter :param regex_flags: Flags to be passed to the regex search :return: A list of resource_cache that matched the filter """ filtered_resources = [] if owner is None: owner = self.user_info['username'] if self.auth is None: raise HydroShareUtilityException("Cannot query resources without authentication") all_resources = self.client.resources(owner=owner) regex_filter = re.compile(regex_string, regex_flags) for resource in all_resources: if regex_string is not None and regex_filter.search(resource['resource_title']) is None: continue resource_object = HydroShareResource(resource) resource_object.files = [os.path.basename(f['url']) for f in self.getResourceFileList(resource_object.id)] filtered_resources.append(resource_object) return filtered_resources def UploadFiles(self, files, resource): # type: ([str], HydroShareResource) -> bool if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: for csv_file in files: try: self.client.deleteResourceFile(resource.id, os.path.basename(csv_file)) except HydroShareNotFound: pass # except Exception as e: # if APP_SETTINGS.H2O_DEBUG and APP_SETTINGS.VERBOSE: # print 'File did not exist in remote: {}, {}'.format(type(e), e) if type(csv_file) != str: csv_file = str(csv_file) self.client.addResourceFile(resource.id, csv_file) msg = "File {} uploaded to remote {}".format(os.path.basename(csv_file), repr(resource)) print(msg) pub.sendMessage('logger', message=msg) except HydroShareException as e: print("Upload failed - could not complete upload to HydroShare due to exception: {}".format(e)) return False except KeyError as e: print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e)) return False return True def setResourcesAsPublic(self, resource_ids): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") for resource_id in resource_ids: try: print('Setting resource {} as public'.format(resource_id)) self.client.setAccessRules(resource_id, public=True) except HydroShareException as e: print("Access rule edit failed - could not set to public due to exception: {}".format(e)) except KeyError as e: print('Incorrectly formatted arguments given. Expected key not found: {}'.format(e)) def deleteFilesInResource(self, resource): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: file_list = self.getResourceFileList(resource.id) for file_info in file_list: msg = 'Deleting resource file: {}'.format(os.path.basename(file_info['url'])) print(msg) pub.sendMessage('logger', message=msg) self.client.deleteResourceFile(resource.id, os.path.basename(file_info['url'])) except Exception as e: print('Could not delete files in resource {}\n{}'.format(resource.id, e)) def getResourceCoveragePeriod(self, resource_id): metadata = self.client.getScienceMetadata(resource_id) period_start = None period_end = None try: xml_tree = ElementTree.fromstring(metadata) description_node = xml_tree.find('rdf:Description', namespaces=self.xml_ns) coverage_node = description_node.find('dc:coverage', namespaces=self.xml_ns) period_node = coverage_node.find('dcterms:period', namespaces=self.xml_ns) value_node = period_node.find('rdf:value', namespaces=self.xml_ns) match = self.re_period.match(value_node.text) if match is not None: period_start = dateutil.parser.parse(match.group('start')) period_end = dateutil.parser.parse(match.group('end')) except Exception as e: print("Unable to find coverage data - encountered exception {}".format(e)) return period_start, period_end def deleteResource(self, resource_id, confirm=True): if self.auth is None: raise HydroShareUtilityException("Cannot modify resources without authentication") try: if confirm: user_input = raw_input('Are you sure you want to delete the resource {}? (y/N): '.format(resource_id)) if user_input.lower() != 'y': return print('Deleting resource {}'.format(resource_id)) self.client.deleteResource(resource_id) except Exception as e: print('Exception encountered while deleting resource {}: {}'.format(resource_id, e)) def createNewResource(self, resource): # type: (ResourceTemplate) -> HydroShareResource """ :param resource: :type resource: ResourceTemplate :return: :rtype: str """ if self.auth is None: raise HydroShareUtilityException("Cannot create resource without authentication") # http://hs-restclient.readthedocs.io/en/latest/ if resource is not None: metadata = [] fundingagency = {} fundingagency_attr_map = ( ('funding_agency', 'agency_name'), ('award_title', 'award_title'), ('award_number', 'award_number'), ('agency_url', 'agency_url') ) for reskey, fakey in fundingagency_attr_map: # reskey - resource attribute name # fakey - funding agency attribute name value = getattr(resource, reskey).strip() if len(value): fundingagency[fakey] = value if len(fundingagency.keys()): metadata.append({'fundingagency': fundingagency}) resource_id = self.client.createResource(resource_type='CompositeResource', title=resource.title, abstract=resource.abstract, keywords=resource.keywords, metadata=json.dumps(metadata, encoding='ascii')) hs_resource = HydroShareResource({'resource_id': resource_id}) self.getMetadataForResource(hs_resource) return hs_resource return None