def authenticate_me(self, **kwargs): username = input("Enter Username: "******"Enter Password: "******"Either credentials invalid or unable to connect to HydroShare." ) return False
def authenticate_me(self, **kwargs): username = kwargs.get('username') or input("Enter Username: "******"Enter Password: "******"Either credentials invalid or unable to connect to {}.". format(self.name)) return False
def authenticate_me(self, **kwargs): connection_info = 'https://data.kitware.com/api/v1' username = kwargs['username'] password = kwargs['password'] try: gc = girder_client.GirderClient(apiUrl=connection_info) gc.authenticate(username, password) db = get_db() with db_session: p = db.Providers.select().filter(provider=self.name).first() provider_metadata = { 'provider': self.name, 'username': username, 'password': password, } if p is None: db.Providers(**provider_metadata) else: p.set(**provider_metadata) return True except: log.error( "Either credentials invalid or unable to connect to the Girder live server." ) return False
def stage_for_download(uris, options=None): """Apply download options before downloading Args: uris (string or list, Required): uris of datasets to stage for download options (dict or list of dicts, Optional, Default=None): options to be passed to quest.api.download function specified for each dataset If options is a dict, then apply same options to all datasets, else each dict in list is used for each respective dataset Returns: uris (list): staged dataset uids """ uris = listify(uris) display_name = None datasets = [] # TODO classify uris and ensure only datasets if not isinstance(options, list): options = [options] * len(uris) db = get_db() for dataset_uri, kwargs in zip(uris, options): if isinstance(kwargs, param.Parameterized): kwargs = dict(kwargs.get_param_values()) dataset_metadata = get_metadata(dataset_uri)[dataset_uri] parameter = kwargs.get('parameter') if kwargs else None parameter_name = parameter or 'no_parameter' if dataset_metadata['display_name'] == dataset_uri: catalog_entry = dataset_metadata['catalog_entry'] provider, service, _ = parse_service_uri(catalog_entry) display_name = '{0}-{1}-{2}'.format(provider, parameter_name, dataset_uri[:7]) quest_metadata = { 'display_name': display_name or dataset_metadata['display_name'], 'options': kwargs, 'status': DatasetStatus.STAGED, 'parameter': parameter } with db_session: dataset = db.Dataset[dataset_uri] dataset.set(**quest_metadata) datasets.append(dataset_uri) return datasets
def set_active_project(name): """Set active QUEST project. Args: name (string, Required): name of a project Returns: project (string): name of project currently set as active """ path = _get_projects_index_file() contents = util.read_yaml(path) if name not in contents['projects'].keys(): raise ValueError('Project %s does not exist' % name) contents.update({'active_project': name}) util.write_yaml(path, contents) get_db(active_db(), reconnect=True) # change active database return name
def _copy_dataset(dataset_metadata, collection_path, destination_collection_path): new_name = util.uuid('dataset') db = get_db() with db_session: db_metadata = db.Dataset[dataset_metadata['name']].to_dict() db_metadata.update(name=new_name) db.Dataset(**db_metadata) _update_dataset_file_location(shutil.copy2, db_metadata, collection_path, destination_collection_path) return new_name
def new_catalog_entry(geometry=None, geom_type=None, geom_coords=None, metadata=None): """Add a new entry to a catalog either a quest local catalog (table) or file. Args: geometry (string or Shapely.geometry.shape, optional, Default=None): well-known-text or Shapely shape representing the geometry of the catalog_entry. Alternatively `geom_type` and `geom_coords` can be passed. geom_type (string, Optional, Default=None): geometry type of catalog_entry (i.e. point/line/polygon) geom_coords (string or list, Optional, Default=None): geometric coordinates specified as valid geojson coordinates (i.e. a list of lists i.e. '[[-94.0, 23.2], [-94.2, 23.4] ...]' --------- OR --------- [[-94.0, 23.2], [-94.2, 23.4] ...] etc) metadata (dict, Optional, Default=None): optional metadata at the new catalog_entry Returns ------- uri (string): uri of newly created entry """ if geometry is None and geom_coords is not None and geom_type is not None: if isinstance(geom_coords, str): geom_coords = json.loads(geom_coords) geometry = shape({"coordinates": geom_coords, "type": geom_type}) if hasattr(geometry, 'wkt'): geometry = geometry.wkt catalog_id = util.uuid('catalog_entry') data = { 'service_id': catalog_id, 'geometry': geometry, 'metadata': metadata, } db = get_db() with db_session: db.QuestCatalog(**data) return construct_service_uri('quest', 'quest', catalog_id)
def new_collection(name, display_name=None, description=None, metadata=None): """Create a new collection. Create a new collection by creating a new folder in project directory and adding collection metadata in project database. Args: name (string, Required): Name of the collection used in all quest function calls,must be unique. Will also be the folder name of the collection display_name (string, Optional, Default=None): display name for collection description (string, Optional, Default=None): description of collection metadata (dict, Optional, Default=None): user defined metadata Returns: collection (dict) details of the newly created collection """ name = name.lower() collections = _load_collections() if name in collections: raise ValueError('Collection %s already exists' % name) if display_name is None: display_name = name if description is None: description = '' if metadata is None: metadata = {} path = os.path.join(_get_project_dir(), name) os.makedirs(path, exist_ok=True) db = get_db() with db_session: db.Collection(name=name, display_name=display_name, description=description, metadata=metadata) return _load_collection(name)
def get_auth_status(uri): """Check to see if a provider has been authenticated Args: uri (string, Required): uri of 'user service' Returns: True on success False on not authenticated """ db = get_db() with db_session: p = db.Providers.select().filter(provider=uri).first() if p is None: return False return True
def download_datasets(datasets, raise_on_error=False): """Download datasets that have been staged with stage_for_download. Args: datasets (string or list, Required): datasets to download raise_on_error (bool, Optional, Default=False): if True, if an error occurs raise an exception async: (bool, Optional, Default=False) if True, download in background Returns: status (dict): download status of datasets """ datasets = get_metadata(datasets, as_dataframe=True) if datasets.empty: return # filter out non download datasets datasets = datasets[datasets['source'] == DatasetSource.WEB_SERVICE] db = get_db() project_path = _get_project_dir() status = {} for idx, dataset in datasets.iterrows(): collection_path = os.path.join(project_path, dataset['collection']) catalog_entry = dataset["catalog_entry"] try: update_metadata(idx, quest_metadata={'status': DatasetStatus.PENDING}) kwargs = dataset['options'] or dict() all_metadata = download(catalog_entry, file_path=collection_path, dataset=idx, **kwargs) metadata = all_metadata.pop('metadata', None) quest_metadata = all_metadata quest_metadata.update({ 'status': DatasetStatus.DOWNLOADED, 'message': 'success', }) except Exception as e: if raise_on_error: raise quest_metadata = { 'status': DatasetStatus.FAILED_DOWNLOAD, 'message': str(e), } metadata = None status[idx] = quest_metadata['status'] quest_metadata.update({'metadata': metadata}) with db_session: dataset = db.Dataset[idx] dataset.set(**quest_metadata) return status
def new_dataset(catalog_entry, collection, source=None, display_name=None, description=None, file_path=None, metadata=None, name=None): """Create a new dataset in a collection. Args: catalog_entry (string, Required): catalog_entry uri collection (string, Required): name of collection to create dataset in source (string, Optional, Default=None): type of the dataset such as timeseries or raster display_name (string, Optional, Default=None): display name for dataset description (string, Optional, Default=None): description of dataset file_path (string, Optional, Default=None): path location to save new dataset's data metadata (dict, Optional, Default=None): user defined metadata name (dict, Optional, Default=None): optionally pass in a UUID starting with d as name, otherwise it will be generated Returns: uri (string): uid of dataset """ if collection not in get_collections(): raise ValueError("Collection {} does not exist".format(collection)) if not isinstance(catalog_entry, pd.DataFrame): catalog_entry = get_metadata(catalog_entry, as_dataframe=True) try: catalog_entry = catalog_entry['name'][0] except IndexError: raise ValueError('Entry {} dose not exist'.format(catalog_entry)) name = name or uuid('dataset') assert name.startswith('d') and is_uuid(name) if source is None: source = DatasetSource.USER if display_name is None: display_name = name if metadata is None: metadata = {} quest_metadata = { 'name': name, 'collection': collection, 'catalog_entry': catalog_entry, 'source': source, 'display_name': display_name, 'description': description, 'file_path': file_path, 'metadata': metadata, } if source == DatasetSource.WEB_SERVICE: quest_metadata.update({'status': DatasetStatus.NOT_STAGED}) db = get_db() with db_session: db.Dataset(**quest_metadata) return name
def delete(uris): """Delete metadata for resource(s) WARNING: deleting a collection deletes all associated datasets Args: uris (string, comma separated string or list of strings, Required): uri(s) of collection, and/or dataset to delete Returns: status (bool): True on success """ # if uri list is empty do nothing if not uris: return True # group uris by type grouped_uris = util.classify_uris(uris, as_dataframe=False, exclude=['services', 'publishers'], require_same_type=True) resource = list(grouped_uris)[0] uris = grouped_uris[resource] db = get_db() for uri in uris: if resource == 'collections': if uri not in get_collections(): logger.error('Collection does not exist: %s', uri) raise ValueError('Collection does not exists') with db_session: datasets = db.Dataset.select( lambda d: d.collection.name == uri) if datasets.count() > 0: datasets.delete() db.Collection[uri].delete() path = _get_project_dir() path = os.path.join(path, uri) if os.path.exists(path): logger.info('deleting all data under path: %s' % path) shutil.rmtree(path) if resource == 'datasets': with db_session: dataset = db.Dataset[uri] if dataset.source == 'derived': catalog_entry_datasets = select_datasets( lambda d: d.catalog_entry == dataset.catalog_entry) if len(catalog_entry_datasets) == 1: _, _, catalog_id = util.parse_service_uri( dataset.catalog_entry) db.QuestCatalog[catalog_id].delete() try: os.remove(dataset.file_path) except (OSError, TypeError): pass dataset.delete() return True
def _load_collections(): """load list of collections.""" db = get_db() with db_session: return {c.name: c.to_dict() for c in db.Collection.select(lambda c: c)}
def _load_collection(name): """load collection.""" db = get_db() with db_session: return db.Collection.select(lambda c: c.name == name).first().to_dict()