def load_dataset( self, datastore: Datastore, filestore: Filestore, file_id: Optional[str] = None, url: Optional[str] = None, detect_headers: bool = True, infer_types: bool = True, load_format: str = 'csv', options: List[Dict[str, str]] = [], username: str = None, password: str = None, resources: Optional[Dict[str, Any]] = None, reload: bool = False, human_readable_name: Optional[str] = None, proposed_schema: List[Tuple[str, str]] = []) -> VizualApiResult: """Create (or load) a new dataset from a given file or Uri. It is guaranteed that either the file identifier or the url are not None but one of them will be None. The user name and password may only be given if an url is given. The resources refer to any resoures (e.g., file identifier) that have been generated by a previous execution of the respective task. This allows to associate an identifier with a downloaded file to avoid future downloads (unless the reload flag is True). ^--- Vistrails will automatically skip re-execution, so the only reason that we'd re-execute the cell is if the user manually asked us to. If that's the case, we should actually reload the file (e.g., because we may be reloading with different parameters). Parameters ---------- datastore : Datastore to retireve and update datasets filestore: Filestore to retrieve uploaded datasets file_id: Identifier for a file in an associated filestore url: Identifier for a web resource detect_headers: Detect column names in loaded file if True infer_types: Infer column types for loaded dataset if True load_format: Format identifier options: Additional options for Mimirs load command username: User name for authentication when accessing restricted resources password: Password for authentication when accessing restricted resources resources: Dictionary of additional resources (i.e., key,value pairs) that were generated during a previous execution of the associated module reload: If set to false, avoid reloading the data if possible. human_readable_name: A user-facing name for this table proposed_schema: A list of name/type pairs that will override the inferred column names/types if present Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ dataset = None f_handle = None result_resources = dict() if url is not None: if (debug_is_on()): print("LOAD URL: {}".format(url)) # If the same url has been previously used to generate a dataset # we do not need to download the file and re-create the dataset. if not reload and not resources is None and base.RESOURCE_URL in resources and base.RESOURCE_DATASET in resources: # Check if the previous download matches the given Uri if resources[base.RESOURCE_URL] == url: ds_id = resources[base.RESOURCE_DATASET] if (debug_is_on()): print(" ... re-using existing dataset {}".format( ds_id)) dataset = datastore.get_dataset(ds_id) result_resources[base.RESOURCE_URL] = url elif file_id is not None: if debug_is_on(): print("LOAD FILE: {}".format(file_id)) # If the same file has been previously used to generate a dataset # we do not need to re-create it. if (not reload) and (resources is not None) and ( base.RESOURCE_FILEID in resources) and (base.RESOURCE_DATASET in resources): if resources[base.RESOURCE_FILEID] == file_id: ds_id = resources[base.RESOURCE_DATASET] # if(debug_is_on()): print(" ... re-using existing dataset {}".format(ds_id)) dataset = datastore.get_dataset(ds_id) print("DATASET: {}".format(dataset)) # If the dataset is None we will load the dataset from an uploaded # file. Need to get the file handle for the file here. if dataset is None: print("getting file") f_handle = filestore.get_file(file_id) if (f_handle is None): raise ValueError( "The uploaded file got deleted, try re-uploading.") result_resources[base.RESOURCE_FILEID] = file_id else: raise ValueError('no source identifier given for load') # If the dataset is still None at this point we need to call the # load_dataset method of the datastore to load it. if dataset is None: if (url is None and f_handle is None): raise ValueError("Need an URL or an Uploaded File to load") assert (isinstance(datastore, MimirDatastore)) if (debug_is_on()): print(" ... loading dataset {} / {}".format(url, f_handle)) dataset = datastore.load_dataset( f_handle=f_handle, url=url, detect_headers=detect_headers, infer_types=infer_types, load_format=load_format, human_readable_name=human_readable_name, options=options, proposed_schema=proposed_schema) result_resources[base.RESOURCE_DATASET] = dataset.identifier return VizualApiResult(dataset=dataset, resources=result_resources) # Ensure that file name references a previously uploaded file. f_handle = self.fileserver.get_file(file_id) if f_handle is None: raise ValueError('unknown file \'' + file_id + '\'')
def load_dataset( self, datastore: Datastore, filestore: Filestore, file_id: Optional[str] = None, url: Optional[str] = None, detect_headers: bool = True, infer_types: bool = True, load_format: str = 'csv', options: List[Dict[str, str]] = [], username: str = None, password: str = None, resources: Optional[Dict[str, Any]] = None, reload: bool = False, human_readable_name: Optional[str] = None, proposed_schema: List[Tuple[str, str]] = []) -> VizualApiResult: """Create (or load) a new dataset from a given file or Uri. It is guaranteed that either the file identifier or the url are not None but one of them will be None. The user name and password may only be given if an url is given. The resources refer to any resoures (e.g., file identifier) that have been generated by a previous execution of the respective task. This allows to associate an identifier with a downloaded file to avoid future downloads (unless the reload flag is True). Parameters ---------- datastore : vizier.datastore.fs.base.FileSystemDatastore Datastore to retireve and update datasets filestore: vizier.filestore.Filestore Filestore to retrieve uploaded datasets file_id: string, optional Identifier for a file in an associated filestore url: string, optional Identifier for a web resource detect_headers: bool, optional Detect column names in loaded file if True infer_types: bool, optional Infer column types for loaded dataset if True load_format: string, optional Format identifier options: list, optional Additional options for Mimirs load command username: string, optional User name for authentication when accessing restricted resources password: string, optional Password for authentication when accessing restricted resources resources: dict, optional Dictionary of additional resources (i.e., key,value pairs) that were generated during a previous execution of the associated module reload: bool, optional Flag to force download of a remote resource even if it was downloaded previously Returns ------- vizier.engine.packages.vizual.api.VizualApiResult """ dataset = None result_resources = dict() if url is not None: # If the same url has been previously used to generate a dataset # we do not need to download the file and re-create the dataset. if not reload and not resources is None and base.RESOURCE_URL in resources and base.RESOURCE_DATASET in resources: # Check if the previous download matches the given Uri if resources[base.RESOURCE_URL] == url: ds_id = resources[base.RESOURCE_DATASET] dataset = datastore.get_dataset(ds_id) # If dataset is still None we need to create a new dataset by # downloading the given Uri if dataset is None: assert (isinstance(datastore, FileSystemDatastore)) dataset = datastore.download_dataset( url=url, username=username, password=password, ) result_resources[base.RESOURCE_URL] = url else: # either url or file_id must not be None assert (file_id is not None) # If the same file has been previously used to generate a dataset # we do not need to re-create it. if not resources is None and base.RESOURCE_FILEID in resources and base.RESOURCE_DATASET in resources: if resources[base.RESOURCE_FILEID] == file_id: ds_id = resources[base.RESOURCE_DATASET] dataset = datastore.get_dataset(ds_id) # If dataset is still None we need to create a new dataset from the # specified file if dataset is None: dataset = datastore.load_dataset( f_handle=filestore.get_file(file_id), proposed_schema=proposed_schema) result_resources[base.RESOURCE_FILEID] = file_id # Ensure that the dataset is not None at this point if dataset is None: raise ValueError('unknown file or resource') result_resources[base.RESOURCE_DATASET] = dataset.identifier return VizualApiResult(dataset=dataset, resources=result_resources)