def commit(self, message: str = "") -> str: """| Saves the current state of the dataset and returns the commit id. Checks out automatically to an auto branch if the current commit is not the head of the branch Only saves the dataset without any version control information if the dataset was created before Hub v1.3.0 Parameters ---------- message: str, optional The commit message to store along with the commit """ if self._commit_id is None: warnings.warn( "This dataset was created before version control, it does not support it. commit will behave same as flush" ) self.flush() elif "r" in self._mode: raise ReadModeException("commit") else: self._auto_checkout() stored_commit_id = self._commit_id self._commit_id = generate_hash() new_node = VersionNode(self._commit_id, self._branch) self._version_node.insert(new_node, message) self._version_node = new_node self._branch_node_map[self._branch] = new_node self._commit_node_map[self._commit_id] = new_node self.flush() return stored_commit_id
def checkout(self, address: str, create: bool = False) -> str: """| Changes the state of the dataset to the address mentioned. Creates a new branch if address isn't a commit id or branch name and create is True. Always checks out to the head of a branch if the address specified is a branch name. Returns the commit id of the commit that has been switched to. Only works if dataset was created on or after Hub v1.3.0 Parameters ---------- address: str The branch name or commit id to checkout to create: bool, optional Specifying create as True creates a new branch from the current commit if the address isn't an existing branch name or commit id """ if self._commit_id is None: raise VersioningNotSupportedException("checkout") self.flush() if address in self._branch_node_map.keys(): self._branch = address self._version_node = self._branch_node_map[address] self._commit_id = self._version_node.commit_id elif address in self._commit_node_map.keys(): self._version_node = self._commit_node_map[address] self._branch = self._version_node.branch self._commit_id = self._version_node.commit_id elif create: if "r" in self._mode: raise ReadModeException("checkout to create new branch") self._branch = address new_commit_id = generate_hash() new_node = VersionNode(new_commit_id, self._branch) if not self._version_node.children: for key in self.keys: self._tensors[key].fs_map.copy_all_chunks( self._commit_id, new_commit_id ) if self._version_node.parent is not None: self._version_node.parent.insert( new_node, f"switched to new branch {address}" ) else: self._version_node.insert(new_node, f"switched to new branch {address}") self._version_node = new_node self._commit_id = new_commit_id self._branch_node_map[self._branch] = new_node self._commit_node_map[self._commit_id] = new_node self.flush() else: raise AddressNotFound(address) return self._commit_id
def __init__( self, url: str, mode: str = None, shape=None, schema=None, token=None, fs=None, fs_map=None, meta_information=dict(), cache: int = defaults.DEFAULT_MEMORY_CACHE_SIZE, storage_cache: int = defaults.DEFAULT_STORAGE_CACHE_SIZE, lock_cache=True, tokenizer=None, lazy: bool = True, public: bool = True, name: str = None, ): """| Open a new or existing dataset for read/write Parameters ---------- url: str The url where dataset is located/should be created mode: str, optional (default to "a") Python way to tell whether dataset is for read or write (ex. "r", "w", "a") shape: tuple, optional Tuple with (num_samples,) format, where num_samples is number of samples schema: optional Describes the data of a single sample. Hub schemas are used for that Required for 'a' and 'w' modes token: str or dict, optional If url is refering to a place where authorization is required, token is the parameter to pass the credentials, it can be filepath or dict fs: optional fs_map: optional meta_information: optional ,give information about dataset in a dictionary. cache: int, optional Size of the memory cache. Default is 64MB (2**26) if 0, False or None, then cache is not used storage_cache: int, optional Size of the storage cache. Default is 256MB (2**28) if 0, False or None, then storage cache is not used lock_cache: bool, optional Lock the cache for avoiding multiprocessing errors lazy: bool, optional Setting this to False will stop lazy computation and will allow items to be accessed without .compute() public: bool, optional only applicable if using hub storage, ignored otherwise setting this to False allows only the user who created it to access the dataset and the dataset won't be visible in the visualizer to the public name: str, optional only applicable when using hub storage, this is the name that shows up on the visualizer """ shape = norm_shape(shape) if len(shape) != 1: raise ShapeLengthException() storage_cache = norm_cache(storage_cache) if cache else 0 cache = norm_cache(cache) schema: SchemaDict = featurify(schema) if schema else None self._url = url self._token = token self.tokenizer = tokenizer self.lazy = lazy self._name = name self._fs, self._path = ((fs, url) if fs else get_fs_and_path( self._url, token=token, public=public)) self._cache = cache self._storage_cache = storage_cache self.lock_cache = lock_cache self.verison = "1.x" mode = self._get_mode(mode, self._fs) self._mode = mode needcreate = self._check_and_prepare_dir() fs_map = fs_map or get_storage_map(self._fs, self._path, cache, lock=lock_cache, storage_cache=storage_cache) self._fs_map = fs_map self._meta_information = meta_information self.username = None self.dataset_name = None if not needcreate: self.meta = json.loads(fs_map[defaults.META_FILE].decode("utf-8")) self._name = self.meta.get("name") or None self._shape = tuple(self.meta["shape"]) self._schema = hub.schema.deserialize.deserialize( self.meta["schema"]) self._meta_information = self.meta.get("meta_info") or dict() self._flat_tensors = tuple(flatten(self._schema)) try: version_info = pickle.loads(fs_map[defaults.VERSION_INFO]) self._branch_node_map = version_info.get("branch_node_map") self._commit_node_map = version_info.get("commit_node_map") self._chunk_commit_map = version_info.get("chunk_commit_map") if not (self._branch_node_map and self._commit_node_map and self._chunk_commit_map): raise InvalidVersionInfoException() self._branch = "master" self._version_node = self._branch_node_map[self._branch] self._commit_id = self._version_node.commit_id except KeyError: self._commit_id = None self._branch = None self._version_node = None self._branch_node_map = None self._commit_node_map = None self._chunk_commit_map = None except InvalidVersionInfoException: self._commit_id = None self._branch = None self._version_node = None self._branch_node_map = None self._commit_node_map = None self._chunk_commit_map = None self._tensors = dict(self._open_storage_tensors()) if shape != (None, ) and shape != self._shape: raise TypeError( f"Shape in metafile [{self._shape}] and shape in arguments [{shape}] are !=, use mode='w' to overwrite dataset" ) if schema is not None and sorted(schema.dict_.keys()) != sorted( self._schema.dict_.keys()): raise TypeError( "Schema in metafile and schema in arguments do not match, use mode='w' to overwrite dataset" ) else: if shape[0] is None: raise ShapeArgumentNotFoundException() if schema is None: raise SchemaArgumentNotFoundException() try: if shape is None: raise ShapeArgumentNotFoundException() if schema is None: raise SchemaArgumentNotFoundException() self._schema = schema self._shape = tuple(shape) self.meta = self._store_meta() self._meta_information = meta_information self._flat_tensors = tuple(flatten(self.schema)) self._commit_id = generate_hash() self._branch = "master" self._version_node = VersionNode(self._commit_id, self._branch) self._branch_node_map = {self._branch: self._version_node} self._commit_node_map = {self._commit_id: self._version_node} self._chunk_commit_map = { path: defaultdict(set) for schema, path in self._flat_tensors } self._tensors = dict(self._generate_storage_tensors()) except Exception as e: try: self.close() except Exception: pass self._fs.rm(self._path, recursive=True) logger.error("Deleting the dataset " + traceback.format_exc() + str(e)) raise self.flush() self.indexes = list(range(self._shape[0])) if self._path.startswith("s3://snark-hub-dev/" ) or self._path.startswith("s3://snark-hub/"): subpath = self._path[5:] spl = subpath.split("/") if len(spl) < 4: raise ValueError("Invalid Path for dataset") self.username = spl[-2] self.dataset_name = spl[-1] if needcreate: HubControlClient().create_dataset_entry(self.username, self.dataset_name, self.meta, public=public)