def __init__(self, parent, items, xIsListOrTuple, yIsListOrTuple): self.parent = parent self.items = items self.xIsListOrTuple = xIsListOrTuple self.yIsListOrTuple = yIsListOrTuple inherit_dataset_params(parent, self) if hasattr(parent, "name"): self.origName = getattr(parent, "name")
def __init__(self, parent): super().__init__() self.expectsItem = False self.parent = parent if isinstance(parent, PreprocessedDataSet) or isinstance( parent, DataSet): self._parent_supports_target = True else: self._parent_supports_target = False inherit_dataset_params(parent, self)
def wrapper(input, **kwargs): f = func if inspect.isclass(f): n = f.__name__ f = f(**kwargs) f.__name__ = n kwargs = {} if isinstance(input, CompositeDataSet): components = list( map(lambda x: PreprocessedDataSet(x, f, expectsItem, **kwargs), input.components)) compositeDS = CompositeDataSet(components) inherit_dataset_params(input, compositeDS) if hasattr(input, "name"): compositeDS.name = input.name + f.__name__ + str( _sorted_args(kwargs)) compositeDS.origName = compositeDS.name return compositeDS else: return PreprocessedDataSet(input, f, expectsItem, **kwargs)
def __init__(self, parent, items): self.parent = parent self.items = items inherit_dataset_params(parent, self)
def ccc1(input): try: if hasattr(context.context, "no_cache"): return input name = "data" id = "dataset" l = len(input) if hasattr(input, "name"): id = getattr(input, "name") name = id.replace("{", "").replace("[", "").replace( "/", "").replace("\\", "").replace("]", "").replace( "}", "").replace(" ", "").replace(",", "").replace( "\'", "").replace(":", "") name = get_cache_dir() + name if name in storage: r = storage[name] inherit_dataset_params(input, r) return r if isinstance(input, CompositeDataSet): components = list(map(lambda x: ccc1(x), input.components)) compositeDS = CompositeDataSet(components) inherit_dataset_params(input, compositeDS) if hasattr(input, "name"): compositeDS.origName = input.name return compositeDS i0 = input[0] i0x = i0.x xIsList = isinstance(i0x, list) i0y = i0.y if not xIsList: shapeX = np.concatenate(([l], i0x.shape)) else: shapeX = list( map(lambda x: np.concatenate(([l], x.shape)), i0x)) yIsList = isinstance(i0y, list) if not yIsList: shapeY = np.concatenate(([l], i0y.shape)) else: shapeY = list( map(lambda x: np.concatenate(([l], x.shape)), i0y)) data = None ext = "dscache" if os.path.exists(name): if not os.path.isdir(name): #old style data = load(name) elif os.path.exists(f"{name}/x_0.{ext}"): if not xIsList: data = (np.zeros(shapeX, i0x.dtype), np.zeros(shapeY, i0y.dtype)) else: data = (list( map(lambda x: np.zeros(x, i0x[0].dtype), shapeX)), np.zeros(shapeY, i0y.dtype)) try: readArray(data[0], f"{name}/x_", ext, "Loading X cache...", l) except ValueError: raise ValueError( f"Stored X has unexpected size for dataset '{name}'. Path: " + name) try: readArray(data[1], f"{name}/y_", ext, "Loading Y cache...", l) except ValueError: raise ValueError( f"Stored Y has unexpected size for dataset '{name}'. Path: " + name) if data is None: if not xIsList: data = (np.zeros(shapeX, i0x.dtype), np.zeros(shapeY, i0y.dtype)) else: data = (list( map(lambda x: np.zeros(x, i0x[0].dtype), shapeX)), np.zeros(shapeY, i0y.dtype)) # if not xIsList: # def func(i): # data[0][i] = input[i].x # data[1][i] = input[i].y # else: # def func(i): # for j in range(len(shapeX)): # data[0][j][i] = input[i].x[j] # data[1][i] = input[i].y # pool = Pool(4) # zip(*pool.map(func, range(0, l))) for i in tqdm.tqdm(range(l), "building disk cache for:" + id): if not xIsList: data[0][i] = input[i].x else: for j in range(len(shapeX)): data[0][j][i] = input[i].x[j] data[1][i] = input[i].y if not os.path.isdir(name): os.mkdir(name) dumpArray(data[0], f"{name}/x_", ext, "Saving X cache...") dumpArray(data[1], f"{name}/y_", ext, "Saving Y cache...") result = DiskCache(input, data) inherit_dataset_params(input, result) if hasattr(input, "name"): result.origName = input.name storage[name] = result return result finally: pass
def ccc1(input): try: name = cache_name(input) name = get_cache_dir() + name if name in storage: return storage[name] if isinstance(input, CompositeDataSet): components = list(map(lambda x: ccc1(x), input.components)) compositeDS = CompositeDataSet(components) inherit_dataset_params(input, compositeDS) if hasattr(input, "name"): compositeDS.origName = input.name return compositeDS data = None xStructPath = f"{name}/x.struct" yStructPath = f"{name}/y.struct" blocksCountPath = f"{name}/blocks_count.int" if os.path.exists(xStructPath) and os.path.exists( yStructPath) and os.path.exists(blocksCountPath): blocksCount = load(blocksCountPath) xStruct = load(xStructPath) yStruct = load(yStructPath) xIsListOrTuple = xStruct[2] in ["list", "tuple"] yIsListOrTuple = yStruct[2] in ["list", "tuple"] xData, yData = init_buffers(xStruct, yStruct) for blockInd in tqdm.tqdm(range(blocksCount), "loading disk cache for:" + name): if not xIsListOrTuple: blockPath = f"{name}/x_{blockInd}.dscache" if os.path.exists(blockPath): xBuff = load(blockPath) for x in xBuff: xData.append(x) else: raise Exception(f"Cache block is missing: {name}") else: for c in range(len(xStruct[0])): blockPath = f"{name}/x_{blockInd}_{c}.dscache" if os.path.exists(blockPath): xBuff = load(blockPath) for x in xBuff: xData[c].append(x) else: raise Exception( f"Cache block is missing: {name}") if not yIsListOrTuple: blockPath = f"{name}/y_{blockInd}.dscache" if os.path.exists(blockPath): yBuff = load(blockPath) for y in yBuff: yData.append(y) else: raise Exception(f"Cache block is missing: {name}") else: for c in range(len(yStruct[0])): blockPath = f"{name}/y_{blockInd}_{c}.dscache" if os.path.exists(blockPath): yBuff = load(blockPath) for y in yBuff: yData[c].append(y) else: raise Exception( f"Cache block is missing: {name}") data = (xData, yData) if data is None: if not os.path.isdir(name): os.mkdir(name) i0 = input[0] i0x = i0.x i0y = i0.y l = len(input) xStruct = inspect_structure(i0x) yStruct = inspect_structure(i0y) xIsListOrTuple = xStruct[2] in ["list", "tuple"] yIsListOrTuple = yStruct[2] in ["list", "tuple"] xData, yData = init_buffers(xStruct, yStruct) buffSize = 0 barrier = 64 * 1024 * 1024 blockInd = 0 for i in tqdm.tqdm(range(l), "building disk cache for:" + id): item = input[i] if not xIsListOrTuple: xData.append(item.x) else: for c in range(len(xStruct[0])): xData[c].append(item.x[c]) if not yIsListOrTuple: yData.append(item.y) else: for c in range(len(yStruct[0])): yData[c].append(item.y[c]) buffSize += get_size(item.x) buffSize += get_size(item.y) if buffSize > barrier or i == l - 1: if not xIsListOrTuple: arr = xData if xStruct[0][0].startswith("int") or xStruct[0][ 0].startswith("float"): arr = np.array(arr) save(f"{name}/x_{blockInd}.dscache", arr) else: for c in range(len(xStruct[0])): arr = xData[c] if xStruct[0][c].startswith("int") or xStruct[ 0][c].startswith("float"): arr = np.array(arr) save(f"{name}/x_{blockInd}_{c}.dscache", arr) if not yIsListOrTuple: arr = yData if yStruct[0][0].startswith("int") or yStruct[0][ 0].startswith("float"): arr = np.array(arr) save(f"{name}/y_{blockInd}.dscache", arr) else: for c in range(len(yStruct[0])): arr = yData[c] if yStruct[0][c].startswith("int") or yStruct[ 0][c].startswith("float"): arr = np.array(arr) save(f"{name}/y_{blockInd}_{c}.dscache", arr) buffSize = 0 blockInd += 1 xData, yData = init_buffers(xStruct, yStruct) pass save(xStructPath, xStruct) save(yStructPath, yStruct) save(blocksCountPath, blockInd) return ccc1(input) result = DiskCache1(input, data, xIsListOrTuple, yIsListOrTuple) storage[name] = result return result finally: pass