def transform(self, Xb, yb): shared_array_name = str(uuid4()) fnames, labels = Xb, yb args = [] da_args = self.da_args() for i, fname in enumerate(fnames): args.append((i, shared_array_name, fname, da_args)) if self.num_image_channels is None: test_img = data.load_augment(fnames[0], **da_args) self.num_image_channels = test_img.shape[-1] try: shared_array = SharedArray.create( shared_array_name, [len(Xb), self.w, self.h, self.num_image_channels], dtype=np.float32) self.pool.map(load_shared, args) Xb = np.array(shared_array, dtype=np.float32) finally: SharedArray.delete(shared_array_name) # if labels is not None: # labels = labels[:, np.newaxis] return Xb, labels
def transform(self, Xb, yb): shared_array_name = str(uuid4()) try: shared_array = SharedArray.create( shared_array_name, [len(Xb), 3, self.config.get('w'), self.config.get('h')], dtype=np.float32) fnames, labels = super(SharedIterator, self).transform(Xb, yb) args = [] for i, fname in enumerate(fnames): kwargs = {k: self.config.get(k) for k in ['w', 'h']} if not self.deterministic: kwargs.update({k: self.config.get(k) for k in ['aug_params', 'sigma']}) kwargs['transform'] = getattr(self, 'tf', None) kwargs['color_vec'] = getattr(self, 'color_vec', None) args.append((i, shared_array_name, fname, kwargs)) self.pool.map(load_shared, args) Xb = np.array(shared_array, dtype=np.float32) finally: SharedArray.delete(shared_array_name) if labels is not None: labels = labels[:, np.newaxis] return Xb, labels
def __init__(self, file_path, file_name): print('class', DBSCAN.eps, DBSCAN.minpts) self.core_points = [] self.core_point_labels = [] self.core_points_index = [] self.border_points_index = [] self.border_points = [] self.border_point_labels = [] self.noise_points = [] # self.nearest_neighbours = {} # use for small values, space complexity is O(n^2) self.n_threads = cpu_count() self.features = [] self.labels = [] self.features, self.labels = process_dataset( file_path, file_name) # limit the size of the dataset size = 10000 self.features, self.labels = self.features[:size, :], self.labels[: size] print('features: \n', self.features.shape) try: sa.delete("shm://features") except Exception as e: print('file does not exist') self.shared_memory = sa.create("shm://features", self.features.shape) # copy the array into the shared memory for row_index in range(self.features.shape[0]): for point_index in range(self.features.shape[1]): self.shared_memory[row_index, point_index] = self.features[row_index, point_index] self.clusters = []
def transform(self, Xb, yb): shared_array_name = str(uuid4()) try: shared_array = SharedArray.create( shared_array_name, [len(Xb), 3, self.config.get('w'), self.config.get('h')], dtype=np.float32) fnames, labels = super(SharedIterator, self).transform(Xb, yb) args = [] for i, fname in enumerate(fnames): kwargs = {k: self.config.get(k) for k in ['w', 'h']} if not self.deterministic: kwargs.update({ k: self.config.get(k) for k in ['aug_params', 'sigma'] }) kwargs['transform'] = getattr(self, 'tf', None) kwargs['color_vec'] = getattr(self, 'color_vec', None) args.append((i, shared_array_name, fname, kwargs)) self.pool.map(load_shared, args) Xb = np.array(shared_array, dtype=np.float32) finally: SharedArray.delete(shared_array_name) if labels is not None: labels = labels[:, np.newaxis] return Xb, labels
def transform(self, fundus, grade): shared_array_fundus_rescale_name = str(uuid4()) shared_array_fundus_rescale_mean_subtract_name = str(uuid4()) try: shared_array_fundus_mean_subt = SharedArray.create( shared_array_fundus_rescale_name, [len(fundus), img_h, img_w, 3], dtype=np.float32) shared_array_fundus_z = SharedArray.create( shared_array_fundus_rescale_mean_subtract_name, [len(fundus), img_h, img_w, 3], dtype=np.float32) args = [] for i, _ in enumerate(fundus): args.append((i, shared_array_fundus_rescale_name, shared_array_fundus_rescale_mean_subtract_name, fundus[i], self.is_train)) self.pool.map(load_shared, args) fundus_rescale = np.array(shared_array_fundus_mean_subt, dtype=np.float32) fundus_rescale_mean_subtract = np.array(shared_array_fundus_z, dtype=np.float32) finally: SharedArray.delete(shared_array_fundus_rescale_name) SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name) return fundus, fundus_rescale, fundus_rescale_mean_subtract, grade
def run(self): """ # TODO: write description """ try: self.t0 = time.time() self.t1 = self.t0 q = self.channel.queue_declare(queue='detector') self.channel.queue_declare(queue='time_logs') if q.method.message_count >= 59: time.sleep(1) frame_num, timestamp, images_list = self.batch_generator.__next__() self.log_time("Took next batch:") sh_mem_adress = f"shm://{self.module_name}_{frame_num}" try: shared_mem = sa.create(sh_mem_adress, np.shape(images_list)) except: sa.delete(sh_mem_adress) shared_mem = sa.create(sh_mem_adress, np.shape(images_list)) self.log_time('Created shared memory') shared_mem[:] = np.array(images_list) self.log_time('Copied to shared memory:') self.channel.basic_publish(exchange='', routing_key='detector', body=sh_mem_adress) self.log_time('Published message:') ######################################################################## del frame_num, timestamp, images_list self.log_time('Full time:', from_start=True) except StopIteration: # no more frames left in videos_provider print('stop iter')
def shard_array_to_s3_mp(self, array, indices, s3_bucket, s3_keys): """Shard array to S3 in parallel. :param ndarray array: array to be put into S3 :param list indices: indices corrsponding to the s3 keys :param str s3_bucket: S3 bucket to use :param list s3_keys: List of S3 keys corresponding to the indices. """ def work_shard_array_to_s3(s3_key, index, array_name, s3_bucket): array = sa.attach(array_name) if sys.version_info >= (3, 5): data = bytes(array[index].data) else: data = bytes(np.ascontiguousarray(array[index]).data) if self.enable_compression: cctx = zstd.ZstdCompressor(level=9, write_content_size=True) data = cctx.compress(data) self.s3aio.s3io.put_bytes(s3_bucket, s3_key, data) array_name = '_'.join(['SA3IO', str(uuid.uuid4()), str(os.getpid())]) sa.create(array_name, shape=array.shape, dtype=array.dtype) shared_array = sa.attach(array_name) shared_array[:] = array results = self.pool.map(work_shard_array_to_s3, s3_keys, indices, repeat(array_name), repeat(s3_bucket)) sa.delete(array_name)
def delete_created_arrays(self): """Delete all created shared memory arrays. Arrays are prefixed by 'S3' or 'DCCORE'. """ for a in self.list_created_arrays(): sa.delete(a)
def __exit__(self, *args): for array in self._shared: try: sa.delete(array) except FileNotFoundError: pass
def main(): """Main function""" filepath, name, prefix, dtype = parse_arguments() if name is None: name = os.path.splitext(os.path.basename(filepath))[0] if prefix is not None: name = prefix + '_' + name print("Loading data from '{}'.".format(filepath)) if filepath.endswith('.npy'): data = np.load(filepath) data = data.astype(dtype) print("Saving data to shared memory.") sa.delete(name) sa_array = sa.create(name, data.shape, data.dtype) np.copyto(sa_array, data) else: with np.load(filepath) as loaded: print("Saving data to shared memory.") sa_array = sa.create(name, loaded['shape'], dtype) sa_array[[x for x in loaded['nonzero']]] = True print("Successfully saved: (name='{}', shape={}, dtype={})".format( name, sa_array.shape, sa_array.dtype))
def to_shared_memory(object, name): logging.info("Writing to shared memory %s" % name) meta_information = {} for property_name in object.properties: data = object.__getattribute__(property_name) if data is None: data = np.zeros(0) # Wrap single ints in arrays if data.shape == (): data = np.array([data], dtype=data.dtype) data_type = data.dtype data_shape = data.shape meta_information[property_name] = (data_type, data_shape) # Make shared memory and copy data to buffer #logging.info("Field %s has shape %s and type %s" % (property_name, data_shape, data_type)) try: sa.delete(name + "_" + property_name) logging.info("Deleted already shared memory") except FileNotFoundError: logging.info("No existing shared memory, can create new one") shared_array = sa.create(name + "_" + property_name, data_shape, data_type) shared_array[:] = data f = open(name + "_meta.shm", "wb") pickle.dump(meta_information, f) logging.info("Done writing to shared memory")
def create_new_sa_array(name, shape, dtype): try: sa.delete(name) except FileNotFoundError: pass finally: sa_array = sa.create(name, shape, dtype=dtype) return sa_array
def get_publisher(channel: str, shape: tuple, dtype) -> np.ndarray: # Create an array in shared memory. short_name = channel.split("://")[-1] mapping = {e.name.decode(): e for e in sa.list()} if short_name in mapping: array = mapping[short_name] if array.dtype == dtype and array.dims == shape: return sa.attach(channel) sa.delete(short_name) return sa.create(channel, shape, dtype)
def __del__(self): if self.use_shared_memory: self.logger.info('Deleting GT database from shared memory') cur_rank, num_gpus = common_utils.get_dist_info() sa_key = self.sampler_cfg.DB_DATA_PATH[0] if cur_rank % num_gpus == 0 and os.path.exists( f"/dev/shm/{sa_key}"): SharedArray.delete(f"shm://{sa_key}") if num_gpus > 1: dist.barrier() self.logger.info('GT database has been removed from shared memory')
def get_byte_range_mp(self, s3_bucket, s3_key, s3_start, s3_end, block_size, new_session=False): """Gets bytes from a S3 object within a range in parallel. :param str s3_bucket: name of the s3 bucket. :param str s3_key: name of the s3 key. :param int s3_start: begin of range. :param int s3_end: begin of range. :param int block_size: block size for download. :param bool new_session: Flag to create a new session or reuse existing session. True: create new session False: reuse existing session :return: Requested bytes """ def work_get(block_number, array_name, s3_bucket, s3_key, s3_max_size, block_size): start = block_number * block_size end = (block_number + 1) * block_size if end > s3_max_size: end = s3_max_size d = self.get_byte_range(s3_bucket, s3_key, start, end, True) # d = np.frombuffer(d, dtype=np.uint8, count=-1, offset=0) shared_array = sa.attach(array_name) shared_array[start:end] = d if not self.enable_s3: return self.get_byte_range(s3_bucket, s3_key, s3_start, s3_end, new_session) s3 = self.s3_resource(new_session) s3o = s3.Bucket(s3_bucket).Object(s3_key).get() s3_max_size = s3o['ContentLength'] s3_obj_size = s3_end - s3_start num_streams = int(np.ceil(s3_obj_size / block_size)) blocks = range(num_streams) array_name = '_'.join( ['S3IO', s3_bucket, s3_key, str(uuid.uuid4()), str(os.getpid())]) sa.create(array_name, shape=s3_obj_size, dtype=np.uint8) shared_array = sa.attach(array_name) self.pool.map(work_get, blocks, repeat(array_name), repeat(s3_bucket), repeat(s3_key), repeat(s3_max_size), repeat(block_size)) sa.delete(array_name) return shared_array
def create_shared_array(name, shape, dtype): """Create shared array. Prompt if a file with the same name existed.""" try: return sa.create(name, shape, dtype) except FileExistsError: response = "" while response.lower() not in ["y", "n", "yes", "no"]: response = input("Existing array (also named " + name + ") was found. Replace it? (y/n) ") if response.lower() in ("n", "no"): sys.exit(0) sa.delete(name) return sa.create(name, shape, dtype)
def get_silhouette(profile, cluster, stepwise, pool): logging.info('Calculating pairwise distance ...') dist = getDistance(profile, 'p_dist', pool) with NamedTemporaryFile(dir='.', prefix='HCCeval_') as file: dist_buf = 'file://{0}.dist'.format(file.name) dist2 = sa.create(dist_buf, dist.shape[:2], dist.dtype) dist2[:] = dist[:, :, 0] + dist[:, :, 0].T del dist logging.info('Calculating Silhouette score ...') silhouette = np.array( pool.map(get_silhouette2, [[dist_buf, tag] for tag in cluster.T])) sa.delete(dist_buf) return silhouette
def clear( self, name=None ): # I previously wrote a __del__ function but it will automatically clear memory after script call if name is None: for key in self.keys: sa.delete(key) self.keys = [] else: delkeys = [x for x in self.keys if x.startswith(name)] for key in delkeys: sa.delete(key) self.keys = [x for x in self.keys if not x.startswith(name)]
def getDistance(data, func_name, pool, start=0, allowed_missing=0.0): with NamedTemporaryFile(dir='.', prefix='HCC_') as file : prefix = 'file://{0}'.format(file.name) func = eval(func_name) mat_buf = '{0}.mat.sa'.format(prefix) mat = sa.create(mat_buf, shape = data.shape, dtype = data.dtype) mat[:] = data[:] dist_buf = '{0}.dist.sa'.format(prefix) dist = sa.create(dist_buf, shape = [mat.shape[0] - start, mat.shape[0], 2], dtype = np.int32) dist[:] = 0 __parallel_dist(mat_buf, func, dist_buf, mat.shape, pool, start, allowed_missing) sa.delete(mat_buf) os.unlink(dist_buf[7:]) return dist
def delete_shared_memory(file_names, wlabel=True): for fname in file_names: fn = fname.split('/')[-1][:12] if os.path.exists("/dev/shm/{}_xyz".format(fn)): SA.delete("shm://{}_xyz".format(fn)) SA.delete("shm://{}_rgb".format(fn)) if wlabel: SA.delete("shm://{}_label".format(fn)) SA.delete("shm://{}_instance_label".format(fn))
def load_delete(label): sa_name = args.dataset + split + '_' + label if args.cmd == 'load': data = dataset[label] sa_data = sa.create("shm://" + sa_name, np.shape(data), dtype=data.dtype) print('Transferring %s to shared memory ' % (sa_name)) sa_data[:] = data elif args.cmd == 'delete': sa.delete(sa_name) print('Deleted %s from the shared memory' % sa_name) else: raise NotImplementedError
def generate_stability_for_medoid(self, masked_parts_medoid, spatial_states_for_sw, l_labels_sorted): """ Generate Stability maps for dynamic parcels for one seed :param masked_parts_medoid: Dask array :param spatial_states_for_sw: :param l_labels_sorted: list :param chunksize_voxels: int Size of the chunk in an array :return: darr_stab_maps len(l_labels_sorted) """ try: SharedArray.delete('stab_maps') except: pass stab_maps = SharedArray.create( 'stab_maps', (len(l_labels_sorted), masked_parts_medoid.shape[1])) def compute_stability_map(masked_parts_medoid, spatial_states_for_sw, state, idx): stab_maps[idx, :] = masked_parts_medoid[spatial_states_for_sw == state, :].mean(axis=0) processes = [] idx = 0 for state in l_labels_sorted: process = Process(target=compute_stability_map, args=(masked_parts_medoid, spatial_states_for_sw, state, idx)) processes.append(process) process.start() idx += 1 for process in processes: process.join() SharedArray.delete('stab_maps') return stab_maps
def transform(self, Xb, yb): shared_array_name = str(uuid4()) try: shared_array = SharedArray.create( shared_array_name, [len(Xb), self.w, self.h, 3], dtype=np.float32) fnames, labels = Xb, yb args = [] da_args = self.da_args() for i, fname in enumerate(fnames): args.append((i, shared_array_name, fname, da_args)) self.pool.map(load_shared, args) Xb = np.array(shared_array, dtype=np.float32) finally: SharedArray.delete(shared_array_name) return Xb, labels
def data_func(measurement): if not use_threads: data = numpy.full(sources.shape + geobox.shape, measurement['nodata'], dtype=measurement['dtype']) for index, datasets in numpy.ndenumerate(sources.values): _fuse_measurement( data[index], datasets, geobox, measurement, fuse_func=fuse_func, skip_broken_datasets=skip_broken_datasets, driver_manager=driver_manager) else: def work_load_data(array_name, index, datasets): data = sa.attach(array_name) _fuse_measurement( data[index], datasets, geobox, measurement, fuse_func=fuse_func, skip_broken_datasets=skip_broken_datasets, driver_manager=driver_manager) array_name = '_'.join( ['DCCORE', str(uuid.uuid4()), str(os.getpid())]) sa.create(array_name, shape=sources.shape + geobox.shape, dtype=measurement['dtype']) data = sa.attach(array_name) data[:] = measurement['nodata'] pool = ThreadPool(32) pool.map(work_load_data, repeat(array_name), *zip(*numpy.ndenumerate(sources.values))) sa.delete(array_name) return data
def transform(self, fundus, vessel, grade): shared_array_fundus_mean_subt_name = str(uuid4()) shared_array_fundus_z_name = str(uuid4()) shared_array_vessel_name = str(uuid4()) try: shared_array_fundus_mean_subt = SharedArray.create( shared_array_fundus_mean_subt_name, [len(fundus), img_h, img_w, 3], dtype=np.float32) shared_array_fundus_z = SharedArray.create( shared_array_fundus_z_name, [len(fundus), img_h, img_w, 3], dtype=np.float32) shared_array_vessel = SharedArray.create( shared_array_vessel_name, [len(fundus), img_h, img_w, 1], dtype=np.float32) n_grades = len(grade) if self.grade_type == "DR": grade_onehot = np.zeros((n_grades, n_grade_dr)) elif self.grade_type == "DME": grade_onehot = np.zeros((n_grades, n_grade_dme)) for i in range(n_grades): grade_onehot[i, grade[i]] = 1 args = [] for i, _ in enumerate(fundus): args.append((i, shared_array_fundus_mean_subt_name, shared_array_fundus_z_name, shared_array_vessel_name, fundus[i], vessel[i], self.is_train, self.normalize)) self.pool.map(load_shared, args) fundus_mean_subt_img = np.array(shared_array_fundus_mean_subt, dtype=np.float32) fundus_z_img = np.array(shared_array_fundus_z, dtype=np.float32) vessel_img = np.array(shared_array_vessel, dtype=np.float32) finally: SharedArray.delete(shared_array_fundus_mean_subt_name) SharedArray.delete(shared_array_fundus_z_name) SharedArray.delete(shared_array_vessel_name) return fundus, fundus_mean_subt_img, fundus_z_img, vessel_img, grade_onehot
def transform(self, fundus, vessel, coords): shared_array_fundus_name = str(uuid4()) shared_array_vessel_name = str(uuid4()) shared_array_lm_name = str(uuid4()) try: shared_array_fundus = SharedArray.create( shared_array_fundus_name, [len(fundus), img_h, img_w, 3], dtype=np.float32) shared_array_vessel = SharedArray.create( shared_array_vessel_name, [len(fundus), img_h, img_w, 1], dtype=np.float32) shared_array_lm = SharedArray.create(shared_array_lm_name, [len(fundus), 4], dtype=np.float32) args = [] for i, fname in enumerate(fundus): args.append((i, shared_array_fundus_name, shared_array_vessel_name, shared_array_lm_name, fundus[i], vessel[i], coords[i], self.is_train)) self.pool.map(load_shared, args) fundus_img = np.array(shared_array_fundus, dtype=np.float32) vessel_img = np.array(shared_array_vessel, dtype=np.float32) coords_arr = np.array(shared_array_lm, dtype=np.float32) finally: SharedArray.delete(shared_array_fundus_name) SharedArray.delete(shared_array_vessel_name) SharedArray.delete(shared_array_lm_name) return fundus_img, vessel_img, coords_arr, fundus
def transform(self, fundus, grade): shared_array_ex_name = str(uuid4()) shared_array_he_name = str(uuid4()) shared_array_ma_name = str(uuid4()) shared_array_se_name = str(uuid4()) shared_array_fundus_rescale_mean_subtract_name = str(uuid4()) try: shared_array_ex = SharedArray.create( shared_array_ex_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32) shared_array_he = SharedArray.create( shared_array_he_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32) shared_array_ma = SharedArray.create( shared_array_ma_name, (len(fundus),) + feature_shape_ma, dtype=np.float32) shared_array_se = SharedArray.create( shared_array_se_name, (len(fundus),) + feature_shape_se, dtype=np.float32) shared_array_fundus_rescale_mean_subtract = SharedArray.create( shared_array_fundus_rescale_mean_subtract_name, (len(fundus),) + img_shape, dtype=np.float32) args = [] for i, _ in enumerate(fundus): args.append((i, shared_array_ex_name, shared_array_he_name, shared_array_ma_name, shared_array_se_name, shared_array_fundus_rescale_mean_subtract_name, fundus[i], self.features_home, self.is_train)) self.pool.map(load_shared, args) ex = np.array(shared_array_ex, dtype=np.float32) he = np.array(shared_array_he, dtype=np.float32) ma = np.array(shared_array_ma, dtype=np.float32) se = np.array(shared_array_se, dtype=np.float32) fundus_rescale_mean_subtract = np.array(shared_array_fundus_rescale_mean_subtract, dtype=np.float32) finally: SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name) SharedArray.delete(shared_array_ex_name) SharedArray.delete(shared_array_he_name) SharedArray.delete(shared_array_ma_name) SharedArray.delete(shared_array_se_name) return fundus, ex, he, ma, se, fundus_rescale_mean_subtract, grade
def clean_shared_memory(self): self.logger.info( f'Clean training data from shared memory (file limit={self.shared_memory_file_limit})' ) cur_rank, num_gpus = common_utils.get_dist_info() all_infos = self.infos[:self.shared_memory_file_limit] \ if self.shared_memory_file_limit < len(self.infos) else self.infos cur_infos = all_infos[cur_rank::num_gpus] for info in cur_infos: pc_info = info['point_cloud'] sequence_name = pc_info['lidar_sequence'] sample_idx = pc_info['sample_idx'] sa_key = f'{sequence_name}___{sample_idx}' if not os.path.exists(f"/dev/shm/{sa_key}"): continue SharedArray.delete(f"shm://{sa_key}") if num_gpus > 1: dist.barrier() self.logger.info('Training data has been deleted from shared memory')
def callback(self, method, body): with self.cycle_time.labels(module=self.module_name, name=socket.gethostname()).time(): self.t0 = time.time() self.t1 = self.t0 torch.cuda.set_device(np.random.randint(10 % 3)) message = body.decode() if message == 'END': self.channel.basic_publish(exchange='', routing_key='reid', body=body) return frame_num = map( int, message.split('_')[-1]) # takes frame_num from adress images_list = sa.attach(message) self.log_time("Read image from shm:") bboxes = self.detector.predict_with_scores(images_list) self.log_time("Detector predicted:") bboxes = np.array([tensor[0].numpy() for tensor in bboxes[0]]) self.log_time("Detector output into array converted:") if bboxes.shape[0] != 0: sh_mem_adress = f"shm://{self.module_name}_{frame_num}" try: shared_mem = sa.create(sh_mem_adress, bboxes.shape) except: sa.delete(sh_mem_adress) shared_mem = sa.create(sh_mem_adress, bboxes.shape) self.log_time("Shared memory created:") # copy image to shared memory shared_mem[:] = np.array(bboxes) self.log_time("Detector copied to shared memory:") sa.delete(message) sa.delete(sh_mem_adress) del images_list, bboxes torch.cuda.empty_cache() self.channel.basic_ack(delivery_tag=method.delivery_tag) self.log_time('Full time:', from_start=True) self.last_success.labels( module=self.module_name, name=socket.gethostname()).set_to_current_time() #with self.cycle_time.labels(module=self.module_name, name=socket.gethostname()).time(): push_to_gateway('pushgateway:9091', job='Test ' + str(self.start_time), registry=self.registry)
def transform(self, fundus_fnames, vessel_fnames, seg_fnames): assert len(fundus_fnames) == len(vessel_fnames) and len( fundus_fnames) == len(seg_fnames) n_imgs = len(fundus_fnames) fundus_shared_array_name = str(uuid4()) vessel_shared_array_name = str(uuid4()) seg_shared_array_name = str(uuid4()) try: fundus_shared_array = SharedArray.create(fundus_shared_array_name, [n_imgs, img_h, img_w, 3], dtype=np.float32) vessel_shared_array = SharedArray.create(vessel_shared_array_name, [n_imgs, img_h, img_w, 1], dtype=np.float32) seg_shared_array = SharedArray.create( seg_shared_array_name, [len(seg_fnames), img_h, img_w, 1], dtype=np.float32) args = [] for i in range(n_imgs): args.append( (i, fundus_shared_array_name, vessel_shared_array_name, seg_shared_array_name, self.augment, fundus_fnames[i], vessel_fnames[i], seg_fnames[i])) self.pool.map(load_shared, args) funduses = np.array(fundus_shared_array, dtype=np.float32) vessels = np.array(vessel_shared_array, dtype=np.float32) segs = np.array(seg_shared_array, dtype=np.float32) finally: SharedArray.delete(fundus_shared_array_name) SharedArray.delete(vessel_shared_array_name) SharedArray.delete(seg_shared_array_name) return fundus_fnames, funduses, vessels, segs
def shutdown(self): if self.multiprocessing: self.executor.close() sa.delete(self.sharedprefix + 'W') sa.delete(self.sharedprefix + 'V') sa.delete(self.sharedprefix + 'Tau2') sa.delete(self.sharedprefix + 'sigma2') sa.delete(self.sharedprefix + 'lam2') sa.delete(self.sharedprefix + 'Constraints_A') sa.delete(self.sharedprefix + 'Constraints_C') sa.delete(self.sharedprefix + 'Delta_data') sa.delete(self.sharedprefix + 'Delta_row') sa.delete(self.sharedprefix + 'Delta_col') if self.Row_constraints is not None: sa.delete(self.sharedprefix + 'Row_constraints') if self.Mu_ep is not None: sa.delete(self.sharedprefix + 'Mu_ep') sa.delete(self.sharedprefix + 'Sigma_ep') else: self.executor.shutdown()
def _cleanup(self): if self.wfunc is None: sa.delete(self.id)
def cleanup(): print('Cleaning up') sa.delete('creature_gfx')