def transform(self, Xb, yb):
shared_array_name = str(uuid4())
fnames, labels = Xb, yb
args = []
da_args = self.da_args()
for i, fname in enumerate(fnames):
args.append((i, shared_array_name, fname, da_args))
if self.num_image_channels is None:
test_img = data.load_augment(fnames[0], **da_args)
self.num_image_channels = test_img.shape[-1]
try:
shared_array = SharedArray.create(
shared_array_name,
[len(Xb), self.w, self.h, self.num_image_channels],
dtype=np.float32)
self.pool.map(load_shared, args)
Xb = np.array(shared_array, dtype=np.float32)
finally:
SharedArray.delete(shared_array_name)
# if labels is not None:
# labels = labels[:, np.newaxis]
return Xb, labels
def transform(self, Xb, yb):
shared_array_name = str(uuid4())
try:
shared_array = SharedArray.create(
shared_array_name, [len(Xb), 3, self.config.get('w'),
self.config.get('h')], dtype=np.float32)
fnames, labels = super(SharedIterator, self).transform(Xb, yb)
args = []
for i, fname in enumerate(fnames):
kwargs = {k: self.config.get(k) for k in ['w', 'h']}
if not self.deterministic:
kwargs.update({k: self.config.get(k)
for k in ['aug_params', 'sigma']})
kwargs['transform'] = getattr(self, 'tf', None)
kwargs['color_vec'] = getattr(self, 'color_vec', None)
args.append((i, shared_array_name, fname, kwargs))
self.pool.map(load_shared, args)
Xb = np.array(shared_array, dtype=np.float32)
finally:
SharedArray.delete(shared_array_name)
if labels is not None:
labels = labels[:, np.newaxis]
return Xb, labels
def __init__(self, file_path, file_name):
print('class', DBSCAN.eps, DBSCAN.minpts)
self.core_points = []
self.core_point_labels = []
self.core_points_index = []
self.border_points_index = []
self.border_points = []
self.border_point_labels = []
self.noise_points = []
# self.nearest_neighbours = {} # use for small values, space complexity is O(n^2)
self.n_threads = cpu_count()
self.features = []
self.labels = []
self.features, self.labels = process_dataset(
file_path, file_name) # limit the size of the dataset
size = 10000
self.features, self.labels = self.features[:size, :], self.labels[:
size]
print('features: \n', self.features.shape)
try:
sa.delete("shm://features")
except Exception as e:
print('file does not exist')
self.shared_memory = sa.create("shm://features", self.features.shape)
# copy the array into the shared memory
for row_index in range(self.features.shape[0]):
for point_index in range(self.features.shape[1]):
self.shared_memory[row_index,
point_index] = self.features[row_index,
point_index]
self.clusters = []
def transform(self, Xb, yb):
shared_array_name = str(uuid4())
try:
shared_array = SharedArray.create(
shared_array_name,
[len(Xb), 3,
self.config.get('w'),
self.config.get('h')],
dtype=np.float32)
fnames, labels = super(SharedIterator, self).transform(Xb, yb)
args = []
for i, fname in enumerate(fnames):
kwargs = {k: self.config.get(k) for k in ['w', 'h']}
if not self.deterministic:
kwargs.update({
k: self.config.get(k)
for k in ['aug_params', 'sigma']
})
kwargs['transform'] = getattr(self, 'tf', None)
kwargs['color_vec'] = getattr(self, 'color_vec', None)
args.append((i, shared_array_name, fname, kwargs))
self.pool.map(load_shared, args)
Xb = np.array(shared_array, dtype=np.float32)
finally:
SharedArray.delete(shared_array_name)
if labels is not None:
labels = labels[:, np.newaxis]
return Xb, labels
def transform(self, fundus, grade):
shared_array_fundus_rescale_name = str(uuid4())
shared_array_fundus_rescale_mean_subtract_name = str(uuid4())
try:
shared_array_fundus_mean_subt = SharedArray.create(
shared_array_fundus_rescale_name,
[len(fundus), img_h, img_w, 3],
dtype=np.float32)
shared_array_fundus_z = SharedArray.create(
shared_array_fundus_rescale_mean_subtract_name,
[len(fundus), img_h, img_w, 3],
dtype=np.float32)
args = []
for i, _ in enumerate(fundus):
args.append((i, shared_array_fundus_rescale_name,
shared_array_fundus_rescale_mean_subtract_name,
fundus[i], self.is_train))
self.pool.map(load_shared, args)
fundus_rescale = np.array(shared_array_fundus_mean_subt,
dtype=np.float32)
fundus_rescale_mean_subtract = np.array(shared_array_fundus_z,
dtype=np.float32)
finally:
SharedArray.delete(shared_array_fundus_rescale_name)
SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name)
return fundus, fundus_rescale, fundus_rescale_mean_subtract, grade
def run(self):
"""
# TODO: write description
"""
try:
self.t0 = time.time()
self.t1 = self.t0
q = self.channel.queue_declare(queue='detector')
self.channel.queue_declare(queue='time_logs')
if q.method.message_count >= 59:
time.sleep(1)
frame_num, timestamp, images_list = self.batch_generator.__next__()
self.log_time("Took next batch:")
sh_mem_adress = f"shm://{self.module_name}_{frame_num}"
try:
shared_mem = sa.create(sh_mem_adress, np.shape(images_list))
except:
sa.delete(sh_mem_adress)
shared_mem = sa.create(sh_mem_adress, np.shape(images_list))
self.log_time('Created shared memory')
shared_mem[:] = np.array(images_list)
self.log_time('Copied to shared memory:')
self.channel.basic_publish(exchange='',
routing_key='detector',
body=sh_mem_adress)
self.log_time('Published message:')
########################################################################
del frame_num, timestamp, images_list
self.log_time('Full time:', from_start=True)
except StopIteration: # no more frames left in videos_provider
print('stop iter')
def shard_array_to_s3_mp(self, array, indices, s3_bucket, s3_keys):
"""Shard array to S3 in parallel.
:param ndarray array: array to be put into S3
:param list indices: indices corrsponding to the s3 keys
:param str s3_bucket: S3 bucket to use
:param list s3_keys: List of S3 keys corresponding to the indices.
"""
def work_shard_array_to_s3(s3_key, index, array_name, s3_bucket):
array = sa.attach(array_name)
if sys.version_info >= (3, 5):
data = bytes(array[index].data)
else:
data = bytes(np.ascontiguousarray(array[index]).data)
if self.enable_compression:
cctx = zstd.ZstdCompressor(level=9, write_content_size=True)
data = cctx.compress(data)
self.s3aio.s3io.put_bytes(s3_bucket, s3_key, data)
array_name = '_'.join(['SA3IO', str(uuid.uuid4()), str(os.getpid())])
sa.create(array_name, shape=array.shape, dtype=array.dtype)
shared_array = sa.attach(array_name)
shared_array[:] = array
results = self.pool.map(work_shard_array_to_s3, s3_keys, indices,
repeat(array_name), repeat(s3_bucket))
sa.delete(array_name)
def delete_created_arrays(self):
"""Delete all created shared memory arrays.
Arrays are prefixed by 'S3' or 'DCCORE'.
"""
for a in self.list_created_arrays():
sa.delete(a)
def __exit__(self, *args):
for array in self._shared:
try:
sa.delete(array)
except FileNotFoundError:
pass
def main():
"""Main function"""
filepath, name, prefix, dtype = parse_arguments()
if name is None:
name = os.path.splitext(os.path.basename(filepath))[0]
if prefix is not None:
name = prefix + '_' + name
print("Loading data from '{}'.".format(filepath))
if filepath.endswith('.npy'):
data = np.load(filepath)
data = data.astype(dtype)
print("Saving data to shared memory.")
sa.delete(name)
sa_array = sa.create(name, data.shape, data.dtype)
np.copyto(sa_array, data)
else:
with np.load(filepath) as loaded:
print("Saving data to shared memory.")
sa_array = sa.create(name, loaded['shape'], dtype)
sa_array[[x for x in loaded['nonzero']]] = True
print("Successfully saved: (name='{}', shape={}, dtype={})".format(
name, sa_array.shape, sa_array.dtype))
def to_shared_memory(object, name):
logging.info("Writing to shared memory %s" % name)
meta_information = {}
for property_name in object.properties:
data = object.__getattribute__(property_name)
if data is None:
data = np.zeros(0)
# Wrap single ints in arrays
if data.shape == ():
data = np.array([data], dtype=data.dtype)
data_type = data.dtype
data_shape = data.shape
meta_information[property_name] = (data_type, data_shape)
# Make shared memory and copy data to buffer
#logging.info("Field %s has shape %s and type %s" % (property_name, data_shape, data_type))
try:
sa.delete(name + "_" + property_name)
logging.info("Deleted already shared memory")
except FileNotFoundError:
logging.info("No existing shared memory, can create new one")
shared_array = sa.create(name + "_" + property_name, data_shape, data_type)
shared_array[:] = data
f = open(name + "_meta.shm", "wb")
pickle.dump(meta_information, f)
logging.info("Done writing to shared memory")
def create_new_sa_array(name, shape, dtype):
try:
sa.delete(name)
except FileNotFoundError:
pass
finally:
sa_array = sa.create(name, shape, dtype=dtype)
return sa_array
def create_new_sa_array(name, shape, dtype):
try:
sa.delete(name)
except FileNotFoundError:
pass
finally:
sa_array = sa.create(name, shape, dtype=dtype)
return sa_array
def get_publisher(channel: str, shape: tuple, dtype) -> np.ndarray:
# Create an array in shared memory.
short_name = channel.split("://")[-1]
mapping = {e.name.decode(): e for e in sa.list()}
if short_name in mapping:
array = mapping[short_name]
if array.dtype == dtype and array.dims == shape:
return sa.attach(channel)
sa.delete(short_name)
return sa.create(channel, shape, dtype)
def __del__(self):
if self.use_shared_memory:
self.logger.info('Deleting GT database from shared memory')
cur_rank, num_gpus = common_utils.get_dist_info()
sa_key = self.sampler_cfg.DB_DATA_PATH[0]
if cur_rank % num_gpus == 0 and os.path.exists(
f"/dev/shm/{sa_key}"):
SharedArray.delete(f"shm://{sa_key}")
if num_gpus > 1:
dist.barrier()
self.logger.info('GT database has been removed from shared memory')
def get_byte_range_mp(self,
s3_bucket,
s3_key,
s3_start,
s3_end,
block_size,
new_session=False):
"""Gets bytes from a S3 object within a range in parallel.
:param str s3_bucket: name of the s3 bucket.
:param str s3_key: name of the s3 key.
:param int s3_start: begin of range.
:param int s3_end: begin of range.
:param int block_size: block size for download.
:param bool new_session: Flag to create a new session or reuse existing session.
True: create new session
False: reuse existing session
:return: Requested bytes
"""
def work_get(block_number, array_name, s3_bucket, s3_key, s3_max_size,
block_size):
start = block_number * block_size
end = (block_number + 1) * block_size
if end > s3_max_size:
end = s3_max_size
d = self.get_byte_range(s3_bucket, s3_key, start, end, True)
# d = np.frombuffer(d, dtype=np.uint8, count=-1, offset=0)
shared_array = sa.attach(array_name)
shared_array[start:end] = d
if not self.enable_s3:
return self.get_byte_range(s3_bucket, s3_key, s3_start, s3_end,
new_session)
s3 = self.s3_resource(new_session)
s3o = s3.Bucket(s3_bucket).Object(s3_key).get()
s3_max_size = s3o['ContentLength']
s3_obj_size = s3_end - s3_start
num_streams = int(np.ceil(s3_obj_size / block_size))
blocks = range(num_streams)
array_name = '_'.join(
['S3IO', s3_bucket, s3_key,
str(uuid.uuid4()),
str(os.getpid())])
sa.create(array_name, shape=s3_obj_size, dtype=np.uint8)
shared_array = sa.attach(array_name)
self.pool.map(work_get, blocks, repeat(array_name), repeat(s3_bucket),
repeat(s3_key), repeat(s3_max_size), repeat(block_size))
sa.delete(array_name)
return shared_array
def create_shared_array(name, shape, dtype):
"""Create shared array. Prompt if a file with the same name existed."""
try:
return sa.create(name, shape, dtype)
except FileExistsError:
response = ""
while response.lower() not in ["y", "n", "yes", "no"]:
response = input("Existing array (also named " + name +
") was found. Replace it? (y/n) ")
if response.lower() in ("n", "no"):
sys.exit(0)
sa.delete(name)
return sa.create(name, shape, dtype)
def get_silhouette(profile, cluster, stepwise, pool):
logging.info('Calculating pairwise distance ...')
dist = getDistance(profile, 'p_dist', pool)
with NamedTemporaryFile(dir='.', prefix='HCCeval_') as file:
dist_buf = 'file://{0}.dist'.format(file.name)
dist2 = sa.create(dist_buf, dist.shape[:2], dist.dtype)
dist2[:] = dist[:, :, 0] + dist[:, :, 0].T
del dist
logging.info('Calculating Silhouette score ...')
silhouette = np.array(
pool.map(get_silhouette2, [[dist_buf, tag] for tag in cluster.T]))
sa.delete(dist_buf)
return silhouette
def clear(
self,
name=None
): # I previously wrote a __del__ function but it will automatically clear memory after script call
if name is None:
for key in self.keys:
sa.delete(key)
self.keys = []
else:
delkeys = [x for x in self.keys if x.startswith(name)]
for key in delkeys:
sa.delete(key)
self.keys = [x for x in self.keys if not x.startswith(name)]
def getDistance(data, func_name, pool, start=0, allowed_missing=0.0):
with NamedTemporaryFile(dir='.', prefix='HCC_') as file :
prefix = 'file://{0}'.format(file.name)
func = eval(func_name)
mat_buf = '{0}.mat.sa'.format(prefix)
mat = sa.create(mat_buf, shape = data.shape, dtype = data.dtype)
mat[:] = data[:]
dist_buf = '{0}.dist.sa'.format(prefix)
dist = sa.create(dist_buf, shape = [mat.shape[0] - start, mat.shape[0], 2], dtype = np.int32)
dist[:] = 0
__parallel_dist(mat_buf, func, dist_buf, mat.shape, pool, start, allowed_missing)
sa.delete(mat_buf)
os.unlink(dist_buf[7:])
return dist
def delete_shared_memory(file_names, wlabel=True):
for fname in file_names:
fn = fname.split('/')[-1][:12]
if os.path.exists("/dev/shm/{}_xyz".format(fn)):
SA.delete("shm://{}_xyz".format(fn))
SA.delete("shm://{}_rgb".format(fn))
if wlabel:
SA.delete("shm://{}_label".format(fn))
SA.delete("shm://{}_instance_label".format(fn))
def load_delete(label):
sa_name = args.dataset + split + '_' + label
if args.cmd == 'load':
data = dataset[label]
sa_data = sa.create("shm://" + sa_name,
np.shape(data),
dtype=data.dtype)
print('Transferring %s to shared memory ' % (sa_name))
sa_data[:] = data
elif args.cmd == 'delete':
sa.delete(sa_name)
print('Deleted %s from the shared memory' % sa_name)
else:
raise NotImplementedError
def generate_stability_for_medoid(self, masked_parts_medoid,
spatial_states_for_sw, l_labels_sorted):
"""
Generate Stability maps for dynamic parcels for one seed
:param masked_parts_medoid: Dask array
:param spatial_states_for_sw:
:param l_labels_sorted: list
:param chunksize_voxels: int
Size of the chunk in an array
:return:
darr_stab_maps
len(l_labels_sorted)
"""
try:
SharedArray.delete('stab_maps')
except:
pass
stab_maps = SharedArray.create(
'stab_maps', (len(l_labels_sorted), masked_parts_medoid.shape[1]))
def compute_stability_map(masked_parts_medoid, spatial_states_for_sw,
state, idx):
stab_maps[idx, :] = masked_parts_medoid[spatial_states_for_sw ==
state, :].mean(axis=0)
processes = []
idx = 0
for state in l_labels_sorted:
process = Process(target=compute_stability_map,
args=(masked_parts_medoid, spatial_states_for_sw,
state, idx))
processes.append(process)
process.start()
idx += 1
for process in processes:
process.join()
SharedArray.delete('stab_maps')
return stab_maps
def transform(self, Xb, yb):
shared_array_name = str(uuid4())
try:
shared_array = SharedArray.create(
shared_array_name, [len(Xb), self.w, self.h, 3], dtype=np.float32)
fnames, labels = Xb, yb
args = []
da_args = self.da_args()
for i, fname in enumerate(fnames):
args.append((i, shared_array_name, fname, da_args))
self.pool.map(load_shared, args)
Xb = np.array(shared_array, dtype=np.float32)
finally:
SharedArray.delete(shared_array_name)
return Xb, labels
def data_func(measurement):
if not use_threads:
data = numpy.full(sources.shape + geobox.shape,
measurement['nodata'],
dtype=measurement['dtype'])
for index, datasets in numpy.ndenumerate(sources.values):
_fuse_measurement(
data[index],
datasets,
geobox,
measurement,
fuse_func=fuse_func,
skip_broken_datasets=skip_broken_datasets,
driver_manager=driver_manager)
else:
def work_load_data(array_name, index, datasets):
data = sa.attach(array_name)
_fuse_measurement(
data[index],
datasets,
geobox,
measurement,
fuse_func=fuse_func,
skip_broken_datasets=skip_broken_datasets,
driver_manager=driver_manager)
array_name = '_'.join(
['DCCORE',
str(uuid.uuid4()),
str(os.getpid())])
sa.create(array_name,
shape=sources.shape + geobox.shape,
dtype=measurement['dtype'])
data = sa.attach(array_name)
data[:] = measurement['nodata']
pool = ThreadPool(32)
pool.map(work_load_data, repeat(array_name),
*zip(*numpy.ndenumerate(sources.values)))
sa.delete(array_name)
return data
def transform(self, fundus, vessel, grade):
shared_array_fundus_mean_subt_name = str(uuid4())
shared_array_fundus_z_name = str(uuid4())
shared_array_vessel_name = str(uuid4())
try:
shared_array_fundus_mean_subt = SharedArray.create(
shared_array_fundus_mean_subt_name, [len(fundus), img_h, img_w, 3], dtype=np.float32)
shared_array_fundus_z = SharedArray.create(
shared_array_fundus_z_name, [len(fundus), img_h, img_w, 3], dtype=np.float32)
shared_array_vessel = SharedArray.create(
shared_array_vessel_name, [len(fundus), img_h, img_w, 1], dtype=np.float32)
n_grades = len(grade)
if self.grade_type == "DR":
grade_onehot = np.zeros((n_grades, n_grade_dr))
elif self.grade_type == "DME":
grade_onehot = np.zeros((n_grades, n_grade_dme))
for i in range(n_grades):
grade_onehot[i, grade[i]] = 1
args = []
for i, _ in enumerate(fundus):
args.append((i, shared_array_fundus_mean_subt_name, shared_array_fundus_z_name, shared_array_vessel_name, fundus[i], vessel[i], self.is_train, self.normalize))
self.pool.map(load_shared, args)
fundus_mean_subt_img = np.array(shared_array_fundus_mean_subt, dtype=np.float32)
fundus_z_img = np.array(shared_array_fundus_z, dtype=np.float32)
vessel_img = np.array(shared_array_vessel, dtype=np.float32)
finally:
SharedArray.delete(shared_array_fundus_mean_subt_name)
SharedArray.delete(shared_array_fundus_z_name)
SharedArray.delete(shared_array_vessel_name)
return fundus, fundus_mean_subt_img, fundus_z_img, vessel_img, grade_onehot
def transform(self, fundus, vessel, coords):
shared_array_fundus_name = str(uuid4())
shared_array_vessel_name = str(uuid4())
shared_array_lm_name = str(uuid4())
try:
shared_array_fundus = SharedArray.create(
shared_array_fundus_name, [len(fundus), img_h, img_w, 3],
dtype=np.float32)
shared_array_vessel = SharedArray.create(
shared_array_vessel_name, [len(fundus), img_h, img_w, 1],
dtype=np.float32)
shared_array_lm = SharedArray.create(shared_array_lm_name,
[len(fundus), 4],
dtype=np.float32)
args = []
for i, fname in enumerate(fundus):
args.append((i, shared_array_fundus_name,
shared_array_vessel_name, shared_array_lm_name,
fundus[i], vessel[i], coords[i], self.is_train))
self.pool.map(load_shared, args)
fundus_img = np.array(shared_array_fundus, dtype=np.float32)
vessel_img = np.array(shared_array_vessel, dtype=np.float32)
coords_arr = np.array(shared_array_lm, dtype=np.float32)
finally:
SharedArray.delete(shared_array_fundus_name)
SharedArray.delete(shared_array_vessel_name)
SharedArray.delete(shared_array_lm_name)
return fundus_img, vessel_img, coords_arr, fundus
def transform(self, fundus, grade):
shared_array_ex_name = str(uuid4())
shared_array_he_name = str(uuid4())
shared_array_ma_name = str(uuid4())
shared_array_se_name = str(uuid4())
shared_array_fundus_rescale_mean_subtract_name = str(uuid4())
try:
shared_array_ex = SharedArray.create(
shared_array_ex_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32)
shared_array_he = SharedArray.create(
shared_array_he_name, (len(fundus),) + feature_shape_ex_he, dtype=np.float32)
shared_array_ma = SharedArray.create(
shared_array_ma_name, (len(fundus),) + feature_shape_ma, dtype=np.float32)
shared_array_se = SharedArray.create(
shared_array_se_name, (len(fundus),) + feature_shape_se, dtype=np.float32)
shared_array_fundus_rescale_mean_subtract = SharedArray.create(
shared_array_fundus_rescale_mean_subtract_name, (len(fundus),) + img_shape, dtype=np.float32)
args = []
for i, _ in enumerate(fundus):
args.append((i, shared_array_ex_name, shared_array_he_name, shared_array_ma_name, shared_array_se_name,
shared_array_fundus_rescale_mean_subtract_name, fundus[i], self.features_home, self.is_train))
self.pool.map(load_shared, args)
ex = np.array(shared_array_ex, dtype=np.float32)
he = np.array(shared_array_he, dtype=np.float32)
ma = np.array(shared_array_ma, dtype=np.float32)
se = np.array(shared_array_se, dtype=np.float32)
fundus_rescale_mean_subtract = np.array(shared_array_fundus_rescale_mean_subtract, dtype=np.float32)
finally:
SharedArray.delete(shared_array_fundus_rescale_mean_subtract_name)
SharedArray.delete(shared_array_ex_name)
SharedArray.delete(shared_array_he_name)
SharedArray.delete(shared_array_ma_name)
SharedArray.delete(shared_array_se_name)
return fundus, ex, he, ma, se, fundus_rescale_mean_subtract, grade
def clean_shared_memory(self):
self.logger.info(
f'Clean training data from shared memory (file limit={self.shared_memory_file_limit})'
)
cur_rank, num_gpus = common_utils.get_dist_info()
all_infos = self.infos[:self.shared_memory_file_limit] \
if self.shared_memory_file_limit < len(self.infos) else self.infos
cur_infos = all_infos[cur_rank::num_gpus]
for info in cur_infos:
pc_info = info['point_cloud']
sequence_name = pc_info['lidar_sequence']
sample_idx = pc_info['sample_idx']
sa_key = f'{sequence_name}___{sample_idx}'
if not os.path.exists(f"/dev/shm/{sa_key}"):
continue
SharedArray.delete(f"shm://{sa_key}")
if num_gpus > 1:
dist.barrier()
self.logger.info('Training data has been deleted from shared memory')
def callback(self, method, body):
with self.cycle_time.labels(module=self.module_name,
name=socket.gethostname()).time():
self.t0 = time.time()
self.t1 = self.t0
torch.cuda.set_device(np.random.randint(10 % 3))
message = body.decode()
if message == 'END':
self.channel.basic_publish(exchange='',
routing_key='reid',
body=body)
return
frame_num = map(
int,
message.split('_')[-1]) # takes frame_num from adress
images_list = sa.attach(message)
self.log_time("Read image from shm:")
bboxes = self.detector.predict_with_scores(images_list)
self.log_time("Detector predicted:")
bboxes = np.array([tensor[0].numpy() for tensor in bboxes[0]])
self.log_time("Detector output into array converted:")
if bboxes.shape[0] != 0:
sh_mem_adress = f"shm://{self.module_name}_{frame_num}"
try:
shared_mem = sa.create(sh_mem_adress, bboxes.shape)
except:
sa.delete(sh_mem_adress)
shared_mem = sa.create(sh_mem_adress, bboxes.shape)
self.log_time("Shared memory created:")
# copy image to shared memory
shared_mem[:] = np.array(bboxes)
self.log_time("Detector copied to shared memory:")
sa.delete(message)
sa.delete(sh_mem_adress)
del images_list, bboxes
torch.cuda.empty_cache()
self.channel.basic_ack(delivery_tag=method.delivery_tag)
self.log_time('Full time:', from_start=True)
self.last_success.labels(
module=self.module_name,
name=socket.gethostname()).set_to_current_time()
#with self.cycle_time.labels(module=self.module_name, name=socket.gethostname()).time():
push_to_gateway('pushgateway:9091',
job='Test ' + str(self.start_time),
registry=self.registry)
def transform(self, fundus_fnames, vessel_fnames, seg_fnames):
assert len(fundus_fnames) == len(vessel_fnames) and len(
fundus_fnames) == len(seg_fnames)
n_imgs = len(fundus_fnames)
fundus_shared_array_name = str(uuid4())
vessel_shared_array_name = str(uuid4())
seg_shared_array_name = str(uuid4())
try:
fundus_shared_array = SharedArray.create(fundus_shared_array_name,
[n_imgs, img_h, img_w, 3],
dtype=np.float32)
vessel_shared_array = SharedArray.create(vessel_shared_array_name,
[n_imgs, img_h, img_w, 1],
dtype=np.float32)
seg_shared_array = SharedArray.create(
seg_shared_array_name, [len(seg_fnames), img_h, img_w, 1],
dtype=np.float32)
args = []
for i in range(n_imgs):
args.append(
(i, fundus_shared_array_name, vessel_shared_array_name,
seg_shared_array_name, self.augment, fundus_fnames[i],
vessel_fnames[i], seg_fnames[i]))
self.pool.map(load_shared, args)
funduses = np.array(fundus_shared_array, dtype=np.float32)
vessels = np.array(vessel_shared_array, dtype=np.float32)
segs = np.array(seg_shared_array, dtype=np.float32)
finally:
SharedArray.delete(fundus_shared_array_name)
SharedArray.delete(vessel_shared_array_name)
SharedArray.delete(seg_shared_array_name)
return fundus_fnames, funduses, vessels, segs
def shutdown(self):
if self.multiprocessing:
self.executor.close()
sa.delete(self.sharedprefix + 'W')
sa.delete(self.sharedprefix + 'V')
sa.delete(self.sharedprefix + 'Tau2')
sa.delete(self.sharedprefix + 'sigma2')
sa.delete(self.sharedprefix + 'lam2')
sa.delete(self.sharedprefix + 'Constraints_A')
sa.delete(self.sharedprefix + 'Constraints_C')
sa.delete(self.sharedprefix + 'Delta_data')
sa.delete(self.sharedprefix + 'Delta_row')
sa.delete(self.sharedprefix + 'Delta_col')
if self.Row_constraints is not None:
sa.delete(self.sharedprefix + 'Row_constraints')
if self.Mu_ep is not None:
sa.delete(self.sharedprefix + 'Mu_ep')
sa.delete(self.sharedprefix + 'Sigma_ep')
else:
self.executor.shutdown()
def _cleanup(self):
if self.wfunc is None:
sa.delete(self.id)
def cleanup():
print('Cleaning up')
sa.delete('creature_gfx')