def _preprocess_pipeline(file_path, destination_folder):
"""
"""
#print('detect_and_align_faces - PID: {}'.format(os.getpid()))
gc.unfreeze()
raw_image = _read_image(file_path)
detected_image = _detect_faces(raw_image)
cropped_face = _align_face(detected_image)
result = _save_image(cropped_face, destination_folder)
return _log_results(result)
def __getitem__(self, idx):
try:
gc.unfreeze()
self.lock.acquire()
#print("STARTING")
self.ref = pysam.FastaFile(self.ref_fasta)
#get the coordinates for the current batch
coords = self.get_coords(idx)
#print("GOT COORDS")
#get the inputs
X = []
for cur_input_index in range(self.num_inputs):
cur_input = self.input_path[cur_input_index]
if cur_input == "seq":
cur_x = one_hot_encode(
self.transform_seq(self.get_seq(coords)))
if self.expand_dims == True:
cur_x = np.expand_dims(cur_x, axis=1)
else:
#extract values from pandas df
cur_x = self.transform_vals(
self.get_pd_vals(coords, cur_input))
X.append(cur_x)
#get the outputs
y = []
for cur_output_index in range(self.num_outputs):
cur_output = self.output_path[cur_output_index]
if cur_output == "seq":
cur_y = one_hot_encode(
self.transform_seq(self.get_seq(coords)))
if self.expand_dims == True:
cur_y = np.expand_dims(cur_y, axis=1)
else:
cur_y = self.transform_vals(
self.get_pd_vals(coords, cur_output))
y.append(cur_y)
self.lock.release()
#return the batch as an X,y tuple
#print("SUCCESS")
if self.return_coords is False:
return (X, y)
else:
return (X, y, coords)
except Exception as e:
print(str(e) + " from id:" + str(idx))
kill_child_processes(os.getpid())
raise
def start_http(app: web.Application,
http_port: int = port,
use_fork: bool = fork):
"""
Create app instance(s) binding a port.
:param app: the app to execute in server instances
:param http_port: port to bind
:param use_fork: fork or not to use more than one CPU (process)
"""
http_socket = netutil.bind_sockets(http_port) # HTTP socket
if use_fork:
try: # try to create threads
if gc.isenabled():
# collection before a POSIX fork() call may free pages for future allocation
gc.freeze()
process.fork_processes(0) # fork
except KeyboardInterrupt: # except KeyboardInterrupt to "properly" exit
ioloop.IOLoop.current().stop()
gc.unfreeze()
exit(0)
except AttributeError: # OS without fork() support ...
logger.warning('Can not fork, continuing with only one thread ...')
# do nothing and continue without multi-threading
logger.info('Start an HTTP request handler on port : ' + str(http_port))
server = httpserver.HTTPServer(app)
server.add_sockets(http_socket) # bind http port
global servers
servers.append((server, ioloop.IOLoop.current()))
# try to stay forever alive to satisfy user's requests, except on KeyboardInterrupt to "properly" exit
try:
ioloop.IOLoop.current().start()
except KeyboardInterrupt:
server.close_all_connections()
server.stop()
ioloop.IOLoop.current().stop()
def start(number):
freeze()
pid = os.fork()
unfreeze()
if pid:
children[pid] = number
return None
# child process
seed = int(hexlify(os.urandom(16)), 16)
random.seed(seed)
global TASK_ID
for sig in pass_signals:
signal.signal(sig, lambda c, *_: exit(c))
TASK_ID = number
entrypoint()
exit(0)
def test_freeze(self):
gc.freeze()
self.assertGreater(gc.get_freeze_count(), 0)
gc.unfreeze()
self.assertEqual(gc.get_freeze_count(), 0)
def test_get_all_referrers(self):
obj = 'test_get_all_referrers'
referrer = [obj]
gc.freeze()
self.assertIn(referrer, gc.get_all_referrers(obj))
gc.unfreeze()
def test_get_frozen_heap(self):
gc.freeze()
self.assertEqual(gc.get_freeze_count(), len(gc.get_frozen_objects()))
gc.unfreeze()
def test_freeze(self):
gc.freeze()
self.assertGreater(gc.get_freeze_count(), 0)
gc.unfreeze()
self.assertEqual(gc.get_freeze_count(), 0)
for o in olist:
map[type(o)] += 1
return map
def diff_type_count_maps(before, after):
return [(k, after[k] - before[k]) for k in after if after[k] != before[k]]
if __name__ == '__main__':
# \todo This is an attempt to see the Python objects that are not collected,
# but this doesn't work very well, as things always leak a bit.
test_leaks = False
if test_leaks:
gc.unfreeze()
gc.collect()
obs_before = get_all_objects()
unittest.main(exit=False)
if test_leaks:
gc.unfreeze()
gc.collect()
obs_after = get_all_objects([obs_before]) # type: ignore
before = create_type_count_map(obs_before) # type: ignore
after = create_type_count_map(obs_after)
def __getitem__(self, idx):
gc.unfreeze()
self.ref = pysam.FastaFile(self.ref_fasta)
#get the coordinates for the current batch
tdb_batch_indices = self.get_tdb_indices_for_batch(
idx) #coords is a df with 'chrom' and 'pos' columns.
coords = None
if self.return_coords is True:
#get the chromosome coordinates that correspond to indices
coords = self.get_coords(tdb_batch_indices, idx)
#get the inputs
X = []
#iterate through the list of model inputs
for cur_input_index in range(self.num_inputs):
cur_input = self.tdb_input_source_attribute[cur_input_index]
cur_x = None
#iterate through the stacked channels of the current input
for cur_input_channel_index in range(len(cur_input)):
cur_input_channel = cur_input[cur_input_channel_index]
if cur_input_channel == "seq":
#get the one-hot encoded sequence
if coords is None:
coords = self.get_coords(tdb_batch_indices, idx)
cur_seq = one_hot_encode(
self.get_seq(
coords, self.tdb_input_flank[cur_input_index]
[cur_input_channel_index]))
if cur_x is None:
cur_x = cur_seq
else:
cur_x = np.concatenate((cur_x, cur_seq), axis=-1)
else:
#extract values from tdb
cur_vals = self.get_tdb_vals(
tdb_batch_indices, cur_input_channel,
self.tdb_input_flank[cur_input_index]
[cur_input_channel_index],
self.input_dataset_indices[cur_input_index]
[cur_input_channel_index])
aggregate_vals = self.aggregate_vals(
cur_vals, self.tdb_input_aggregation[cur_input_index]
[cur_input_channel_index])
transformed_vals = self.transform_vals(
aggregate_vals,
self.tdb_input_transformation[cur_input_index]
[cur_input_channel_index])
if cur_x is None:
cur_x = transformed_vals
else:
cur_x = np.concatenate((cur_x, transformed_vals),
axis=-1)
#perform reverse complementation, if specified
if self.add_revcomp is True:
cur_x = np.concatenate((cur_x, np.flip(cur_x)), axis=0)
X.append(cur_x)
#get the outputs
y = []
for cur_output_index in range(self.num_outputs):
cur_y = None
cur_output = self.tdb_output_source_attribute[cur_output_index]
for cur_output_channel_index in range(len(cur_output)):
cur_output_channel = cur_output[cur_output_channel_index]
if cur_output_channel == "seq":
#get the one-hot encoded sequence
if coords is None:
coords = get_coords(tdb_batch_indices, idx)
cur_seq = one_hot_encode(
self.get_seq(
coords, self.tdb_output_flank[cur_output_index]
[cur_output_channel_index]))
if cur_y is None:
cur_y = cur_seq
else:
cur_y = np.concatenate((cur_y, cur_seq), axis=-1)
else:
#extract values from tdb
cur_vals = self.get_tdb_vals(
tdb_batch_indices, cur_output_channel,
self.tdb_output_flank[cur_output_index]
[cur_output_channel_index],
self.output_dataset_indices[cur_output_index]
[cur_output_channel_index])
aggregate_vals = self.aggregate_vals(
cur_vals, self.tdb_output_aggregation[cur_output_index]
[cur_output_channel_index])
transformed_vals = self.transform_vals(
aggregate_vals,
self.tdb_output_transformation[cur_output_index]
[cur_output_channel_index])
if cur_y is None:
cur_y = transformed_vals
else:
cur_y = np.concatenate((cur_y, transformed_vals),
axis=-1)
if self.add_revcomp is True:
cur_y = np.concatenate((cur_y, np.flip(cur_y)), axis=0)
y.append(cur_y)
if self.return_coords is True:
coords_updated = []
if self.add_revcomp == True:
for i in coords:
coords_updated.append(i + ['p'])
for i in coords:
coords_updated.append(i + ['r'])
else:
for i in coords:
coords_updated.append(i + ['.'])
coords = np.string_(coords_updated)
coords = coords.astype('S256')
filtered_X, filtered_y, filtered_coords = self.remove_data_out_of_range(
X, y, coords)
else:
filtered_X, filtered_y, filtered_coords = self.remove_data_out_of_range(
X, y)
if filtered_X[0].size == 0:
#empty!
try:
return self.__getitem__(idx + 1)
except:
#we are at the last index, wrap around
return self.__getitem__(0)
if self.return_coords is True:
#print(str(filtered_coords))
return (filtered_X, filtered_y, filtered_coords)
else:
return (filtered_X, filtered_y)
