def _preprocess_pipeline(file_path, destination_folder): """ """ #print('detect_and_align_faces - PID: {}'.format(os.getpid())) gc.unfreeze() raw_image = _read_image(file_path) detected_image = _detect_faces(raw_image) cropped_face = _align_face(detected_image) result = _save_image(cropped_face, destination_folder) return _log_results(result)
def __getitem__(self, idx): try: gc.unfreeze() self.lock.acquire() #print("STARTING") self.ref = pysam.FastaFile(self.ref_fasta) #get the coordinates for the current batch coords = self.get_coords(idx) #print("GOT COORDS") #get the inputs X = [] for cur_input_index in range(self.num_inputs): cur_input = self.input_path[cur_input_index] if cur_input == "seq": cur_x = one_hot_encode( self.transform_seq(self.get_seq(coords))) if self.expand_dims == True: cur_x = np.expand_dims(cur_x, axis=1) else: #extract values from pandas df cur_x = self.transform_vals( self.get_pd_vals(coords, cur_input)) X.append(cur_x) #get the outputs y = [] for cur_output_index in range(self.num_outputs): cur_output = self.output_path[cur_output_index] if cur_output == "seq": cur_y = one_hot_encode( self.transform_seq(self.get_seq(coords))) if self.expand_dims == True: cur_y = np.expand_dims(cur_y, axis=1) else: cur_y = self.transform_vals( self.get_pd_vals(coords, cur_output)) y.append(cur_y) self.lock.release() #return the batch as an X,y tuple #print("SUCCESS") if self.return_coords is False: return (X, y) else: return (X, y, coords) except Exception as e: print(str(e) + " from id:" + str(idx)) kill_child_processes(os.getpid()) raise
def start_http(app: web.Application, http_port: int = port, use_fork: bool = fork): """ Create app instance(s) binding a port. :param app: the app to execute in server instances :param http_port: port to bind :param use_fork: fork or not to use more than one CPU (process) """ http_socket = netutil.bind_sockets(http_port) # HTTP socket if use_fork: try: # try to create threads if gc.isenabled(): # collection before a POSIX fork() call may free pages for future allocation gc.freeze() process.fork_processes(0) # fork except KeyboardInterrupt: # except KeyboardInterrupt to "properly" exit ioloop.IOLoop.current().stop() gc.unfreeze() exit(0) except AttributeError: # OS without fork() support ... logger.warning('Can not fork, continuing with only one thread ...') # do nothing and continue without multi-threading logger.info('Start an HTTP request handler on port : ' + str(http_port)) server = httpserver.HTTPServer(app) server.add_sockets(http_socket) # bind http port global servers servers.append((server, ioloop.IOLoop.current())) # try to stay forever alive to satisfy user's requests, except on KeyboardInterrupt to "properly" exit try: ioloop.IOLoop.current().start() except KeyboardInterrupt: server.close_all_connections() server.stop() ioloop.IOLoop.current().stop()
def start(number): freeze() pid = os.fork() unfreeze() if pid: children[pid] = number return None # child process seed = int(hexlify(os.urandom(16)), 16) random.seed(seed) global TASK_ID for sig in pass_signals: signal.signal(sig, lambda c, *_: exit(c)) TASK_ID = number entrypoint() exit(0)
def test_freeze(self): gc.freeze() self.assertGreater(gc.get_freeze_count(), 0) gc.unfreeze() self.assertEqual(gc.get_freeze_count(), 0)
def test_get_all_referrers(self): obj = 'test_get_all_referrers' referrer = [obj] gc.freeze() self.assertIn(referrer, gc.get_all_referrers(obj)) gc.unfreeze()
def test_get_frozen_heap(self): gc.freeze() self.assertEqual(gc.get_freeze_count(), len(gc.get_frozen_objects())) gc.unfreeze()
def test_freeze(self): gc.freeze() self.assertGreater(gc.get_freeze_count(), 0) gc.unfreeze() self.assertEqual(gc.get_freeze_count(), 0)
for o in olist: map[type(o)] += 1 return map def diff_type_count_maps(before, after): return [(k, after[k] - before[k]) for k in after if after[k] != before[k]] if __name__ == '__main__': # \todo This is an attempt to see the Python objects that are not collected, # but this doesn't work very well, as things always leak a bit. test_leaks = False if test_leaks: gc.unfreeze() gc.collect() obs_before = get_all_objects() unittest.main(exit=False) if test_leaks: gc.unfreeze() gc.collect() obs_after = get_all_objects([obs_before]) # type: ignore before = create_type_count_map(obs_before) # type: ignore after = create_type_count_map(obs_after)
def __getitem__(self, idx): gc.unfreeze() self.ref = pysam.FastaFile(self.ref_fasta) #get the coordinates for the current batch tdb_batch_indices = self.get_tdb_indices_for_batch( idx) #coords is a df with 'chrom' and 'pos' columns. coords = None if self.return_coords is True: #get the chromosome coordinates that correspond to indices coords = self.get_coords(tdb_batch_indices, idx) #get the inputs X = [] #iterate through the list of model inputs for cur_input_index in range(self.num_inputs): cur_input = self.tdb_input_source_attribute[cur_input_index] cur_x = None #iterate through the stacked channels of the current input for cur_input_channel_index in range(len(cur_input)): cur_input_channel = cur_input[cur_input_channel_index] if cur_input_channel == "seq": #get the one-hot encoded sequence if coords is None: coords = self.get_coords(tdb_batch_indices, idx) cur_seq = one_hot_encode( self.get_seq( coords, self.tdb_input_flank[cur_input_index] [cur_input_channel_index])) if cur_x is None: cur_x = cur_seq else: cur_x = np.concatenate((cur_x, cur_seq), axis=-1) else: #extract values from tdb cur_vals = self.get_tdb_vals( tdb_batch_indices, cur_input_channel, self.tdb_input_flank[cur_input_index] [cur_input_channel_index], self.input_dataset_indices[cur_input_index] [cur_input_channel_index]) aggregate_vals = self.aggregate_vals( cur_vals, self.tdb_input_aggregation[cur_input_index] [cur_input_channel_index]) transformed_vals = self.transform_vals( aggregate_vals, self.tdb_input_transformation[cur_input_index] [cur_input_channel_index]) if cur_x is None: cur_x = transformed_vals else: cur_x = np.concatenate((cur_x, transformed_vals), axis=-1) #perform reverse complementation, if specified if self.add_revcomp is True: cur_x = np.concatenate((cur_x, np.flip(cur_x)), axis=0) X.append(cur_x) #get the outputs y = [] for cur_output_index in range(self.num_outputs): cur_y = None cur_output = self.tdb_output_source_attribute[cur_output_index] for cur_output_channel_index in range(len(cur_output)): cur_output_channel = cur_output[cur_output_channel_index] if cur_output_channel == "seq": #get the one-hot encoded sequence if coords is None: coords = get_coords(tdb_batch_indices, idx) cur_seq = one_hot_encode( self.get_seq( coords, self.tdb_output_flank[cur_output_index] [cur_output_channel_index])) if cur_y is None: cur_y = cur_seq else: cur_y = np.concatenate((cur_y, cur_seq), axis=-1) else: #extract values from tdb cur_vals = self.get_tdb_vals( tdb_batch_indices, cur_output_channel, self.tdb_output_flank[cur_output_index] [cur_output_channel_index], self.output_dataset_indices[cur_output_index] [cur_output_channel_index]) aggregate_vals = self.aggregate_vals( cur_vals, self.tdb_output_aggregation[cur_output_index] [cur_output_channel_index]) transformed_vals = self.transform_vals( aggregate_vals, self.tdb_output_transformation[cur_output_index] [cur_output_channel_index]) if cur_y is None: cur_y = transformed_vals else: cur_y = np.concatenate((cur_y, transformed_vals), axis=-1) if self.add_revcomp is True: cur_y = np.concatenate((cur_y, np.flip(cur_y)), axis=0) y.append(cur_y) if self.return_coords is True: coords_updated = [] if self.add_revcomp == True: for i in coords: coords_updated.append(i + ['p']) for i in coords: coords_updated.append(i + ['r']) else: for i in coords: coords_updated.append(i + ['.']) coords = np.string_(coords_updated) coords = coords.astype('S256') filtered_X, filtered_y, filtered_coords = self.remove_data_out_of_range( X, y, coords) else: filtered_X, filtered_y, filtered_coords = self.remove_data_out_of_range( X, y) if filtered_X[0].size == 0: #empty! try: return self.__getitem__(idx + 1) except: #we are at the last index, wrap around return self.__getitem__(0) if self.return_coords is True: #print(str(filtered_coords)) return (filtered_X, filtered_y, filtered_coords) else: return (filtered_X, filtered_y)