def queue_from_lower(self, queue: multiprocessing.Queue):
if self.__started:
raise multiprocessing.ProcessError(
'LayerStack should not be changed after its processes were started.'
)
self.queue_from_lower = queue
self.layers[len(self.layers) - 1].queue_from_lower = queue
def run_command(cmdline):
try:
cmdline = " ".join(cmdline)
p = subprocess.run(cmdline, shell=True, stdout=subprocess.PIPE, universal_newlines = True)
return (p.stdout, p.returncode)
except KeyboardInterrupt:
raise multiprocessing.ProcessError()
def get_env_metas(*env_names, ctx=multiprocessing):
"""Spawn a subprocess and use that to get metadata about an environment
(env_spec, observation_space, action_space, etc.). Can optionally be passed
a custom multiprocessing context to spawn subprocess with (e.g. so you can
use 'spawn' method rather than the default 'fork').
This is useful for environments that pollute some global state of the
process which constructs them. For instance, the MILBench environments
create some X resources that cannot be forked gracefully. If you do fork
and then try to create a new env in the child, then you will end up with
inscrutable resource errors."""
mgr = ctx.Manager()
rv_dict = mgr.dict()
proc = ctx.Process(target=_get_env_meta_target, args=(env_names, rv_dict))
try:
proc.start()
proc.join(30)
finally:
proc.terminate()
if proc.exitcode != 0:
raise multiprocessing.ProcessError(
f"nonzero exit code {proc.exitcode} when collecting metadata "
f"for '{env_names}'")
result = rv_dict['result']
return result
def create_job(self, job_spec):
logger.debug("[docker]create_job: %s", job_spec)
cwd = os.getcwd()
volumes = {cwd: {'bind': cwd, 'mode': 'rw'},
HOME_DIR: {'bind': HOME_DIR, 'mode': 'rw'}}
debug = config.debug
tty = debug
stdin_open = debug
if job_spec.image is None:
image = config.default_image
else:
image = job_spec.image
try:
container = self.client.containers.run(
image,
job_spec.command,
name=job_spec.name + '-' + str(uuid.uuid4()),
volumes=volumes,
cap_add=["SYS_PTRACE"],
tty=tty,
stdin_open=stdin_open,
detach=True
)
except docker.errors.ImageNotFound:
raise mp.ProcessError(
"Docker image \"{}\" not found or cannot be pulled from "
"docker registry.".format(job_spec.image))
job = DockerJob(container, container.id)
# delayed
container._fiber_backend_reloading = False
return job
def my_callback(i):
"""
Increments a global variable by i in the main process.
"""
if multiprocessing.current_process().name != "MainProcess":
raise multiprocessing.ProcessError("The global is being incremented in the wrong process!")
global count
count += i
print "Counter in main process is now: %s" % count
def run_command(cmdline):
try:
p = subprocess.Popen(cmdline, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
output, _ = p.communicate()
if p.returncode != 0:
raise CommandException(output.decode("utf-8"))
return output.decode("utf-8")
except KeyboardInterrupt:
raise multiprocessing.ProcessError()
def clustering_cpu(dispatcher, temp_storage):
_refresh_session()
processes = []
log_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
total_edge_count = mp.Value(ctypes.c_uint64, 0)
for pid in range(num_of_threads):
processes.append(mp.Process(target=_worker, args=(pid, dispatcher, temp_storage, total_edge_count,
log_lock, merge_lock, exit_signal)))
reporter = Thread(target=_reporter, args=(dispatcher, math.ceil(len(session.internal_index)/block_dimensions[0]),
log_lock, exit_signal))
logging.info('......Start clustering using CPU with {} subprocesses......'.format(num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads): processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info('Received KeyboardInterrupt, clustering canceled.')
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
_clean_temporary(temp_storage)
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
_clean_temporary(temp_storage)
raise mp.ProcessError(err_msg)
return total_edge_count
def get_backend(name=None):
"""
Returns a working Fiber backend. If `name` is specified, returns a
backend specified by `name`.
"""
global _backends
if name is None:
if config.backend is not None:
name = config.backend
else:
name = auto_select_backend()
else:
if name not in available_backend:
raise mp.ProcessError("Invalid backend: {}".format(name))
_backend = _backends.get(name, None)
if _backend is None:
_backend = importlib.import_module(
"fiber.{}_backend".format(name)).Backend()
_backends[name] = _backend
return _backend
def insert(self,
layer: LayerProcess,
on_top_of: LayerProcess = None,
below_of: LayerProcess = None):
"""
Insert a layer into the layer stack, placing it on top of or beneath another layer. Either on_top_of or below_of
must be provided.
:param layer: The layer to insert.
:param on_top_of: The layer on top of which the new layer should be inserted. Must not be used together with
below_of.
:param below_of: The layer beneath of which the new layer should be inserted. Must not be used together with
on_top_of.
:raises multiprocessing.ProcessError if this method is called after the layer stack was started.
:raises TypeError if the layer to insert is None, or on_top_of and below_of are used together.
:raises ValueError if on_top_of/below_of is not a layer in this LayerStack.
"""
if self.__started:
raise multiprocessing.ProcessError(
'LayerStack should not be changed after its processes were started.'
)
if layer is None:
raise TypeError('Layer is None.')
# Make sure that exactly one out of on_top_of, below_of is provided.
if (on_top_of is None) == (below_of is None):
raise TypeError('Needs either on_top_of xor below_of')
insert_above: bool = on_top_of is not None
ref_layer = on_top_of if insert_above else below_of
# Find position of the reference layer
for i in range(len(self.layers)):
if self.layers[i] == ref_layer:
# If the new layer should be inserted above the reference layer, it should be placed at the current
# position of the reflayer, pushing the reflayer and following down by one. Else the reflayer should
# remain at its position, only subsequent layers should be pushed down by one.
if insert_above:
self.__insert(layer, i)
else:
self.__insert(layer, i + 1)
return
raise ValueError('Reference layer is not in the layer stack.')
def get_listen_addr(self):
ip = None
# if fiber.current_process() is multiprocessing.current_process():
if not isinstance(fiber.current_process(), fiber.Process):
# not inside docker
# logger.debug("use interface docker0")
ifce = "eth0"
ip = find_ip_by_net_interface(ifce)
else:
# inside a Fiber process
# logger.debug("use interface eth0")
# ip = find_ip_by_net_interface("eth0")
ip, ifce = find_listen_address()
# ip = find_ip_by_net_interface(ifce)
if ip is None:
raise mp.ProcessError(
"Can't find a usable IPv4 address to listen. ifce_name: {}, "
"ifces: {}".format(ifce, psutil.net_if_addrs()))
# use 0 to bind to a random free port number
return ip, 0, ifce
def init_fiber(proc_name=None, **kwargs):
"""
Initialize Fiber. This function is called when you want to re-initialize
Fiber with new config values and also re-init loggers.
:param proc_name: process name of current process
:param kwargs: If kwargs is not None, init Fiber system with corresponding
key/value pairs in kwargs as config keys and values.
"""
# check backend
if "backend" in kwargs:
backend = kwargs["backend"]
if (backend not in fiber_backend.available_backend
and backend is not None):
raise mp.ProcessError("Invalid backend: {}".format(backend))
updates = fiber_config.init(**kwargs)
if "log_level" in updates or "log_file" in updates:
_config = fiber_config.get_object()
init_logger(_config, proc_name=proc_name)
def get_listen_addr(self):
ip = None
if sys.platform == "darwin":
# use the same hostname for both master and non master process
# because docker.for.mac.localhost resolves to different inside
# and outside docker container. "docker.for.mac.localhost" is
# the name that doesn't change in and outside the container.
return "docker.for.mac.localhost", 0
if not isinstance(fiber.current_process(), fiber.Process):
# not inside docker
ifce = "docker0"
ip = find_ip_by_net_interface(ifce)
else:
# inside a Fiber process
ip, ifce = find_listen_address()
if ip is None:
raise mp.ProcessError(
"Can't find a usable IPv4 address to listen. ifce_name: {}, "
"ifces: {}".format(ifce, psutil.net_if_addrs()))
# use 0 to bind to a random free port number
return ip, 0, ifce
def build_index():
_refresh_session()
if len(session.ms_exp_files) == 0:
err_msg = '\nNo ms exp files specified.'
logging.error(err_msg)
raise Exception(err_msg)
mzxml._refresh_session()
mzml._refresh_session()
processes = []
q = mp.Queue()
log_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
valid_ms2_count = mp.Value(ctypes.c_uint64, 0)
total_ms2_count = mp.Value(ctypes.c_uint64, 0)
finish_count = mp.Value(ctypes.c_uint64, 0)
num_files = len(session.ms_exp_files)
temp_storage = Path.cwd().joinpath(str(uuid4()))
temp_storage.mkdir()
temp_storage = temp_storage.resolve()
Path(temp_storage, 'prm').touch()
Path(temp_storage, 'chg').touch()
Path(temp_storage, 'fli').touch()
Path(temp_storage, 'off').touch()
Path(temp_storage, 'iid').touch()
Path(temp_storage, 'nvi').touch()
for n, path in enumerate(session.ms_exp_files):
q.put((n, path))
for i in range(num_of_threads):
q.put(None)
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, q, temp_storage, valid_ms2_count,
total_ms2_count, finish_count, log_lock,
merge_lock, exit_signal)))
reporter = Thread(target=_reporter,
args=(finish_count, num_files, log_lock, exit_signal))
logging.info(
'......Start building index with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, index building canceled.')
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
_error_clean(temp_storage, q)
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
logging.debug('Emptying queue')
_error_clean(temp_storage, q)
raise mp.ProcessError(err_msg)
if valid_ms2_count.value == 0:
err_msg = '\nNo valid MS2 scan has been found.'
logging.error(err_msg)
_error_clean(temp_storage, q)
raise Exception(err_msg)
logging.info(
'......Finish building index, found {} valid MS2 scan (out of {} in total)......'
.format(valid_ms2_count.value, total_ms2_count.value))
try:
_integrate(temp_storage, valid_ms2_count.value)
except Exception:
err_msg = '\nFailed to integrate index data.'
logging.error(err_msg)
raise
_clean_temporary(temp_storage)
session.config.ii_finished.value = True
logging.debug('Mounting internal index file')
session.mount_internal_index()
return
def _export_clusters(file, num_of_threads=os.cpu_count(), specific_cluster=None):
_refresh_session()
processes = []
log_lock = mp.Lock()
count_lock = mp.Lock()
read_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
finish_count = mp.Value(ctypes.c_uint64, 0)
if not file.exists():
with file.open('w', encoding='utf-8') as fp:
fp.write(_get_header_line('Columns'))
fp.write(_get_header_line('Cluster'))
fp.write('ID\tNum of nodes\tNum of edges\tNum of identifications\t' +
'Major identification\tIdentified ratio\tAverage precursor mass\n')
fp.write(_get_header_line('Nodes'))
fp.write('ID\tFile\tScan num\tIdentification\tProbability\n')
fp.write(_get_header_line('Edges'))
fp.write('ID of source\tID of target\tDot product\n')
fp.write('\n' + _get_header_line('Clusters'))
if specific_cluster:
_write_cluster(None, session.clusters.cursor, specific_cluster, file, log_lock, read_lock, merge_lock)
return
session.clusters.connect()
num_clusters = session.clusters.cursor.execute('SELECT "num_of_clusters" FROM "metadata"').fetchone()[0]
session.clusters.disconnect()
for pid in range(num_of_threads):
processes.append(mp.Process(target=_worker, args=(pid, file, num_clusters, finish_count, log_lock,
count_lock, read_lock, merge_lock, exit_signal)))
reporter = Thread(target=_reporter, args=(finish_count, num_clusters, log_lock, exit_signal))
logging.info('......Start cluster export with {} subprocesses......'.format(num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads): processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info('Received KeyboardInterrupt, cluster enrichment canceled.')
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
raise mp.ProcessError(err_msg)
logging.info('......Finish cluster export......')
logging.debug('Reconnecting clusters file')
session.clusters.connect()
return
def rank_transform():
_refresh_session()
processes = []
index_length = len(session.internal_index)
dispatcher = _Dispatcher(0, index_length, 10000)
log_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
file_path = Path.cwd().joinpath(session.name +
FILE_EXTENSION_RANKED_SPECTRA)
if file_path.exists(): file_path.unlink()
data_order = (DATA_TYPE_SIZE[RT_MZ_DATA_TYPE] * num_of_peaks,
DATA_TYPE_SIZE[RT_INTENSITY_DATA_TYPE] * num_of_peaks)
offsets = [FILE_HEADER_SIZE]
for n in range(len(data_order) - 1):
offsets.append(
sum(data_order[:n + 1]) * index_length + FILE_HEADER_SIZE)
total_size = sum(data_order) * index_length + FILE_HEADER_SIZE
try:
subprocess.check_output(
('fallocate', '-l', str(total_size), str(file_path)),
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
err_msg = '\nFailed to allocate space for ranked spectra storage.'\
'\nstderr: {}'.format(e.output)
logging.error(err_msg)
raise
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, dispatcher, offsets, log_lock, merge_lock,
exit_signal)))
reporter = Thread(target=_reporter,
args=(dispatcher, index_length, log_lock, exit_signal))
logging.info(
'......Start rank transformation with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, index building canceled.')
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
raise mp.ProcessError(err_msg)
logging.info('......Finish rank transformation......')
_write_metadata(file_path, offsets)
session.config.rt_finished.value = True
logging.debug('Mounting ranked spectra file')
session.mount_ranked_spectra()
return
def _enrich_clusters(update=False,
num_of_threads=os.cpu_count(),
specific_cluster=None):
_refresh_session()
if not session.iden_lut:
logging.info(
'No identification lookup table is mounted, cannot proceed.')
return
processes = []
log_lock = mp.Lock()
count_lock = mp.Lock()
iden_read_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
finish_count = mp.Value(ctypes.c_uint64, 1)
if specific_cluster:
clusters_conn = sqlite3.connect(str(session.clusters.file_path))
clusters_cur = clusters_conn.cursor()
session.iden_lut.connect()
clusters_cur.execute(
'SELECT "cluster_id", "num_idens", "pickled" FROM "clusters" WHERE "cluster_id" = ?',
(specific_cluster, ))
cluster = clusters_cur.fetchone()
data_no_iden = []
data_with_iden = []
bytes_count = 0
_add_iden_stat(cluster, data_no_iden, data_with_iden, bytes_count,
update, iden_read_lock)
_push(None, clusters_conn, clusters_cur, data_no_iden, data_with_iden,
log_lock, merge_lock)
clusters_conn.close()
return
session.clusters.connect()
num_clusters = session.clusters.cursor.execute(
'SELECT "num_of_clusters" FROM "metadata"').fetchone()[0]
session.clusters.disconnect()
session.iden_lut.disconnect()
chunk_size = math.ceil(num_clusters / num_of_threads)
ranges = [(start, min(num_clusters, start + chunk_size))
for start in range(0, num_clusters, chunk_size)]
num_of_threads = num_of_threads if len(ranges) == num_of_threads else len(
ranges)
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, update, ranges[pid], finish_count, log_lock,
count_lock, iden_read_lock, merge_lock,
exit_signal)))
reporter = Thread(target=_reporter,
args=(finish_count, num_clusters, log_lock, exit_signal))
logging.info(
'......Start cluster enrichment with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, cluster enrichment canceled.')
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
raise mp.ProcessError(err_msg)
logging.info('......Finish cluster enrichment......')
logging.debug('Reconnecting clusters and identification lookup table file')
session.clusters.connect()
session.iden_lut.connect()
return
def refine_cluster():
_refresh_session()
if len(outlier_threshold) == 0:
logging.info(
'Outlier threshold list is empty, skip cluster refinement.')
return
processes = []
log_lock = mp.Lock()
count_lock = mp.Lock()
read_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
finish_count = mp.Value(ctypes.c_uint64, 0)
total_count = mp.Value(ctypes.c_uint64, 0)
raw_clusters = session.clusters.file_path
raw_clusters = sqlite3.connect(str(raw_clusters.absolute()))
cur = raw_clusters.cursor()
num_raw_clusters = cur.execute(
'SELECT "num_of_clusters" FROM "metadata"').fetchone()[0]
raw_clusters.close()
refined_clusters = Path.cwd().joinpath(
session.name + FILE_EXTENSION_TEMPORARY).absolute()
if refined_clusters.exists():
refined_clusters.unlink()
refined_clusters = sqlite3.connect(str(refined_clusters))
cur = refined_clusters.cursor()
cur.execute('''
CREATE TABLE "metadata" (
"version" TEXT,
"file_type" TEXT,
"num_of_clusters" INTEGER,
"magic_label" BLOB,
"session_label" BLOB,
"index_label" BLOB
)''')
metadata = (VERSION, HEADER_LABEL_CLUSTERS, 0, pickle.dumps(uuid4()),
pickle.dumps(session.magic_label),
pickle.dumps(session.internal_index.magic_label))
cur.execute('INSERT INTO "metadata" VALUES (?, ?, ?, ?, ?, ?)', metadata)
cur.execute('''
CREATE TABLE "clusters" (
"cluster_id" INTEGER PRIMARY KEY AUTOINCREMENT,
"num_nodes" INTEGER,
"num_edges" INTEGER,
"num_idens" INTEGER,
"major_iden" TEXT,
"iden_ratio" REAL,
"pre_mass_avg" REAL,
"pickled" BLOB
)''')
refined_clusters.commit()
refined_clusters.close()
chunk_size = math.ceil(num_raw_clusters / set_num_of_threads)
ranges = [(start, min(num_raw_clusters, start + chunk_size))
for start in range(0, num_raw_clusters, chunk_size)]
num_of_threads = set_num_of_threads if len(
ranges) == set_num_of_threads else len(ranges)
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, ranges[pid], finish_count, total_count,
log_lock, count_lock, read_lock, merge_lock,
exit_signal)))
reporter = Thread(target=_reporter,
args=(finish_count, num_raw_clusters, log_lock,
exit_signal))
logging.info(
'......Start cluster refinement with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, identification import canceled.'
)
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
raise mp.ProcessError(err_msg)
if finish_count.value == 0:
err_msg = '\nNo cluster has been made.'
logging.error(err_msg)
raise Exception(err_msg)
if keep_raw:
Path.cwd().joinpath(session.name + FILE_EXTENSION_CLUSTERS).rename(
session.name + FILE_EXTENSION_RAW_CLUSTERS)
Path.cwd().joinpath(session.name + FILE_EXTENSION_TEMPORARY).rename(
session.name + FILE_EXTENSION_CLUSTERS)
session.config.cr_finished.value = True
logging.info(
'......Finish cluster refinement, formed {} clusters......'.format(
total_count.value))
return finish_count.value
def import_identification():
_refresh_session()
if len(session.iden_files) == 0:
logging.info(
'No identification file specified, skip identification import.')
return
pepxml._refresh_session()
processes = []
q = mp.Queue()
log_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
valid_iden_count = mp.Value(ctypes.c_uint64, 0)
total_iden_count = mp.Value(ctypes.c_uint64, 0)
finish_count = mp.Value(ctypes.c_uint64, 0)
num_files = len(session.iden_files)
iden_lut = Path.cwd().joinpath(session.name + FILE_EXTENSION_IDEN_LUT)
if iden_lut.exists():
iden_lut.unlink()
iden_lut = sqlite3.connect(str(iden_lut.absolute()))
cur = iden_lut.cursor()
cur.execute('''
CREATE TABLE "metadata" (
"version" TEXT,
"file_type" TEXT,
"mods_pos_data_type" TEXT,
"mods_mass_data_type" TEXT,
"magic_label" BLOB,
"session_label" BLOB
)''')
metadata = (VERSION, HEADER_LABEL_IDEN_LUT, ID_MODS_POS_DATA_TYPE,
ID_MODS_MASS_DATA_TYPE, pickle.dumps(uuid4()),
pickle.dumps(session.magic_label))
cur.execute('INSERT INTO "metadata" VALUES (?, ?, ?, ?, ?, ?)', metadata)
iden_lut.commit()
iden_lut.close()
for n, path in enumerate(session.iden_files):
q.put((n, path))
for i in range(num_of_threads):
q.put(None)
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, q, valid_iden_count, total_iden_count,
finish_count, log_lock, merge_lock, exit_signal)))
reporter = Thread(target=_reporter,
args=(finish_count, num_files, log_lock, exit_signal))
logging.info(
'......Start identification import with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, identification import canceled.'
)
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
_error_clean(q)
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
logging.debug('Emptying queue')
_error_clean(q)
raise mp.ProcessError(err_msg)
if valid_iden_count.value == 0:
err_msg = '\nNo valid identification has been found.'
logging.error(err_msg)
raise Exception(err_msg)
logging.info(
'......Finish identification import, found {} valid identifications (out of {} in total)......'
.format(valid_iden_count.value, total_iden_count.value))
session.config.id_finished.value = True
logging.debug('Mounting identification lookup table')
session.mount_identification_lut()
return
def plotbest(self, pool=None, nth=0):
"""Plot best circuit in the current pool.
Alternatively plot nth circuit from the pool given in arguments."""
if pool == None:
pool = self.pool
try:
circuit = pool[nth][1]
except AttributeError:
#Not scored yet
circuit = pool[nth]
try:
if PLOTTING == 'matplotlib':
for c in xrange(len(self.spice_commands)):
self.save_plot(circuit, i=c, name=str(c))
elif PLOTTING == 'external':
for i in xrange(len(self.spice_commands)):
v = self.eval_single(circuit, self.spice_commands[i])[1]
for k in v.keys():
freq = v[k][0]
gain = v[k][1]
score = self._rank(v, i, k, extra=circuit.extra_value)
goal_val = [
self.ff[i](f,
k,
extra=circuit.extra_value,
generation=self.generation)
for f in freq
]
if self.constraints[
i] != None and self.plot_constraints:
constraint_val = [(freq[c], gain[c])
for c in xrange(len(freq))
if not self.constraints[i]
(freq[c],
gain[c],
k,
extra=circuit.extra_value,
generation=self.generation)]
else:
constraint_val = None
command = os.path.join(
os.path.dirname(os.path.abspath(__file__)),
'plotting.py')
plt = subprocess.Popen(['python', command],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
try:
title = self.plot_titles[i][k]
except (KeyError, TypeError):
title = None
if self.plot_yrange != None and k in self.plot_yrange.keys(
):
yrange = self.plot_yrange[k]
else:
yrange = None
path = os.path.join(os.getcwd(), self.directory)
data = ((freq, gain), k, goal_val, self.sim_type[i],
self.generation, score, path, title, yrange,
self.log_plot[i], constraint_val, i)
output = plt.communicate(str(data))
if output != ('', ''):
raise multiprocessing.ProcessError(
"Plotting failed")
except multiprocessing.ProcessError:
print "Plotting failed"
print output[1]
def _error_callback(result):
print('\tERROR WHEN PROCESSING ', path, file=logfile)
raise _multiprocessing.ProcessError('ERROR WHEN PROCESSING ' +
path)
def make_graphs(temp_storage, edge_list_length):
_refresh_session()
processes = []
log_lock = mp.Lock()
scan_lock = mp.Lock()
count_lock = mp.Lock()
merge_lock = mp.Lock()
exit_signal = mp.Value(ctypes.c_bool, False)
scan_starting_idx = mp.Value(ctypes.c_uint64, 0)
finish_count = mp.Value(ctypes.c_uint64, 0)
clusters = Path.cwd().joinpath(session.name +
FILE_EXTENSION_CLUSTERS).absolute()
if clusters.exists():
clusters.unlink()
clusters = sqlite3.connect(str(clusters))
cur = clusters.cursor()
cur.execute('''
CREATE TABLE "metadata" (
"version" TEXT,
"file_type" TEXT,
"num_of_clusters" INTEGER,
"true_precursor_mass" BOOL,
"magic_label" BLOB,
"session_label" BLOB,
"index_label" BLOB
)''')
metadata = (VERSION, HEADER_LABEL_CLUSTERS, 0,
session.internal_index.true_precursor_mass,
pickle.dumps(uuid4()), pickle.dumps(session.magic_label),
pickle.dumps(session.internal_index.magic_label))
cur.execute('INSERT INTO "metadata" VALUES (?, ?, ?, ?, ?, ?, ?)',
metadata)
cur.execute('''
CREATE TABLE "clusters" (
"cluster_id" INTEGER PRIMARY KEY,
"num_nodes" INTEGER,
"num_edges" INTEGER,
"num_idens" INTEGER,
"idens" TEXT,
"major_iden" TEXT,
"iden_ratio" REAL,
"pre_mass_avg" REAL,
"pickled" BLOB
)''')
clusters.commit()
clusters.close()
for pid in range(num_of_threads):
processes.append(
mp.Process(target=_worker,
args=(pid, scan_starting_idx, temp_storage,
edge_list_length, finish_count, log_lock,
scan_lock, count_lock, merge_lock, exit_signal)))
reporter = Thread(target=_reporter,
args=(finish_count, log_lock, exit_signal))
logging.info('......Start graph making with {} subprocesses......'.format(
num_of_threads))
try:
for i in range(num_of_threads):
processes[i].start()
time.sleep(0.1)
reporter.start()
for i in range(num_of_threads):
processes[i].join()
exit_signal.value = True
if reporter.is_alive():
reporter.join()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.info(
'Received KeyboardInterrupt, identification import canceled.'
)
exit_signal.value = True
with log_lock:
logging.debug('Waiting processes to exit.')
if reporter.is_alive():
reporter.join()
for i in range(num_of_threads):
if processes[i].is_alive():
processes[i].join()
logging.debug('All processes exited.')
raise
# in case that subprocesses were exited with errors
if 1 in [p.exitcode for p in processes]:
err_msg = '\nSubprocesses exited with abnormal exitcode.'
logging.error(err_msg)
raise mp.ProcessError(err_msg)
if finish_count.value == 0:
err_msg = '\nNo cluster has been made.'
logging.error(err_msg)
raise Exception(err_msg)
logging.info('......Finish graph making......')
return finish_count.value
