def _worker(pid, file, num_clusters, finish_count, log_lock, count_lock, read_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug('Cluster export subprocess {} started.'.format(pid))
clusters_conn = sqlite3.connect(str(session.clusters.file_path))
clusters_cur = clusters_conn.cursor()
while True:
if exit_signal.value:
with log_lock:
logging.debug('Subprocess {}: Received exit signal, exits now.'.format(pid))
break
with count_lock:
cluster_id = finish_count.value + 1
if cluster_id > num_clusters: break
finish_count.value += 1
_write_cluster(pid, clusters_cur, cluster_id, file, log_lock, read_lock, merge_lock)
clusters_conn.close()
logging.debug('Subprocess {}: Reached the end of iteration, work done.'.format(pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
clusters_conn.close()
with log_lock:
logging.debug('Subprocess {}: Received KeyboardInterrupt, exits now.'.format(pid))
else:
clusters_conn.close()
with log_lock:
logging.exception('\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, did, dispatcher, temp_storage, total_edge_count, log_lock,
merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug(
'Clustering subprocess {} acquiring device {} started.'.format(
pid, did))
drv.init()
cuda_device = drv.Device(did)
cuda_context = cuda_device.make_context()
compiler_option = [
'--fmad=true',
] if use_fmad else [
'--fmad=false',
]
cuda_module = SourceModule(get_source_code(), options=compiler_option)
cuda_kernel = cuda_module.get_function('compute_dot_product')
cuda_kernel.prepare('PPPPPPPPPP')
threads = []
exit_state = mp.Value(ctypes.c_uint32, 0)
with log_lock:
logging.debug(
'Subprocess {}: Spawning {} threads for CUDA stream concurrency.'
.format(pid, threads_per_device))
for tid in range(threads_per_device):
threads.append(
Thread(target=_thread,
args=(pid, tid, cuda_context, cuda_kernel, dispatcher,
temp_storage, total_edge_count, log_lock,
merge_lock, exit_signal, exit_state)))
for t in threads:
t.start()
for t in threads:
t.join()
if exit_state.value == 1:
err_msg = '\nSubprocess {}: Threads exited with abnormal exitcode.'.format(
pid)
logging.error(err_msg)
raise Exception(err_msg)
drv.stop_profiler()
cuda_context.pop()
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
logging.debug(
'Subprocess {}: Waiting threads to exit.'.format(pid))
for t in threads:
t.join()
drv.stop_profiler()
cuda_context.pop()
else:
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
drv.stop_profiler()
cuda_context.pop()
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, q, temp_storage, valid_ms2_count, total_ms2_count,
finish_count, log_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug('Index building subprocess {} started.'.format(pid))
prm_path = Path(temp_storage, 'prm')
chg_path = Path(temp_storage, 'chg')
fli_path = Path(temp_storage, 'fli')
off_path = Path(temp_storage, 'off')
iid_path = Path(temp_storage, 'iid')
nvi_path = Path(temp_storage, 'nvi')
while True:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
item = q.get()
if item:
file_id, path = item
try:
format_ = ''.join(path.suffixes).lower()
if format_ == '.mzxml':
index, valid, total = mzxml.build_index(
path, log_lock=log_lock)
elif format_ == '.mzml':
index, valid, total = mzml.build_index(
path, log_lock=log_lock)
except Exception:
if ignore_errors:
wrn_msg = 'Subprocess {}: Failed to parse MS exp file: {}'.format(
pid, path)
with log_lock:
logging.warning(wrn_msg)
continue
else:
raise
if valid != 0:
index.file_id = np.full(len(index.file_id),
file_id,
dtype=II_FILE_ID_DATA_TYPE)
with merge_lock:
if input_limit > 0 and valid_ms2_count.value >= input_limit:
with log_lock:
logging.debug(
'Subprocess {}: Exceeded input limit, purges queue and exits now.'
.format(pid))
item = 0
while item is not None:
item = q.get()
break
index.internal_id += valid_ms2_count.value
valid_ms2_count.value += valid
total_ms2_count.value += total
prm = np.memmap(str(prm_path),
dtype=II_PRECURSOR_MASS_DATA_TYPE,
mode='r+',
shape=valid_ms2_count.value)
chg = np.memmap(str(chg_path),
dtype=II_PRECURSOR_CHARGE_DATA_TYPE,
mode='r+',
shape=valid_ms2_count.value)
fli = np.memmap(str(fli_path),
dtype=II_FILE_ID_DATA_TYPE,
mode='r+',
shape=valid_ms2_count.value)
off = np.memmap(str(off_path),
dtype=II_OFFSET_DATA_TYPE,
mode='r+',
shape=(valid_ms2_count.value, 2))
iid = np.memmap(str(iid_path),
dtype=II_INTERNAL_ID_DATA_TYPE,
mode='r+',
shape=valid_ms2_count.value)
nvi = np.memmap(str(nvi_path),
dtype=II_NATIVE_ID_DATA_TYPE,
mode='r+',
shape=valid_ms2_count.value)
prm[-valid:] = index.precursor_mass
chg[-valid:] = index.precursor_charge
fli[-valid:] = index.file_id
off[-valid:] = index.offset
iid[-valid:] = index.internal_id
nvi[-valid:] = index.native_id
else:
wrn_msg = 'Subprocess {}: Found 0 valid MS2 scan in file: {}'.format(
pid, path)
with log_lock:
logging.warning(wrn_msg)
finish_count.value += 1
else:
logging.debug(
'Subprocess {}: Got None object, work done.'.format(pid))
break
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, dispatcher, temp_storage, total_edge_count, log_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug('Clustering subprocess {} started.'.format(pid))
edg_path = Path(temp_storage, 'edg')
dps_path = Path(temp_storage, 'dps')
ranked_spectra = session.ranked_spectra
with log_lock:
logging.debug('Clustering subprocess {}: Start iterating dispatcher.'.format(pid))
previous_row_id = -1
dispatcher.connect(pid, 0)
# iterate dispatcher to get blocks
for row_id, column_id, block in dispatcher.iterate(pid, 0):
if exit_signal.value:
with log_lock:
logging.debug('Subprocess {}: Received exit signal, exits now.'.format(pid))
break
try:
y_range, x_range = block
block_height = y_range[1] - y_range[0]
block_width = x_range[1] - x_range[0]
if row_id != previous_row_id:
with log_lock:
logging.debug('Subprocess {}: Processing row {} (y:{}->{}).'.
format(pid, row_id, *y_range))
previous_row_id = row_id
# get necessary data
precursor_mass = np.concatenate((ranked_spectra.precursor_mass[y_range[0]:y_range[1]],
ranked_spectra.precursor_mass[x_range[0]:x_range[1]]))
mz = np.concatenate((ranked_spectra.mz[y_range[0]:y_range[1]],
ranked_spectra.mz[x_range[0]:x_range[1]]))
intensity = np.concatenate((ranked_spectra.intensity[y_range[0]:y_range[1]],
ranked_spectra.intensity[x_range[0]:x_range[1]]))
block_dimensions = np.array((block_height, block_width), dtype=CG_BLOCK_DIMENSIONS_DATA_TYPE)
offset = np.array((y_range[0], x_range[0]), dtype=CG_OFFSET_DATA_TYPE)
edge = np.empty((block_height*block_width, 2), dtype=CG_EDGE_DATA_TYPE)
dot_product = np.empty(block_height*block_width, dtype=CG_DOT_PRODUCT_DATA_TYPE)
edge_list_size = _cpu_kernel(precursor_mass,
mz,
intensity,
block_dimensions,
offset,
precursor_tolerance,
dot_product_threshold,
num_of_peaks,
edge,
dot_product)
edge = edge[:edge_list_size]
dot_product = dot_product[:edge_list_size]
if abs(precursor_mass[block_height - 1] - precursor_mass[
block_height + block_width - 1]) > precursor_tolerance:
dispatcher.next_row(pid, 0)
with merge_lock:
if edge_list_size != 0:
total_edge_count.value += edge_list_size
edg = np.memmap(str(edg_path), dtype=CG_EDGE_DATA_TYPE, mode='r+', shape=(total_edge_count.value, 2))
dps = np.memmap(str(dps_path), dtype=CG_DOT_PRODUCT_DATA_TYPE, mode='r+', shape=total_edge_count.value)
edg[-edge_list_size:] = edge[:edge_list_size]
dps[-edge_list_size:] = dot_product[:edge_list_size]
except Exception:
err_msg = '\nSubprocess {}: Failed to clustering block (y:{}->{}, x:{}->{}).' \
.format(pid, y_range[0], y_range[1], x_range[0], x_range[1])
with log_lock:
logging.error(err_msg)
raise
with log_lock:
if not exit_signal.value:
logging.debug('Subprocess {}: Reached the end of iteration, work done.'.format(pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.debug('Subprocess {}: Received KeyboardInterrupt, exits now.'.format(pid))
else:
with log_lock:
logging.exception('\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, range_, finish_count, total_count, log_lock, count_lock,
read_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug(
'Cluster refinement subprocess {} started.'.format(pid))
logging.debug('Received range: {}'.format(range_))
raw_clusters = session.clusters.file_path
raw_clusters = sqlite3.connect(str(raw_clusters))
raw_cur = raw_clusters.cursor()
with read_lock:
raw_cur.execute(
'SELECT "pickled" FROM "clusters" ORDER BY "rowid" LIMIT ? OFFSET ?',
(range_[1] - range_[0], range_[0]))
refined_clusters = Path.cwd().joinpath(
session.name + FILE_EXTENSION_TEMPORARY).absolute()
refined_clusters = sqlite3.connect(str(refined_clusters))
refined_cur = refined_clusters.cursor()
# push to database if graphs made storing in RAM exceed GM_BUFFER_SIZE
bytes_count = 0
data = []
while True:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
with read_lock:
clusters = raw_cur.fetchone()
if clusters:
clusters = [pickle.loads(clusters[0])]
else:
_push(pid, refined_clusters, refined_cur, data, log_lock,
merge_lock)
break
for threshold in outlier_threshold:
for i in range(len(clusters)):
clusters[i] = _refinement(clusters[i], threshold)
clusters = [c for sublist in clusters for c in sublist]
for c in clusters:
pickled = pickle.dumps(c)
bytes_count += len(pickled)
data.append((c.num_vertices(), c.num_edges(), pickled))
with count_lock:
finish_count.value += 1
total_count.value += len(clusters)
if bytes_count >= GM_BUFFER_SIZE:
_push(pid, refined_clusters, refined_cur, data, log_lock,
merge_lock)
data.clear()
bytes_count = 0
raw_clusters.close()
refined_clusters.close()
logging.debug(
'Subprocess {}: Reached the end of iteration, work done.'.format(
pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
raw_clusters.close()
refined_clusters.close()
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
raw_clusters.close()
refined_clusters.close()
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, q, valid_iden_count, total_iden_count, finish_count, log_lock,
merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug(
'Identification import subprocess {} started.'.format(pid))
table_path = Path.cwd().joinpath(session.name +
FILE_EXTENSION_IDEN_LUT).absolute()
conn = sqlite3.connect(str(table_path))
cur = conn.cursor()
while True:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
item = q.get()
if item:
file_id, path = item
try:
iden_lut, valid, total = pepxml.import_identification(
path, log_lock=log_lock)
except Exception:
if ignore_errors:
wrn_msg = 'Subprocess {}: Failed to parse identification file: {}'.format(
pid, path)
with log_lock:
logging.warning(wrn_msg)
continue
else:
raise
if valid != 0:
with merge_lock:
for f in iden_lut.keys():
cur.execute('''
CREATE TABLE IF NOT EXISTS "{}" (
"native_id" INTEGER UNIQUE,
"peptide" TEXT,
"charge" INTEGER,
"probability" REAL,
"source" TEXT,
"is_decoy" INTEGER,
"prev_aa" TEXT,
"next_aa" TEXT,
"mods_pos" BLOB,
"mods_mass" BLOB,
"nterm_mod" REAL,
"cterm_mod" REAL,
"iden_file_id" INTEGER,
"l_offset" INTEGER,
"r_offset" INTEGER
)'''.format(f))
conn.commit()
iden_lut = _dict_to_list(iden_lut, file_id)
for f in iden_lut.keys():
cur.execute('BEGIN TRANSACTION')
for id_ in iden_lut[f]:
try:
cur.execute(
'INSERT INTO "{}" VALUES ({})'.format(
f, ', '.join(['?'] * 15)), id_)
except sqlite3.IntegrityError:
if ignore_errors:
wrn_msg = 'Duplicated identification: scan {} in MS file {}.'.format(
id_[0], f)
with log_lock:
logging.warning(wrn_msg)
else:
err_msg = '\nDuplicated identification: scan {} in MS file {}.'\
'\nIdentification file: {}'.format(id_[0], f, path)
with log_lock:
logging.error(err_msg)
conn.commit()
valid_iden_count.value += valid
total_iden_count.value += total
else:
wrn_msg = 'Subprocess {}: Found 0 valid identification in file: {}'.format(
pid, path)
with log_lock:
logging.warning(wrn_msg)
finish_count.value += 1
else:
logging.debug(
'Subprocess {}: Got None object, work done.'.format(pid))
break
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, update, range_, finish_count, log_lock, count_lock,
iden_read_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug(
'Cluster enrichment subprocess {} started.'.format(pid))
logging.debug('Received range: {}'.format(range_))
clusters_conn = sqlite3.connect(str(session.clusters.file_path))
clusters_cur = clusters_conn.cursor()
session.iden_lut.connect()
clusters_cur.execute(
'SELECT "cluster_id", "num_idens", "pickled" FROM "clusters" ORDER BY "rowid" LIMIT ? OFFSET ?',
(range_[1] - range_[0], range_[0]))
bytes_count = 0
data_no_iden = []
data_with_iden = []
while True:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
cluster = clusters_cur.fetchone()
if cluster is None:
_push(pid, clusters_conn, clusters_cur, data_no_iden,
data_with_iden, log_lock, merge_lock)
break
_add_iden_stat(cluster, data_no_iden, data_with_iden, bytes_count,
update, iden_read_lock)
with count_lock:
finish_count.value += 1
if bytes_count >= GM_BUFFER_SIZE:
_push(pid, clusters_conn, clusters_cur, data_no_iden,
data_with_iden, log_lock, merge_lock)
data_no_iden.clear()
data_with_iden.clear()
bytes_count = 0
clusters_conn.close()
session.iden_lut.disconnect()
logging.debug(
'Subprocess {}: Reached the end of iteration, work done.'.format(
pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
clusters_conn.close()
session.iden_lut.disconnect()
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
clusters_conn.close()
session.iden_lut.disconnect()
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, dispatcher, offsets, log_lock, merge_lock, exit_signal):
try:
logging_setup()
with log_lock:
logging.debug(
'Rank transformation subprocess {} started.'.format(pid))
file_path = Path.cwd().joinpath(
session.name + FILE_EXTENSION_RANKED_SPECTRA).resolve()
index_length = len(session.internal_index)
mz_arr = np.memmap(str(file_path), offset=offsets[0], dtype=RT_MZ_DATA_TYPE, mode='r+',\
shape=(index_length, num_of_peaks))
intensity_arr = np.memmap(str(file_path), offset=offsets[1], dtype=RT_INTENSITY_DATA_TYPE, mode='r+',\
shape=(index_length, num_of_peaks))
for chunk in dispatcher:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
try:
start, end = chunk
size = end - start
with log_lock:
logging.debug('Subprocess {}: Transforming spectra {} - {} with size of {}.'\
.format(pid, start, end, size))
temp_index = session.internal_index[start:end]
temp_mz = np.empty((size, num_of_peaks), dtype=np.int32)
temp_intensity = np.empty((size, num_of_peaks),
dtype=np.float32)
for i in range(size):
file, offset = session.ms_exp_files[
temp_index.file_id[i]], temp_index.offset[i]
precursor_mass, precursor_charge = temp_index.precursor_mass[
i], temp_index.precursor_charge[i]
format_ = ''.join(file.suffixes).lower()
if format_ == '.mzxml':
peaks = mzxml.get_peaks(file, offset, log_lock)
elif format_ == '.mzml':
peaks = mzml.get_peaks(file, offset, log_lock)
temp_mz[i], temp_intensity[i] = _rank_transform(
precursor_mass, precursor_charge, peaks[0], peaks[1],
log_lock)
with merge_lock:
mz_arr[start:end], intensity_arr[
start:end] = temp_mz, temp_intensity
mz_arr.flush()
intensity_arr.flush()
except Exception:
err_msg = '\nSubprocess {}: Failed to transform spectrum located at {}'.format(
pid, start + i)
with log_lock:
logging.error(err_msg)
raise
with log_lock:
logging.debug(
'Subprocess {}: Reached the end of iteration, work done.'.
format(pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return
def _worker(pid, scan_starting_idx, temp_storage, edge_list_length,
finish_count, log_lock, scan_lock, count_lock, merge_lock,
exit_signal):
try:
logging_setup()
with log_lock:
logging.debug('Graph making subprocess {} started.'.format(pid))
clusters_path = Path.cwd().joinpath(
session.name + FILE_EXTENSION_CLUSTERS).absolute()
conn = sqlite3.connect(str(clusters_path))
cur = conn.cursor()
edg_path = temp_storage.joinpath('edg')
dps_path = temp_storage.joinpath('dps')
nbg_path = temp_storage.joinpath('nbg')
edge = np.memmap(str(edg_path),
dtype=CG_EDGE_DATA_TYPE,
mode='c',
shape=(edge_list_length, 2))
dps = np.memmap(str(dps_path),
dtype=CG_DOT_PRODUCT_DATA_TYPE,
mode='c',
shape=edge_list_length)
belonging = np.memmap(str(nbg_path),
dtype=FG_PARENT_DATA_TYPE,
mode='c',
shape=edge_list_length)
# push to database if graphs made storing in RAM exceed GM_BUFFER_SIZE
bytes_count = 0
local_finish_count = 0
data = []
while True:
if exit_signal.value:
with log_lock:
logging.debug(
'Subprocess {}: Received exit signal, exits now.'.
format(pid))
break
with scan_lock:
start = scan_starting_idx.value
end = _scan(belonging, start)
scan_starting_idx.value = end
if start == end:
_push(pid, conn, cur, data, log_lock, merge_lock)
with count_lock:
finish_count.value += local_finish_count
break
temp_graph = gt.Graph(directed=False)
temp_graph.vp['iid'] = temp_graph.add_edge_list(edge[start:end],
hashed=True)
temp_graph.ep['dps'] = dps_prop = temp_graph.new_ep('float')
dps_prop.a = dps[start:end]
temp_graphs = []
if use_dbscan:
from ClusterSheep.share.dbscan_point_type import DbscanPointType
noise, border, core = DbscanPointType.NOISE, DbscanPointType.BORDER, DbscanPointType.CORE
edge_degrees = temp_graph.get_total_degrees(
temp_graph.get_vertices())
point_types = np.full(edge_degrees.shape,
noise,
dtype=np.int32)
point_types[edge_degrees >= (dbscan_min_points - 1)] = core
for idx in np.where(point_types == noise)[0]:
neighbors = temp_graph.get_all_neighbors(idx)
neighbor_is_core = point_types[neighbors] == core
if np.any(neighbor_is_core):
point_types[idx] = border
temp_graph.vp['dst'] = dst_prop = temp_graph.new_vp('int')
dst_prop.a = point_types
vfilt = point_types == core
graph_core_only = gt.GraphView(temp_graph, vfilt=vfilt)
comp, hist = gt.label_components(graph_core_only)
comp.a[point_types == noise] = -1
for idx in np.where(point_types == border)[0]:
neighbors = temp_graph.get_all_neighbors(idx)
first_core_neighbor = neighbors[point_types[neighbors] ==
core][0]
clu_idx = comp.a[first_core_neighbor]
comp.a[idx] = clu_idx
hist[clu_idx] += 1
for i in range(len(hist)):
if hist[i] > 1:
temp_graphs.append(
gt.Graph(gt.GraphView(temp_graph,
vfilt=(comp.a == i)),
prune=True,
directed=False))
else:
temp_graphs.append(temp_graph)
for tg in temp_graphs:
pickled = pickle.dumps(tg)
bytes_count += len(pickled)
data.append((tg.num_vertices(), tg.num_edges(), pickled))
local_finish_count += 1
if local_finish_count >= 1000:
with count_lock:
finish_count.value += local_finish_count
local_finish_count = 0
if bytes_count >= GM_BUFFER_SIZE:
_push(pid, conn, cur, data, log_lock, merge_lock)
data.clear()
bytes_count = 0
conn.close()
logging.debug(
'Subprocess {}: Reached the end of iteration, work done.'.format(
pid))
except (Exception, KeyboardInterrupt) as e:
if type(e) is KeyboardInterrupt:
conn.close()
with log_lock:
logging.debug(
'Subprocess {}: Received KeyboardInterrupt, exits now.'.
format(pid))
else:
conn.close()
with log_lock:
logging.exception(
'\nSubprocess {}: Ended unexpectedly. Logging traceback:\n'
'==========TRACEBACK==========\n'.format(pid))
exit_signal.value = True
sys.exit(1)
return