def calc_null_template(labels, data, clf):
# Find maximum allowed folds for cross validation
un, counts = np.unique(labels, return_counts=True)
max_folds = 10
min_folds = 2
folds = np.min([max_folds, np.max([min_folds, np.min(counts)])])
print "Calculating null classification error, {} classes, {} samples".format(
len(un), len(labels))
# Loop through each operation in a threaded manner
def process_task_threaded(i):
null_reps = 1000
op_errs = np.zeros(null_reps)
for j in range(null_reps):
# Shuffle labels
random.shuffle(labels)
# data is type float64 by default but Decision tree classifier works with float32
operation = np.float32(data[:, i])
# Reshape data as we have only one feature at a time
operation = operation.reshape(-1, 1)
# Split into training and test data
t_size = 1 / float(folds)
op_train, op_test, labels_train, labels_test = train_test_split(
operation, labels, test_size=t_size, random_state=23)
op_train = op_train.reshape(-1, 1)
op_test = op_test.reshape(-1, 1)
# Fit classifier on training data
use_clf = copy.deepcopy(clf)
use_clf = use_clf.fit(op_train, labels_train)
# Calculate accuracy on test data
labels_test_predicted = use_clf.predict(op_test)
op_errs[j] = 1 - accuracy_score(labels_test, labels_test_predicted)
return op_errs
random.seed(25)
pool = Pool(processes=8)
error_rates = pool.map(process_task_threaded, range(data.shape[1]))
op_error_rates = np.vstack(error_rates)
mean_error_rates = np.mean(op_error_rates, axis=1)
print "Mean classification error is {}".format(np.mean(mean_error_rates))
return (op_error_rates, mean_error_rates)
def finally_ip():
(ipList,portList) = get_ip_list(url,headers)
#运行pool = ThreadPool(2)有时会出现module '__main__' has no attribute '__spec__'错误 不造如何解决
#尝试过 __spec__=None 的方式 没有什么用
pool = ThreadPool(4)
start_time = time.time()
results = pool.map(test_ip,ipList,portList)
pool.close()
pool.join()
end_time = time.time()
print("并行耗时："+str(end_time-start_time))
return results
def get_american_life_remaining( ):
"""
This downloads *remaining* `This American Life`_ episodes. To determine missing episodes, it first finds the maximum episode number that we have downloaded. It subtracts the episodes we have downloaded from the integer list that runs from 1 to the maximum episode number. Then it downloads these remaining episodes *in parallel*.
"""
#
## get all track numbers, and find what's left
episodes_here = set(filter(None, map(
_get_tal_track, glob.glob( os.path.join(
_default_inputdir, 'PRI.ThisAmericanLife.*.mp3' ) ) ) ) )
episodes_remaining = set(range(1, max( episodes_here ) + 1 ) ) - episodes_here
if len( episodes_remaining ) == 0:
return
with ThreadPool( processes = max(1, cpu_count( ) // 2 ) ) as pool:
time0 = time.time( )
_ = list( pool.map( get_american_life, episodes_remaining ) )
logging.debug( 'was able to download %d missing TAL episodes (%s) in %0.3f seconds.' % (
len( episodes_remaining ), ', '.join(sorted(episodes_remaining)),
time.time( ) - time0 ) )
def process_patient(germline_sample_id):
# Find all non-tumor bulk VCF files for the patient ID.
germline_wb_vcf_paths = list(
control_path.glob(germline_sample_id + "_*_*.vcf"))
# Fetch all cell IDs associated with the patient ID.
experimental_sample_ids = metadata_df.loc[
metadata_df["germline_sample_id"] ==
germline_sample_id]["experimental_sample_id"]
# Use the cell IDs to create a list of all single-cell VCF files
# for the patient.
cell_vcf_paths = [(experimental_path / experimental_sample_id).
with_suffix(".vcf") for experimental_sample_id
in experimental_sample_ids]
# Create a genome interval tree for the patient's germline bulk VCF
# data. Only selects one germline VCF to avoid over-filtering for
# patients with multiple germline VCFs.
germline_tree = create_germline_genome_tree(germline_wb_vcf_paths[0:1])
def process_cell(cell_vcf_path):
if not cell_vcf_path.exists():
return
# If there were any germline VCFs for this patient, append `GF_` to
# the file name to indicate that the output VCF was
# germline-filtered, not just dbSNP-filtered.
out_name_prefix = "" if len(germline_wb_vcf_paths) < 1 else "GF_"
out_vcf_path = out_path / (out_name_prefix + cell_vcf_path.name)
with open(cell_vcf_path, mode='r') as in_file:
with open(out_vcf_path, mode='w') as out_file:
write_filtered_vcf(in_file, germline_tree, out_file)
# TODO: Maybe remove this in Python 3.8.
# This thread pool max-worker count is from the implementation in
# Python 3.8. Assuming that Pathos adopts the same semantics,
# this can be removed.
with ThreadPool(min(32, os.cpu_count() + 4)) as pool:
pool.map(process_cell, cell_vcf_paths)
para_vec = []
for t in para_tokens:
if t in glove.index:
para_vec.append(glove.loc[t].as_matrix())
embed_vecs[para] = para_vec
parser = argparse.ArgumentParser(description='Create Glove paragraph embedding from a list of paragraphs.')
parser.add_argument("-g", "--glove_file", required=True, help="Pretrained Glove file")
parser.add_argument("-pl", "--para_list", required=True, help="List of paragraphs")
parser.add_argument("-pt", "--tokenized_para", required=True, help="Tokenized para file")
parser.add_argument("-o", "--out", required=True, help="Output file")
parser.add_argument("-pn", "--no_processes", type=int, required=True, help="No of parallel processes")
args = vars(parser.parse_args())
glove_file = args["glove_file"]
para_list = args["para_list"]
tokenized_para_file = args["tokenized_para"]
out_file = args["out"]
pno = args["no_processes"]
glove = pd.read_table(glove_file, sep=" ", index_col=0, header=None, quoting=csv.QUOTE_NONE)
with open(para_list, 'r') as pl:
paras = json.load(pl)
tokenized_para = np.load(tokenized_para_file, allow_pickle=True)
print("Data loaded")
print(str(len(paras))+" total paras")
embed_vecs = dict()
with ThreadPool(pno) as pool:
pool.map(construct_para_embedding, paras)
np.save(out_file, embed_vecs)
class Infer(object):
def __init__(self,
in_scale=1,
out_scale=1,
batch_size=8,
width=224,
n_threads=1,
model_directory="",
name="",
offset=1536,
voxel_offset=(0, 0),
cloud_volume=True,
normalization=True,
crop_size=20,
use_blend_map=False,
to_crop=True,
features={
"inputs": "",
"outputs": ""
},
chunk_size_z=64,
output_dim=1):
#Global Parameters
self.offset = offset
self.voxel_offset = voxel_offset
self.chunk_size_z = chunk_size_z
self.in_scale = in_scale
self.out_scale = out_scale
self.width = width
self.pool = ThreadPool(n_threads)
self.model = Graph(directory=model_directory, name=name)
self.blend_map = ip.get_blend_map(self.width / 2, self.width) #FIXME
self.blend_map = np.repeat(np.expand_dims(self.blend_map, -1),
output_dim,
axis=2)
self.batch_size = batch_size
self.cloud_volume = cloud_volume
self.normalization = normalization
self.crop_size = crop_size
self.use_blend_map = use_blend_map
self.to_crop = to_crop
self.features = features
self.output_dim = output_dim
def read(self, args):
(volume, (x, y, z), (off_x, off_y, off_z), i) = args
image = ip.read_without_borders_3d(
volume.vol,
(x + self.voxel_offset[0], y + self.voxel_offset[1], z),
(off_x, off_y, off_z),
scale_ratio=(1.0 / self.scale, 1.0 / self.scale, 1),
voxel_offset=self.voxel_offset)
image = image / 255.0
image = image[:, :, 0]
return image
# Process by batches
def process_old(self, volume, x_y_z, mset):
(x, y, z) = x_y_z
t1 = time.time()
inputs = self.pool.map(
self.read,
[(volume, (x + i * self.scale * self.width / 2, y, 0),
(self.scale * self.width, self.scale * self.width, 1), i)
for i in range(8)])
inputs = np.array(inputs)
t2 = time.time()
print(t2 - t1, 'downloading')
images = self.model.process({"input/image:0": inputs},
["Passes/image_transformed:0"])
t3 = time.time()
images = images[0]
print(t3 - t2, 'processing')
images = [images[i, :, :, 0] for i in range(8)]
images_new = self.pool.map(self.post_process, images) #
i = 0
for image in images_new:
x_temp = x + i * self.scale * self.width / 2
mset[x_temp + self.offset:x_temp + self.offset + image.shape[0],
y + self.offset:y + self.offset +
image.shape[1]] += image[:, :]
i += 1
t4 = time.time()
print(t4 - t3, 'saving')
return mset
'''
def post_process(self, image):
if self.normalization:
image = ip.normalize(image)
image = np.multiply(image, self.blend_map)
image = (ip.resize(image, (self.scale, self.scale), order=0)).astype(np.uint8)
return image
'''
def store(self, output_volume, mset, shape_origin, z):
t1 = time.time()
if self.cloud_volume:
output_volume.vol[:, :, z] = np.expand_dims(
mset[self.offset:self.offset + shape_origin[0],
self.offset:self.offset + shape_origin[1]],
axis=2)
else:
output_volume[:, :] = mset[self.offset:self.offset +
shape_origin[0],
self.offset:self.offset +
shape_origin[1]]
t2 = time.time()
print('saving slice', t2 - t1)
def process_slice(self, input_volume, output_volume, z=0):
step_x = self.batch_size * self.scale * self.width / 2
step_y = self.scale * self.width / 2
shape_origin = np.array(input_volume.shape[0:2])
shape = shape_origin + 2 * self.offset
mset = np.zeros(shape, np.uint8)
for x in range(shape[0] // (step_x)):
for y in range(shape[1] // (step_y) - 1):
print(x * step_x + self.offset,
x * step_x + step_x + self.offset,
y * step_y + self.offset,
y * step_y + step_y + self.offset)
t1 = time.time()
mset = self.process(
input_volume,
(x * step_x - self.offset, y * step_y - self.offset, z),
mset)
t2 = time.time()
print(
str(x) + '/' + str(shape[0] // (step_x) - 1),
str(y) + '/' + str(shape[1] // (step_y) - 1), t2 - t1)
self.store(output_volume, mset, shape_origin, z)
'''Optimized inference'''
def read2d(self, args):
(volume, (x, y), step) = args
image = ip.read_without_borders_2d(volume, (x, y), (step, step),
scale_ratio=(1.0 / self.in_scale,
1.0 / self.in_scale))
image = image / 255.0
image = image[:, :]
return image
def read3d(self, args):
(volume, (x, y), step) = args
image = ip.read_without_borders_3d(np.expand_dims(volume,
-1), (x, y, 0),
(step, step, volume.shape[2]))
image = image / 255.0
image = image[:, :]
return image
def post_process(self, image):
if self.normalization and False:
image = ip.normalize(image)
if self.use_blend_map:
image = np.multiply(image, self.blend_map)
if self.to_crop:
iamge = ip.black_crop(image, self.crop_size)
image = ip.resize(image, (self.in_scale, self.in_scale), order=0)
#image = image.astype(np.uint8)
return image
def process_core(self, images):
images = self.model.process(
{self.features['inputs']: np.expand_dims(images, -1)},
[self.features['outputs']])
images = images[0]
#images = np.squeeze(images)
images = [images[i, :, :] for i in range(self.batch_size)]
images = self.pool.map(self.post_process, images)
return images
def write(self, mset, images, coords, step):
i = 0
crop = self.in_scale * self.crop_size
ishape = images[0].shape[0] - crop
for i in range(len(coords)):
if coords[i][0] == -1:
break
begin = (coords[i][0] + crop, coords[i][1] + crop)
end = (coords[i][0] + ishape, coords[i][1] + ishape)
mset[begin[0]:end[0], begin[1]:end[1]] += images[i][crop:ishape,
crop:ishape]
return mset
'''
Given input returns processed output within the same shape
This process encounters batchification and overlapped cropping
'''
def process(self, input_volume):
#Input organization
shape_origin = np.array(input_volume.shape)
width = self.in_scale * self.width
#print()
shape = [
shape_origin[0] + width, shape_origin[1] + width, self.output_dim
]
mset = np.zeros(shape, np.float32)
step = width - 2 * self.in_scale * self.crop_size
grid = ip.get_grid(shape_origin + self.in_scale * self.crop_size,
step=step,
batch_size=self.batch_size)
i = 0
read = self.read2d
if len(input_volume.shape) == 3:
read = self.read3d
for batch in grid:
t1 = time.time()
images = self.pool.map(read,
[(input_volume, np.array(coord) -
self.in_scale * self.crop_size, width)
for coord in batch])
images = self.process_core(images)
#print(mset.shape, images[0].shape)
mset = self.write(mset, images, batch, step)
t2 = time.time()
#print(str(i)+'/'+str(len(grid)), t2-t1)
i += 1
mset = mset[self.in_scale *
self.crop_size:self.in_scale * self.crop_size +
shape_origin[0], self.in_scale *
self.crop_size:self.in_scale * self.crop_size +
shape_origin[1]]
if self.normalization:
mset = ip.normalize(mset).astype(np.uint8)
return mset
'''
Locations = [(begin, end)]
where begin = [x,y,z] and end=[x',y',z]
'''
#FIXME Add threaded download/upload
#FIXME clean up infer.py
def process_by_superbatch(self, input_volume, output_volume, locations):
crop = self.in_scale * self.crop_size
i = 0
# Make this multithreaded
for begin, end in locations:
print(begin, end)
t1 = time.time()
data = input_volume.vol[begin[0] - crop:end[0] + crop,
begin[1] - crop:end[1] + crop,
begin[-1]][:, :, 0, 0]
t2 = time.time()
print(t2 - t1, 'download')
output = self.process(data)
t3 = time.time()
output = ip.resize(
output[crop:crop + end[0] - begin[0],
crop:crop + end[1] - begin[1]],
(self.out_scale, self.out_scale))
t4 = time.time()
print(t3 - t2, 'process')
output_volume.vol[self.out_scale * begin[0]:self.out_scale *
end[0], self.out_scale *
begin[1]:self.out_scale * end[1],
begin[-1]] = np.expand_dims(output, axis=2)
t5 = time.time()
print(t5 - t4, 'upload')
# Clear cache after processing all 64
i += 1
if i % self.chunk_size_z == 0:
i == 0
#input_volume.vol.flush_cache()
def reactive_process(self, input_volume, output_volume, locations):
crop = self.in_scale * self.crop_size
import rx
from rx import Observable, Observer
scheduler = rx.concurrency.NewThreadScheduler()
def download(args):
(begin, end) = args
print((begin, end))
return (np.zeros((190, 96)), begin, end)
return (input_volume.vol[begin[0]-crop:end[0]+crop,\
begin[1]-crop:end[1]+crop,\
begin[-1]][:,:,0,0],\
begin,end)
def upload(args):
args = (output, begin, end)
return 0
output_volume.vol[self.out_scale * begin[0]:self.out_scale *
end[0], self.out_scale *
begin[1]:self.out_scale * end[1],
begin[-1]] = np.expand_dims(
output[crop:crop + end[0] - begin[0],
crop:crop + end[1] - begin[1]],
axis=2)
def process(output):
#output.map(self.process).map(upload).subscribe(on_next=lambda e: print(e),on_error = lambda e: print(e))
#print(output[0])
t1 = time.time()
def fib(x):
if x == 0 or x == 1:
return 1
return fib(x - 1) + fib(x - 2)
k = fib(40)
print(42)
t2 = time.time()
return output
def down(value):
def go_make_big(subscriber):
subscriber.on_next(download(value))
subscriber.on_completed()
return rx.Observable.create(go_make_big)
t1 = time.time()
pipe = Observable.from_(locations)
pipe.flat_map(lambda x: down(x).subscribe_on(scheduler))\
.map(process)\
.map(upload)\
.subscribe(
on_completed=lambda: print("PROCESS 1 done!"),
on_error = lambda e: print(e))
t2 = time.time()
print('overall', t2 - t1)
from flask import Flask
from scipy.io import loadmat, savemat
from scipy.ndimage import imread
from flask import request
import json
app = Flask(__name__)
output_rect_path = '/tmp/output_rect.mat'
import cavelab as cl
from evaluation import doncc, process_ncc
import numpy as np
n_threads = 40
from pathos.multiprocessing import ProcessPool, ThreadPool
pool = ThreadPool(n_threads)
@app.route('/process_NCC')
def process_NCC():
frames_path = request.args.get('frames_path')
frames = loadmat(frames_path)['frames']
rect_path = request.args.get('rect_path')
init_rect = loadmat(rect_path)['rect'][0]
out_rects = []
print(init_rect)
W = int(init_rect[3])
H = int(init_rect[2])
image_data = imread(frames[0][0][0])
if len(image_data.shape) == 2:
image_data = np.expand_dims(image_data, -1)
class Aligner:
def __init__(self,
model_path,
max_displacement,
crop,
mip_range,
high_mip_chunk,
src_ng_path,
dst_ng_path,
render_low_mip=2,
render_high_mip=8,
is_Xmas=False,
threads=10,
max_chunk=(1024, 1024),
max_render_chunk=(2048 * 2, 2048 * 2),
queue_name=None,
gpu=False):
if queue_name != None:
self.task_handler = TaskHandler(queue_name)
self.distributed = True
else:
self.task_handler = None
self.distributed = False
self.threads = 10
self.process_high_mip = mip_range[1]
self.process_low_mip = mip_range[0]
self.render_low_mip = render_low_mip
self.render_high_mip = render_high_mip
self.high_mip = max(self.render_high_mip, self.process_high_mip)
self.high_mip_chunk = high_mip_chunk
self.max_chunk = max_chunk
self.max_render_chunk = max_render_chunk
self.max_displacement = max_displacement
self.crop_amount = crop
self.org_ng_path = src_ng_path
self.src_ng_path = self.org_ng_path
self.dst_ng_path = os.path.join(dst_ng_path, 'image')
self.tmp_ng_path = os.path.join(dst_ng_path, 'intermediate')
self.res_ng_paths = [
os.path.join(dst_ng_path, 'vec/{}'.format(i))
for i in range(self.process_high_mip + 10)
] #TODO
self.x_res_ng_paths = [os.path.join(r, 'x') for r in self.res_ng_paths]
self.y_res_ng_paths = [os.path.join(r, 'y') for r in self.res_ng_paths]
self.net = Process(model_path, is_Xmas=is_Xmas, cuda=gpu)
self.dst_chunk_sizes = []
self.dst_voxel_offsets = []
self.vec_chunk_sizes = []
self.vec_voxel_offsets = []
self.vec_total_sizes = []
self._create_info_files(max_displacement)
self.pool = ThreadPool(threads)
#if not chunk_size[0] :
# raise Exception("The chunk size has to be aligned with ng chunk size")
def set_chunk_size(self, chunk_size):
self.high_mip_chunk = chunk_size
def _create_info_files(self, max_offset):
tmp_dir = "/tmp/{}".format(os.getpid())
nocache_f = '"Cache-Control: no-cache"'
os.system("mkdir {}".format(tmp_dir))
src_info = cv(self.src_ng_path).info
dst_info = deepcopy(src_info)
##########################################################
#### Create dest info file
##########################################################
chunk_size = dst_info["scales"][0]["chunk_sizes"][0][0]
dst_size_increase = max_offset
if dst_size_increase % chunk_size != 0:
dst_size_increase = dst_size_increase - (dst_size_increase %
max_offset) + chunk_size
scales = dst_info["scales"]
for i in range(len(scales)):
scales[i]["voxel_offset"][0] -= int(dst_size_increase / (2**i))
scales[i]["voxel_offset"][1] -= int(dst_size_increase / (2**i))
scales[i]["size"][0] += int(dst_size_increase / (2**i))
scales[i]["size"][1] += int(dst_size_increase / (2**i))
x_remainder = scales[i]["size"][0] % scales[i]["chunk_sizes"][0][0]
y_remainder = scales[i]["size"][1] % scales[i]["chunk_sizes"][0][1]
x_delta = 0
y_delta = 0
if x_remainder != 0:
x_delta = scales[i]["chunk_sizes"][0][0] - x_remainder
if y_remainder != 0:
y_delta = scales[i]["chunk_sizes"][0][1] - y_remainder
scales[i]["size"][0] += x_delta
scales[i]["size"][1] += y_delta
scales[i]["size"][0] += int(dst_size_increase / (2**i))
scales[i]["size"][1] += int(dst_size_increase / (2**i))
#make it slice-by-slice writable
scales[i]["chunk_sizes"][0][2] = 1
self.dst_chunk_sizes.append(scales[i]["chunk_sizes"][0][0:2])
self.dst_voxel_offsets.append(scales[i]["voxel_offset"])
cv(self.dst_ng_path, info=dst_info).commit_info()
cv(self.tmp_ng_path, info=dst_info).commit_info()
##########################################################
#### Create vec info file
##########################################################
vec_info = deepcopy(src_info)
vec_info["data_type"] = "float32"
scales = deepcopy(vec_info["scales"])
for i in range(len(scales)):
self.vec_chunk_sizes.append(scales[i]["chunk_sizes"][0][0:2])
self.vec_voxel_offsets.append(scales[i]["voxel_offset"])
self.vec_total_sizes.append(scales[i]["size"])
cv(self.x_res_ng_paths[i], info=vec_info).commit_info()
cv(self.y_res_ng_paths[i], info=vec_info).commit_info()
def check_all_params(self):
return True
def get_upchunked_bbox(self, bbox, ng_chunk_size, offset, mip):
raw_x_range = bbox.x_range(mip=mip)
raw_y_range = bbox.y_range(mip=mip)
x_chunk = ng_chunk_size[0]
y_chunk = ng_chunk_size[1]
x_offset = offset[0]
y_offset = offset[1]
x_remainder = ((raw_x_range[0] - x_offset) % x_chunk)
y_remainder = ((raw_y_range[0] - y_offset) % y_chunk)
x_delta = 0
y_delta = 0
if x_remainder != 0:
x_delta = x_chunk - x_remainder
if y_remainder != 0:
y_delta = y_chunk - y_remainder
calign_x_range = [raw_x_range[0] + x_delta, raw_x_range[1]]
calign_y_range = [raw_y_range[0] + y_delta, raw_y_range[1]]
x_start = calign_x_range[0] - x_chunk
y_start = calign_y_range[0] - y_chunk
x_start_m0 = x_start * 2**mip
y_start_m0 = y_start * 2**mip
result = BoundingBox(x_start_m0,
x_start_m0 + bbox.x_size(mip=0),
y_start_m0,
y_start_m0 + bbox.y_size(mip=0),
mip=0,
max_mip=self.process_high_mip)
return result
def break_into_chunks(self,
bbox,
ng_chunk_size,
offset,
mip,
render=False):
chunks = []
raw_x_range = bbox.x_range(mip=mip)
raw_y_range = bbox.y_range(mip=mip)
x_chunk = ng_chunk_size[0]
y_chunk = ng_chunk_size[1]
x_offset = offset[0]
y_offset = offset[1]
x_remainder = ((raw_x_range[0] - x_offset) % x_chunk)
y_remainder = ((raw_y_range[0] - y_offset) % y_chunk)
x_delta = 0
y_delta = 0
if x_remainder != 0:
x_delta = x_chunk - x_remainder
if y_remainder != 0:
y_delta = y_chunk - y_remainder
calign_x_range = [raw_x_range[0] - x_remainder, raw_x_range[1]]
calign_y_range = [raw_y_range[0] - y_remainder, raw_y_range[1]]
x_start = calign_x_range[0] - x_chunk
y_start = calign_y_range[0] - y_chunk
if (self.process_high_mip > mip):
high_mip_scale = 2**(self.process_high_mip - mip)
else:
high_mip_scale = 1
processing_chunk = (int(self.high_mip_chunk[0] * high_mip_scale),
int(self.high_mip_chunk[1] * high_mip_scale))
if not render and (processing_chunk[0] > self.max_chunk[0]
or processing_chunk[1] > self.max_chunk[1]):
processing_chunk = self.max_chunk
elif render and (processing_chunk[0] > self.max_render_chunk[0]
or processing_chunk[1] > self.max_render_chunk[1]):
processing_chunk = self.max_render_chunk
for xs in range(calign_x_range[0], calign_x_range[1],
processing_chunk[0]):
for ys in range(calign_y_range[0], calign_y_range[1],
processing_chunk[1]):
chunks.append(
BoundingBox(xs,
xs + processing_chunk[0],
ys,
ys + processing_chunk[0],
mip=mip,
max_mip=self.high_mip))
return chunks
## Residual computation
def run_net_test(self, s, t, mip):
abs_residual = self.net.process(s, t, mip)
def compute_residual_patch(self, source_z, target_z, out_patch_bbox, mip):
#print ("Computing residual for {}".format(out_patch_bbox.__str__(mip=0)),
# end='', flush=True)
precrop_patch_bbox = deepcopy(out_patch_bbox)
precrop_patch_bbox.uncrop(self.crop_amount, mip=mip)
if mip == self.process_high_mip:
src_patch = data_handler.get_image_data(self.src_ng_path, source_z,
precrop_patch_bbox, mip)
else:
src_patch = data_handler.get_image_data(self.tmp_ng_path, source_z,
precrop_patch_bbox, mip)
tgt_patch = data_handler.get_image_data(self.dst_ng_path, target_z,
precrop_patch_bbox, mip)
abs_residual = self.net.process(src_patch,
tgt_patch,
mip,
crop=self.crop_amount)
#rel_residual = precrop_patch_bbox.spoof_x_y_residual(1024, 0, mip=mip,
# crop_amount=self.crop_amount)
data_handler.save_residual_patch(abs_residual, source_z,
out_patch_bbox, mip,
self.x_res_ng_paths,
self.y_res_ng_paths)
## Patch manipulation
def warp_patch(self, ng_path, z, bbox, res_mip_range, mip):
influence_bbox = deepcopy(bbox)
influence_bbox.uncrop(self.max_displacement, mip=0)
agg_flow = influence_bbox.identity(mip=mip)
agg_flow = np.expand_dims(agg_flow, axis=0)
agg_res = data_handler.get_aggregate_rel_flow(
z, influence_bbox, res_mip_range, mip, self.process_low_mip,
self.process_high_mip, self.x_res_ng_paths, self.y_res_ng_paths)
agg_flow += agg_res
raw_data = data_handler.get_image_data(ng_path, z, influence_bbox, mip)
#no need to warp if flow is identity
#warp introduces noise
if not influence_bbox.is_identity_flow(agg_flow, mip=mip):
warped = warp(raw_data, agg_flow)
else:
#print ("not warping")
warped = raw_data[0]
mip_disp = int(self.max_displacement / 2**mip)
cropped = crop(warped, mip_disp)
result = data_handler.preprocess_data(cropped * 256)
#preprocess divides by 256 and puts it into right dimensions
#this data range is good already, so mult by 256
data_handler.save_image_patch(self.dst_ng_path, result, z, bbox, mip)
def downsample_patch(self, ng_path, z, bbox, mip):
in_data = data_handler.get_image_data(ng_path, z, bbox, mip - 1)
result = downsample_mip(in_data)
return result
## High level services
def copy_section(self, source, dest, z, bbox, mip):
print("moving section {} mip {} to dest".format(z, mip),
end='',
flush=True)
start = time()
chunks = self.break_into_chunks(bbox,
self.dst_chunk_sizes[mip],
self.dst_voxel_offsets[mip],
mip=mip,
render=True)
#for patch_bbox in chunks:
if self.distributed and len(chunks) > self.threads * 2:
for i in range(0, len(chunks), self.threads):
task_patches = []
for j in range(i, min(len(chunks), i + self.threads)):
task_patches.append(chunks[j])
copy_task = make_copy_task_message(z,
source,
dest,
task_patches,
mip=mip)
self.task_handler.send_message(copy_task)
while not self.task_handler.is_empty():
sleep(1)
else:
def chunkwise(patch_bbox):
raw_patch = data_handler.get_image_data(
source, z, patch_bbox, mip)
data_handler.save_image_patch(dest, raw_patch, z, patch_bbox,
mip)
self.pool.map(chunkwise, chunks)
end = time()
print(": {} sec".format(end - start))
def prepare_source(self, z, bbox, mip):
print("Prerendering mip {}".format(mip), end='', flush=True)
start = time()
chunks = self.break_into_chunks(bbox,
self.dst_chunk_sizes[mip],
self.dst_voxel_offsets[mip],
mip=mip,
render=True)
#for patch_bbox in chunks:
def chunkwise(patch_bbox):
self.warp_patch(self.src_ng_path, z, patch_bbox,
(mip + 1, self.process_high_mip), mip)
self.pool.map(chunkwise, chunks)
end = time()
print(": {} sec".format(end - start))
def render(self, z, bbox, mip):
print("Rendering mip {}".format(mip), end='', flush=True)
start = time()
chunks = self.break_into_chunks(bbox,
self.dst_chunk_sizes[mip],
self.dst_voxel_offsets[mip],
mip=mip,
render=True)
if self.distributed:
for i in range(0, len(chunks), self.threads):
task_patches = []
for j in range(i, min(len(chunks), i + self.threads)):
task_patches.append(chunks[j])
render_task = make_render_task_message(z,
task_patches,
mip=mip)
self.task_handler.send_message(render_task)
while not self.task_handler.is_empty():
sleep(1)
else:
def chunkwise(patch_bbox):
print("Rendering {} at mip {}".format(
patch_bbox.__str__(mip=0), mip),
end='',
flush=True)
self.warp_patch(self.src_ng_path, z, patch_bbox,
(mip, self.process_high_mip), mip)
self.pool.map(chunkwise, chunks)
end = time()
print(": {} sec".format(end - start))
def render_section_all_mips(self, z, bbox):
#total_bbox = self.get_upchunked_bbox(bbox, self.dst_chunk_sizes[self.process_high_mip],
# self.dst_voxel_offsets[self.process_high_mip],
# mip=self.process_high_mip)
self.render(z, bbox, self.render_low_mip)
self.downsample(z, bbox, self.render_low_mip, self.render_high_mip)
def downsample(self, z, bbox, source_mip, target_mip):
for m in range(source_mip + 1, target_mip + 1):
print("Downsampleing mip {}: ".format(m), end='', flush=True)
start = time()
chunks = self.break_into_chunks(bbox,
self.dst_chunk_sizes[m],
self.dst_voxel_offsets[m],
mip=m,
render=True)
#for patch_bbox in chunks:
if self.distributed and len(chunks) > self.threads * 2:
for i in range(0, len(chunks), self.threads):
task_patches = []
for j in range(i, min(len(chunks), i + self.threads)):
task_patches.append(chunks[j])
downsample_task = make_downsample_task_message(
z, task_patches, mip=m)
self.task_handler.send_message(downsample_task)
while not self.task_handler.is_empty():
sleep(1)
else:
def chunkwise(patch_bbox):
print("Downsampling {} to mip {}".format(
patch_bbox.__str__(mip=0), m))
downsampled_patch = self.downsample_patch(
self.dst_ng_path, z, patch_bbox, m)
data_handler.save_image_patch(self.dst_ng_path,
downsampled_patch, z,
patch_bbox, m)
self.pool.map(chunkwise, chunks)
end = time()
print(": {} sec".format(end - start))
def compute_section_pair_residuals(self, source_z, target_z, bbox):
for m in range(self.process_high_mip, self.process_low_mip - 1, -1):
print("Running net at mip {}".format(m), end='', flush=True)
start = time()
chunks = self.break_into_chunks(bbox,
self.vec_chunk_sizes[m],
self.vec_voxel_offsets[m],
mip=m)
for patch_bbox in chunks:
#FIXME Torch runs out of memory
#FIXME batchify download and upload
if self.distributed:
residual_task = make_residual_task_message(source_z,
target_z,
patch_bbox,
mip=m)
self.task_handler.send_message(residual_task)
else:
self.compute_residual_patch(source_z,
target_z,
patch_bbox,
mip=m)
#self.pool.map(chunkwise, chunks)
if self.distributed:
while not self.task_handler.is_empty():
sleep(1)
end = time()
print(": {} sec".format(end - start))
if m > self.process_low_mip:
self.prepare_source(source_z, bbox, m - 1)
## Whole stack operations
def align_ng_stack(self,
start_section,
end_section,
bbox,
move_anchor=True):
if not self.check_all_params():
raise Exception("Not all parameters are set")
#if not bbox.is_chunk_aligned(self.dst_ng_path):
# raise Exception("Have to align a chunkaligned size")
start = time()
if move_anchor:
for m in range(self.render_low_mip, self.high_mip):
self.copy_section(self.src_ng_path,
self.dst_ng_path,
start_section,
bbox,
mip=m)
for z in range(start_section, end_section):
self.compute_section_pair_residuals(z + 1, z, bbox)
self.render_section_all_mips(z + 1, bbox)
end = time()
print("Total time for aligning {} slices: {}".format(
end_section - start_section, end - start))
## Distribution
def handle_residual_task(self, message):
source_z = message['source_z']
target_z = message['target_z']
patch_bbox = deserialize_bbox(message['patch_bbox'])
mip = message['mip']
self.compute_residual_patch(source_z, target_z, patch_bbox, mip)
def handle_render_task(self, message):
z = message['z']
patches = [deserialize_bbox(p) for p in message['patches']]
mip = message['mip']
def chunkwise(patch_bbox):
print("Rendering {} at mip {}".format(patch_bbox.__str__(mip=0),
mip),
end='',
flush=True)
self.warp_patch(self.src_ng_path, z, patch_bbox,
(mip, self.process_high_mip), mip)
self.pool.map(chunkwise, patches)
def handle_copy_task(self, message):
z = message['z']
patches = [deserialize_bbox(p) for p in message['patches']]
mip = message['mip']
source = message['source']
dest = message['dest']
def chunkwise(patch_bbox):
raw_patch = data_handler.get_image_data(source, z, patch_bbox, mip)
data_handler.save_image_patch(dest, raw_patch, z, patch_bbox, mip)
self.pool.map(chunkwise, patches)
def handle_downsample_task(self, message):
z = message['z']
patches = [deserialize_bbox(p) for p in message['patches']]
mip = message['mip']
def chunkwise(patch_bbox):
downsampled_patch = self.downsample_patch(self.dst_ng_path, z,
patch_bbox, mip)
data_handler.save_image_patch(self.dst_ng_path, downsampled_patch,
z, patch_bbox, mip)
self.pool.map(chunkwise, patches)
def handle_task_message(self, message):
#message types:
# -compute residual
# -prerender future target
# -render final result
# -downsample
# -copy
#import pdb; pdb.set_trace()
body = json.loads(message['Body'])
task_type = body['type']
if task_type == 'residual_task':
self.handle_residual_task(body)
elif task_type == 'render_task':
self.handle_render_task(body)
elif task_type == 'copy_task':
self.handle_copy_task(body)
elif task_type == 'downsample_task':
self.handle_downsample_task(body)
else:
raise Exception(
"Unsupported task type '{}' received from queue '{}'".format(
task_type, self.task_handler.queue_name))
def listen_for_tasks(self):
while (True):
message = self.task_handler.get_message()
if message != None:
print("Got a job")
s = time()
self.handle_task_message(message)
self.task_handler.delete_message(message)
e = time()
print("Done: {} sec".format(e - s))
else:
sleep(3)
print("Waiting for jobs...")
def run(args, analysis_config=None):
"""
:param args:
:type args:
:param analysis_config: if None, the analysis_config in the args is read
:type analysis_config:
:return:
:rtype:
"""
# initialise this set of simulations runs by creating a directory for output files
model_run_base_directory, simulation_run_description, yaml_struct = load_configuration(
args)
# setup any analysis configuration
if analysis_config is None:
if args.analysis_config:
# find the analysis_config in the yaml file by name
analysis_configs_ = [
item for item in yaml_struct['Metadata'].get(
'analysis_configs', [])
if item['name'] == args.analysis_config
]
if analysis_configs_:
logger.info(
f"Running with analysis_config '{analysis_configs_[0]}'")
analysis_config = analysis_configs_[0]
else:
logger.warning(
f"Could not find analysis_config {args.analysis_config}")
analysis_config = {}
else:
analysis_config = {}
# scenarios to evaluate
scenarios = yaml_struct['Analysis'].get('scenarios', [])
if not 'default' in scenarios:
# 'default' is implicit
scenarios.append('default')
# define aspects/data to store
logger.info(f'Skip store results set to {args.skip_storage}')
output_persistence_config = {
'store_traces': not args.skip_storage,
'process_traces': analysis_config.get('process_traces', [])
}
# a partial to invoke for each scenario
run_scenario_f = partial(
run_scenario,
model_run_base_directory=model_run_base_directory,
simulation_run_description=simulation_run_description,
yaml_struct=yaml_struct,
args=args,
output_persistence_config=output_persistence_config,
analysis_config=analysis_config)
if args.multithreaded:
logger.info("Running in parallel")
from pathos.multiprocessing import ThreadPool
import multiprocessing
num_cores = multiprocessing.cpu_count()
results = ThreadPool(processes=num_cores).map(run_scenario_f,
scenarios)
# results = Parallel(n_jobs=num_cores)(delayed(run_scenario_f)(i) for i in scenarios)
else:
results = []
for scenario in scenarios:
results.append(run_scenario_f(scenario))
# combine all run results
runners = dict((x, y) for x, y in results)
analysis_config.update(yaml_struct['Analysis'])
analysis_config.update(yaml_struct['Metadata'])
if not args.sensitivity:
scenario_paths = {
scenario_name: run_data.sim_control.output_directory
for scenario_name, run_data in runners.items()
}
if args.analysis_config:
summary_analysis(scenario_paths,
model_run_base_directory,
analysis_config,
yaml_struct,
image_filetype=args.filetype)
return runners
def accept(self, data: List[TDACandle]):
def _accept(strat: TickerStrategy):
strat.accept(data)
with ThreadPool(5) as p:
p.map(_accept, self.strategies)