def msgpgs(self, ms, psize, tstart, fileMightBeRegistered=False):
"""
create a progress message from the given info
"""
pgs = ms.processedSize
size = ms.fileSize
tnow = time.time()
if (pgs is None):
pgs = 0
if size == 0:
percent = 100
else:
percent = int(pgs * 100 / size)
tdif = tnow - tstart
if tdif == 0:
speed = 0
else:
speed = (psize + pgs) / (1024*1024*tdif)
if speed == 0:
speed = 0
eta = " --:--"
else:
eta = int((self.totalsize - psize - pgs) / (1024*1024*speed))
speed = int(speed)
if eta != 0:
eta = " {:02d}:{:02d}".format(int(eta/60), eta % 60)
else:
eta = " 00:00"
return "{}{} {:3d}MB/s{}".format("* " if fileMightBeRegistered else "", progressbar(percent,size=21), speed, eta)
def _get_annotations(generator):
""" Get the ground truth annotations from the generator.
The result is a list of lists such that the size is:
all_detections[num_images][num_classes] = annotations[num_detections, 5]
# Arguments
generator : The generator used to retrieve ground truth annotations.
# Returns
A list of lists containing the annotations for each image in the generator.
"""
all_annotations = [[None for i in range(generator.num_classes())] for j in range(generator.size())]
for i in progressbar.progressbar(range(generator.size()), prefix='Parsing annotations: '):
# load the annotations
annotations = generator.load_annotations(i)
# copy detections to all_annotations
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
all_annotations[i][label] = annotations['bboxes'][annotations['labels'] == label, :].copy()
return all_annotations
def _get_detections(generator, model, score_threshold=0.05, max_detections=100, save_path=None):
""" Get the detections from the model using the generator.
The result is a list of lists such that the size is:
all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes]
# Arguments
generator : The generator used to run images through the model.
model : The model to run on the images.
score_threshold : The score confidence threshold to use.
max_detections : The maximum number of detections to use per image.
save_path : The path to save the images with visualized detections to.
# Returns
A list of lists containing the detections for each image in the generator.
"""
all_detections = [[None for i in range(generator.num_classes()) if generator.has_label(i)] for j in range(generator.size())]
for i in progressbar.progressbar(range(generator.size()), prefix='Running network: '):
raw_image = generator.load_image(i)
image = generator.preprocess_image(raw_image.copy())
image, scale = generator.resize_image(image)
if keras.backend.image_data_format() == 'channels_first':
image = image.transpose((2, 0, 1))
# run network
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))[:3]
# correct boxes for image scale
boxes /= scale
# select indices which have a score above the threshold
indices = np.where(scores[0, :] > score_threshold)[0]
# select those scores
scores = scores[0][indices]
# find the order with which to sort the scores
scores_sort = np.argsort(-scores)[:max_detections]
# select detections
image_boxes = boxes[0, indices[scores_sort], :]
image_scores = scores[scores_sort]
image_labels = labels[0, indices[scores_sort]]
image_detections = np.concatenate([image_boxes, np.expand_dims(image_scores, axis=1), np.expand_dims(image_labels, axis=1)], axis=1)
if save_path is not None:
draw_annotations(raw_image, generator.load_annotations(i), label_to_name=generator.label_to_name)
draw_detections(raw_image, image_boxes, image_scores, image_labels, label_to_name=generator.label_to_name)
cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image)
# copy detections to all_detections
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
all_detections[i][label] = image_detections[image_detections[:, -1] == label, :-1]
return all_detections
def lint_files(filenames, pass_classes, clang_args):
def strip_dot_slash(d):
if d.filename.startswith('./'):
d.filename = d.filename[2:]
return d
def interesting_file(d):
if os.path.isabs(d.filename):
return False
if d.filename.startswith('opt'):
return False
return True
def mk_task(filename):
return (filename, pass_classes, clang_args)
diags = []
tasks = (mk_task(f) for f in filenames)
if PARALLEL:
# TODO: accept -jN and/or read process count from config
pool = Pool(processes=4)
try:
for r in progressbar(pool.imap(worker, tasks),
length=len(filenames)):
diags += r
except KeyboardInterrupt:
pool.terminate()
return []
else:
for r in progressbar(imap(worker, tasks), length=len(filenames)):
diags += r
diags = map(strip_dot_slash, diags)
diags = filter(interesting_file, diags)
diags = sorted(diags, key=lambda d: d.line_number)
diags = sorted(diags, key=lambda d: d.filename)
return unique(diags)
def segment(content):
global num_of_line
pb = progressbar(num_of_line)
ctr = 0
contentlist = list()
for item in content:
ctr += 1
pb.progress(ctr)
seg_list = jieba.cut(item)
contentlist.append(','.join(seg_list))
f = open("seged_content.txt", 'wb')
for sentence in contentlist:
f.write(sentence.encode("utf-8"))
f.close()
def evaluate_analogy(args, token_embedding, dataset,
analogy_function='ThreeCosMul'):
"""Evaluation on analogy task."""
# Open vocabulary: Use all known words
if args.analogy_max_vocab:
counter = nlp.data.Counter(token_embedding.idx_to_token[:args.analogy_max_vocab])
else:
counter = nlp.data.Counter(token_embedding.idx_to_token)
vocab = nlp.vocab.Vocab(counter)
vocab.set_embedding(token_embedding)
if args.ignore_oov:
initial_length = len(dataset)
dataset = [
d for d in dataset if d[0] in vocab and d[1] in vocab
and d[2] in vocab and d[3] in vocab
]
num_dropped = initial_length - len(dataset)
if num_dropped:
logging.warning('Dropped %s pairs from %s as the were OOV.',
num_dropped, dataset.__class__.__name__)
dataset_coded = [[vocab[d[0]], vocab[d[1]], vocab[d[2]], vocab[d[3]]]
for d in dataset]
dataset_coded_batched = mx.gluon.data.DataLoader(
dataset_coded, batch_size=args.batch_size)
exclude_question_words = not args.analogy_dont_exclude_question_words
evaluator = nlp.embedding.evaluation.WordEmbeddingAnalogy(
idx_to_vec=vocab.embedding.idx_to_vec,
exclude_question_words=exclude_question_words,
analogy_function=analogy_function)
context = get_context(args)
evaluator.initialize(ctx=context)
if not args.dont_hybridize:
evaluator.hybridize()
acc = mx.metric.Accuracy()
if progressbar is not None:
dataset_coded_batched = progressbar.progressbar(dataset_coded_batched)
for batch in dataset_coded_batched:
batch = batch.as_in_context(context)
words1, words2, words3, words4 = (batch[:, 0], batch[:, 1],
batch[:, 2], batch[:, 3])
pred_idxs = evaluator(words1, words2, words3)
acc.update(pred_idxs[:, 0], words4.astype(np.float32))
logging.info('Accuracy on %s: %s', dataset.__class__.__name__,
acc.get()[1])
return acc.get()[1], len(dataset)
def filter_template(self, func, s=""):
pb = progressbar(len(self.in_q), ".", s)
ctr = 0
while True:
if self.in_q:
ctr += 1
pb.progress(ctr)
job = self.in_q.pop()
func(job)
else:
break
print len(self.out_q)
self.in_q = self.out_q
self.out_q = list()
return
def filter_stopwords(self):
stopwords = self.init_stopwords()
pb = progressbar(len(self.in_q), ".", "remove stopwords")
ctr = 0
while True:
if self.in_q:
ctr += 1
pb.progress(ctr)
job = self.in_q.pop().split(',')
job = filter(lambda x: x not in stopwords, job)
self.out_q.append(' '.join(job))
else:
break
self.in_q = self.out_q
self.out_q = list()
return
def optimize(args, nmax=2500000):
target_dist, m_min, m_max, P, veff_max = args
f_new = np.zeros(1000, dtype=[('m', np.float64), ('veff', np.float64), ('v', np.float64),
('x', np.float64), ('t', np.float64)])
f_new['m'] = np.linspace(m_min, m_max, len(f_new))
params_new = np.zeros(5, dtype=[('m', np.float64), ('veff', np.float64)])
params_new['m'] = np.linspace(m_min, m_max, len(params_new))
params_new['veff'] = 0.1
calculate(f_new, P, dist=target_dist, veff_max=veff_max, params=params_new)
score_best = get_score(f_new, dist=target_dist)
f_orig = f_new.copy()
last_i = 0
for i in progressbar(range(nmax)):
g_params = step(params_new, veff_max=veff_max)
g_g = f_new.copy()
mis_t, mis_x = calculate(g_g, P, dist=target_dist, params=g_params, veff_max=veff_max)
score = get_score(g_g, dist=target_dist)
if (i+1) % (nmax / 20) == 0:
for j in range(2):
g_ind = np.random.randint(0, len(params_new)//2-1) + j * len(params_new)//2
obj = params_new.copy()[0]
obj['m'] = (params_new['m'][g_ind] + params_new['m'][g_ind+1]) / 2
obj['veff'] = (params_new['veff'][g_ind] + params_new['veff'][g_ind + 1]) / 2
params_new = np.insert(params_new, g_ind+1, obj)
g_params = params_new.copy()
if score < score_best:
if (score < 800) and (i-last_i) > 1000 and False:
last_i = i
plt.plot(g_g['m'] / 1000, g_g['veff'] / 1000, label='optimized')
plt.show()
plt.plot(g_g['m'] / 1000, g_g['veff'] / 1000, label='optimized')
score_best = score
f_new = g_g
params_new = g_params
return f_new, f_orig, score_best, mis_t
def render_to_directory(dirname, title, files, category_names):
try:
os.mkdir(dirname)
except OSError:
pass
index_template = load_template('report_index.jinja')
single_file_template = load_template('report_single_file.jinja')
with open(os.path.join(dirname, 'index.html'), 'w') as fo:
fo.write(index_template.render(files=files,
diagnostic_categories=category_names,
title=title).encode('utf8'))
for f in progressbar(files):
for dname, ds in f['diagnostic_groups'].iteritems():
path = os.path.join(dirname, f['name'].replace('/', '_'))
path += '_' + dname + '.html'
text = single_file_template.render(filename=f['name'],
diagnostics=ds)
with open(path, 'w') as fo:
fo.write(text.encode('utf8'))
def process(content):
global num_of_line
pb = progressbar(num_of_line)
ctr = 0
worddict = dict()
wordlist = list()
for item in content:
ctr += 1
pb.progress(ctr)
seg_list = jieba.cut(item)
for word in seg_list:
if worddict.has_key(word):
worddict[word] += 1
else:
worddict[word] = 1
print "total words:", len(worddict)
temp = list()
for k, v in worddict.iteritems():
temp.append([k, v])
wordlist = sorted(temp, key=lambda x: x[1], reverse=True)
f = open("terms.txt", 'wb')
for word in wordlist:
f.write(word[0] + '\t' + str(word[1]) + '\n')
f.close()
times.SetName('Time')
vtkilagr = vtk.vtkIdTypeArray()
vtkilagr.SetNumberOfComponents(1)
vtkilagr.SetNumberOfTuples(pts.GetNumberOfPoints())
vtkilagr.SetName('ILAGR')
vtkexist = vtk.vtkShortArray()
vtkexist.SetNumberOfComponents(1)
vtkexist.SetNumberOfTuples(pts.GetNumberOfPoints())
vtkexist.SetName('EXIST')
#verts = vtk.vtkCellArray()
for j in progressbar.progressbar(range(N)):
pts.SetPoint(j, (x.iloc[j],y.iloc[j],z.iloc[j]))
types.SetTuple1(j, tp.iloc[j])
times.SetTuple1(j, T.iloc[j])
vtkilagr.SetTuple1(j, ilagr.iloc[j])
vtkexist.SetTuple1(j, exist.iloc[j])
# verts.InsertNextCell(1)
# verts.InsertCellPoint(j)
#create connectivity
#get unique IDs
uIds = pd.unique(ilagr)
def preciseNameSearch(*names):
nameData = []
errName = []
# a list contain names that fail to find in preciseSearch
combinedNameData = []
counter = 1
k = 0
totalRecordNum = len(names[0])
for k in progressbar.progressbar(range(len(names[0])),
redirect_stdout=True):
name = names[0][k]
processedName = URI_escape(name)
print("Processing ",
counter,
" / ",
totalRecordNum,
" of records...",
end="")
nameURL = 'https://id.loc.gov/authorities/names/label/' + processedName
try:
nameResponse = requests.head(nameURL)
except:
print("Fail to request: ", name)
errName.append(name)
if nameResponse.status_code == 302:
nameData.append(name)
nameData.append(name)
nameData.append(nameResponse.headers['X-Uri'])
nameData.append(nameResponse.headers['X-Preflabel'])
else:
if nameResponse.status_code == 301:
print(
"\nHTTP 301- Request link has been moved permanently....",
end="")
elif nameResponse.status_code == 404:
print("\nHTTP 404- Request link page doesn't exist....",
end="")
else:
print("\n HTTP",
subjectResponse.status_code,
"- Not a valid response for scraping....",
end="")
#TODO: name not appended, no err file out
nameData.append(name)
nameData.append("null")
nameData.append("null")
nameData.append("null")
combinedNameData.append(nameData)
print("Done!")
if (len(errName) > 0):
errName.insert("Name")
errorOut = open("Err.csv", 'w', encoding='utf-8', newline='')
simpleCSV.writeCSV(["Name with no result"], errorOut)
SimpleCSV.writeCSV(errName, errorOut)
print("Name that cause error is put into Err.csv")
counter = counter + 1
k = k + 1
nameData = []
#add header for output file
combinedNameData.insert(0,
["Raw Name", "Correct Name", "LC_URI", "LC_Label"])
return combinedNameData
def _get_detections(generator,
model,
score_threshold=0.05,
max_detections=100,
save_path=None,
save_path_image=None):
""" Get the detections from the model using the generator.
The result is a list of lists such that the size is:
all_detections[num_images][num_classes] = detections[num_detections, 4 + num_classes]
# Arguments
generator : The generator used to run images through the model.
model : The model to run on the images.
score_threshold : The score confidence threshold to use.
max_detections : The maximum number of detections to use per image.
save_path : The path to save the images with visualized detections to.
# Returns
A list of lists containing the detections for each image in the generator.
"""
all_detections = [[
None for i in range(generator.num_classes()) if generator.has_label(i)
] for j in range(generator.size())]
all_inferences = [None for i in range(generator.size())]
result = {'path': [], 'x1': [], 'y1': [], 'x2': [], 'y2': [], 'class': []}
for i in progressbar.progressbar(range(generator.size()),
prefix='Running network: '):
raw_image = generator.load_image(i)
image = generator.preprocess_image(raw_image.copy())
image, scale = generator.resize_image(image)
image_path = generator.image_path(i)
if keras.backend.image_data_format() == 'channels_first':
image = image.transpose((2, 0, 1))
# run network
start = time.time()
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))[:3]
inference_time = time.time() - start
# correct boxes for image scale
boxes /= scale
# select indices which have a score above the threshold
indices = np.where(scores[0, :] > score_threshold)[0]
# select those scores
scores = scores[0][indices]
# find the order with which to sort the scores
scores_sort = np.argsort(-scores)[:max_detections]
# select detections
image_boxes = boxes[0, indices[scores_sort], :]
image_scores = scores[scores_sort]
image_labels = labels[0, indices[scores_sort]]
image_detections = np.concatenate([
image_boxes,
np.expand_dims(image_scores, axis=1),
np.expand_dims(image_labels, axis=1)
],
axis=1)
if save_path is not None:
selection = np.where(image_scores > score_threshold)[0]
for j in selection:
b = np.array(image_boxes[j, :]).astype(int)
result['path'].append(image_path)
result['x1'].append(b[0])
result['y1'].append(b[1])
result['x2'].append(b[2])
result['y2'].append(b[3])
result['class'].append(image_labels[j])
if save_path_image is not None:
draw_annotations(raw_image,
generator.load_annotations(i),
label_to_name=generator.label_to_name)
draw_detections(raw_image,
image_boxes,
image_scores,
image_labels,
label_to_name=generator.label_to_name,
score_threshold=score_threshold)
cv2.imwrite(os.path.join(save_path_image, '{}.png'.format(i)),
raw_image)
# copy detections to all_detections
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
all_detections[i][label] = image_detections[
image_detections[:, -1] == label, :-1]
all_inferences[i] = inference_time
if save_path:
df = pd.DataFrame(result)
df.to_csv(save_path, sep=',', index=False, header=None)
return all_detections, all_inferences
def connect_knn(args, ws):
if args.no_wait:
return
trial_str = _trial_str(ws.current_trial)
algoprefix = 'pairwise_knn-'
for puzzle_fn, puzzle_name in ws.test_puzzle_generator(args.puzzle_name):
ITE = matio.load(_ibte_fn(
ws, puzzle_name, ws.current_trial))['INTER_BLOOMING_TREE_EDGES']
pds_fn = _puzzle_pds(ws, puzzle_name, ws.current_trial)
d = matio.load(pds_fn)
QF = d['QF']
QB = d['QB']
"""
We need to be aware of one important differents between bloom-from_*.npz files and the PDS
file. Some blooming tree may be empty.
Hence the Bloom No. (denoted by the number in the file name) may not be aligned to
the index in the PDS file.
However, the inter_blooming_tree_edges (ITE) uses bloom no. rather than the PDS index.
Thus the translation is needed.
"""
BLOOM_NO_TO_INDEX = d['BLOOM_NO_TO_INDEX']
INDEX_TO_BLOOM_NO = d['INDEX_TO_BLOOM_NO']
import networkx as nx
# Forest level path
G = nx.Graph()
G.add_nodes_from([i for i in range(len(BLOOM_NO_TO_INDEX))] +
[VIRTUAL_OPEN_SPACE_NODE])
G.add_edges_from(ITE[:, [0, 2]])
openset = []
virtual_edges = []
"""
Connect OpenSet trees to VIRTUAL_OPEN_SPACE_NODE
"""
for tree_index, tree_base in enumerate(QB):
_, tree_end = _extract_bound(QB, QF.shape[0], tree_index)
"""
Translate back to bloom no. to be compatitable with ITE
"""
tree_no = INDEX_TO_BLOOM_NO[tree_index]
# print(tree_base)
# print(tree_end)
for flag in QF[tree_base:tree_end]:
if flag & OPENSPACE_FLAG:
openset.append(tree_no)
virtual_edges.append((tree_no, VIRTUAL_OPEN_SPACE_NODE))
break
util.log("OpenSet {}".format(openset))
G.add_edges_from(virtual_edges)
try:
ids = nx.shortest_path(G, 0, VIRTUAL_OPEN_SPACE_NODE)
util.log('Forest-level shortest path {}'.format(ids))
except nx.exception.NetworkXNoPath:
util.warn(
f'[connect_knn] Cannot find path for puzzle {puzzle_name}')
continue
util.ack(
'[solve1][connect_knn] forest level path (bloom no.) {}'.format(
ids))
ids = [BLOOM_NO_TO_INDEX[i] for i in ids[:-1]]
ids.append(VIRTUAL_OPEN_SPACE_NODE)
util.ack(
'[solve1][connect_knn] forest level path (PDS index) {}'.format(
ids))
"""
We use bloom no. in forest level path, because ITE uses bloom no.
In tree level path we uses pds index instead,
because we tree level data mainly comes from the PDS file.
"""
ITE_meta = {}
def insert_ite(from_fi, from_vi, to_fi, to_vi):
'''
if from_fi == 0 and to_fi == 78:
util.log('ite {}'.format(ite))
'''
if from_fi not in ITE_meta:
ITE_meta[from_fi] = {}
if to_fi not in ITE_meta[from_fi]:
ITE_meta[from_fi][to_fi] = [(from_vi, to_vi)]
else:
ITE_meta[from_fi][to_fi].append((from_vi, to_vi))
for index, ite in progressbar(enumerate(ITE)):
from_fi, from_vi, to_fi, to_vi = ite
from_fi = BLOOM_NO_TO_INDEX[from_fi]
to_fi = BLOOM_NO_TO_INDEX[to_fi]
insert_ite(from_fi, from_vi, to_fi, to_vi)
insert_ite(to_fi, to_vi, from_fi, from_vi)
# from_fi, to_fi = 0, 82
# util.log(f'ITE_meta[{from_fi}][{to_fi}] {ITE_meta[from_fi][to_fi]}')
Q = d['Q']
QE = d['QE']
QEB = d['QEB']
ompl_q = []
bloom0_fn = ws.local_ws(
_rel_bloom_scratch(ws, puzzle_name, ws.current_trial),
f'bloom-from_0.npz')
prev_fi = 0
prev_vi = matio.load(bloom0_fn)['IS_INDICES'][0]
try:
for from_fi, to_fi in zip(ids, ids[1:]):
if to_fi != VIRTUAL_OPEN_SPACE_NODE:
from_vi, to_vi = ITE_meta[from_fi][to_fi][0]
else:
from_fi = prev_fi
from_vi = VIRTUAL_OPEN_SPACE_NODE
# if to_vi == VIRTUAL_OPEN_SPACE_NODE:
# ids = ids[:-1]
ids = tree_level_path(Q, QB, QE, QEB, QF, prev_fi, prev_vi,
from_fi, from_vi)
q_from = QB[from_fi]
local_ids = ids - q_from
util.log(f"Tree {from_fi} IDS {ids}, Local IDS {local_ids}")
qs = Q[ids, :]
util.log(f"Shape {qs.shape}")
"""
bloom_fn = ws.local_ws(_rel_bloom_scratch(ws, puzzle_name, ws.current_trial), f'bloom-from_{from_fi}.npz')
localQ = matio.load(bloom_fn)['BLOOM']
for gi,li in zip(ids, local_ids):
dist = np.linalg.norm(Q[gi] - localQ[li])
assert dist < 1e-3, f'{Q[gi]} != {localQ[li]} gi {gi} li {li}'
"""
ompl_q.append(qs)
# if to_fi != VIRTUAL_OPEN_SPACE_NODE:
# gvi = QB[to_fi] + to_vi
# ompl_q.append(Q[gvi:gvi+1,:])
prev_fi = to_fi
prev_vi = to_vi
# ompl_q.append(tree_level_path(Q, QB, QE, QEB, QF, prev_fi, prev_vi, prev_fi, VIRTUAL_OPEN_SPACE_NODE))
except nx.exception.NetworkXNoPath:
assert False, "Should not happen. Found forest-level path but no tree-level path."
continue
ompl_q = util.safe_concatente(ompl_q, axis=0)
rel_scratch_dir = os.path.join(util.SOLVER_SCRATCH, puzzle_name,
trial_str)
path_out = ws.local_ws(rel_scratch_dir, algoprefix + 'path.txt')
matio.savetxt(path_out, ompl_q)
util.ack("Saving OMPL solution of {} to {}".format(
puzzle_name, path_out))
uw = util.create_unit_world(puzzle_fn)
unit_q = uw.translate_ompl_to_unit(ompl_q)
sol_out = ws.solution_file(puzzle_name, type_name=algoprefix + 'unit')
matio.savetxt(sol_out, unit_q)
util.ack("Saving UNIT solution of {} to {}".format(
puzzle_name, sol_out))
def temp_tuning(image: np.array, T_MAX, N, initial: np.array = None):
if initial is None:
initial = hiprmc.random_initial(image)
T = T_MAX
# T = 0
simulated_image = initial.copy()
original = simulated_image
F_image = hiprmc.fourier_transform(image)
F_old = hiprmc.fourier_transform(simulated_image)
chi_old = hiprmc.chi_square(F_old, F_image)
error, count, accept_rate, temperature = [], [], [], []
t_step = T_MAX / 100.0
for t in progressbar.progressbar(range(0, 100)):
move_count = 0.0
acceptance = 0.0
T = T - t_step
for iter in range(0, np.count_nonzero(simulated_image == 1)):
move_count += 1
x_old = np.random.randint(0, image.shape[0])
y_old = np.random.randint(0, image.shape[1])
while simulated_image[x_old][y_old] != 1:
x_old = np.random.randint(0, image.shape[0])
y_old = np.random.randint(0, image.shape[1])
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
while np.sqrt((y_new - y_old) ** 2 + (x_new - x_old) ** 2) > 5:
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
old_point = simulated_image[x_old][y_old]
new_point = simulated_image[x_new][y_new]
while new_point == old_point:
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
new_point = simulated_image[x_new][y_new]
simulated_image[x_old][y_old] = new_point
simulated_image[x_new][y_new] = old_point
old_array = np.zeros_like(image)
old_array[x_old][y_old] = 1
new_array = np.zeros_like(image)
new_array[x_new][y_new] = 1
F_new = F_old - hiprmc.fourier_transform(old_array) + hiprmc.fourier_transform(new_array)
chi_new = hiprmc.chi_square(F_new, F_image)
delta_chi = chi_new - chi_old
acceptance, chi_old = hiprmc.Metropolis(x_old, y_old, x_new, y_new, old_point, new_point, delta_chi,
simulated_image, T, acceptance, chi_old, chi_new, T_MAX, iter, N)
F_old = hiprmc.fourier_transform(simulated_image)
error.append(chi_old)
count.append(t)
accept_rate.append(acceptance / move_count)
temperature.append(T)
# def gaussian(x, a, b):
# return a * (1 - np.exp(-(x / b) ** 2))
def sigmoid(x, a, b):
return 1.0 / (1.0 + np.exp(-a * (x - b)))
fit_params, fit_params_error = optimize.curve_fit(sigmoid, temperature, accept_rate, p0=[0.5, T_MAX / 2])
# f = plt.figure()
# f.add_subplot(1, 2, 1)
# plt.plot(temperature, accept_rate, 'bo')
# plt.plot(temperature, sigmoid(temperature, *fit_params), 'r-')
# plt.ylabel('Acceptance Rate')
# plt.ylim([0, 1])
# plt.xlabel('Temperature')
# f.add_subplot(1, 2, 2)
# plt.plot(count, error, 'k-')
# plt.ylabel('Chi-Squared')
# plt.xlabel('Monte Carlo Iteration')
# plt.tight_layout()
# plt.show()
# temp_tstar = abs(fit_params[1]) * np.sqrt(-np.log(1 - 0.333333 / fit_params[0]))
temp_tstar = fit_params[1] - np.log(2.3333) / fit_params[0]
t_min = max(0.001, temp_tstar - 0.2)
t_max = min(T_MAX, temp_tstar + 0.2)
t_step = (t_max - t_min) / 1000
T = t_max
simulated_image = original
print(t_min, t_max)
temperature2, accept_rate2 = [], []
for t in progressbar.progressbar(range(0, 1000)):
move_count = 0.0
acceptance = 0.0
for iter in range(0, np.count_nonzero(simulated_image == 1)):
move_count += 1
x_old = np.random.randint(0, image.shape[0])
y_old = np.random.randint(0, image.shape[1])
while simulated_image[x_old][y_old] != 1:
x_old = np.random.randint(0, image.shape[0])
y_old = np.random.randint(0, image.shape[1])
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
while np.sqrt((y_new - y_old) ** 2 + (x_new - x_old) ** 2) > 15:
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
old_point = simulated_image[x_old][y_old]
new_point = simulated_image[x_new][y_new]
while new_point == old_point:
x_new = np.random.randint(0, image.shape[0])
y_new = np.random.randint(0, image.shape[1])
new_point = simulated_image[x_new][y_new]
simulated_image[x_old][y_old] = new_point
simulated_image[x_new][y_new] = old_point
old_array = np.zeros_like(image)
old_array[x_old][y_old] = 1
new_array = np.zeros_like(image)
new_array[x_new][y_new] = 1
F_new = F_old - hiprmc.fourier_transform(old_array) + hiprmc.fourier_transform(new_array)
chi_new = hiprmc.chi_square(F_new, F_image)
delta_chi = chi_new - chi_old
acceptance, chi_old = hiprmc.Metropolis(x_old, y_old, x_new, y_new, old_point, new_point, delta_chi,
simulated_image, T, acceptance, chi_old, chi_new, T_MAX, iter, N)
F_old = hiprmc.fourier_transform(simulated_image)
accept_rate2.append(acceptance / move_count)
temperature2.append(T)
T = T - t_step
fit_params, fit_params_error = optimize.curve_fit(sigmoid, temperature2, accept_rate2, p0=[0.5, temp_tstar])
# temp_tstar = abs(fit_params[1]) * np.sqrt(-np.log(1 - 0.333333 / fit_params[0]))
temp_tstar = fit_params[1] - np.log(2) / fit_params[0]
t_min = min(0.001, temp_tstar - 0.2)
t_max = min(T_MAX, temp_tstar + 0.2)
#
# f = plt.figure()
# f.add_subplot(1, 2, 1)
# plt.plot(temperature2, accept_rate2, 'bo')
# plt.plot(temperature2, sigmoid(temperature2, *fit_params), 'r-')
# plt.ylabel('Acceptance Rate')
# plt.ylim([0,1])
# plt.xlabel('Temperature')
# f.add_subplot(1, 2, 2)
# plt.plot(count, error, 'k-')
# plt.ylabel('Chi-Squared')
# plt.xlabel('Monte Carlo Iteration')
# plt.tight_layout()
# plt.show()
#
# simulated_image = original
# f = plt.figure()
# f.add_subplot(1, 5, 1)
# plt.axis('off')
# plt.title("original image")
# f.add_subplot(1, 5, 2)
# plt.axis('off')
# plt.title("FFT of Image")
# f.add_subplot(1, 5, 3)
# plt.imshow(initial, cmap="Greys", origin="lower")
# plt.title("random start")
# plt.axis('off')
# f.add_subplot(1, 5, 4)
# plt.imshow(simulated_image, cmap="Greys", origin="lower")
# plt.title("updated")
# plt.axis('off')
# f.add_subplot(1, 5, 5)
# plt.imshow(np.log10(abs(hiprmc.fourier_transform(simulated_image))), origin='lower')
# plt.axis('off')
# plt.title("FFT of final state")
# plt.show()
return simulated_image, t_max, t_min
def subset(self):
print('Processing request:')
for i in progressbar.progressbar(range(100)):
time.sleep(0.08)
print('Request processing complete.')
def test(net,
dataloader,
criterion=None,
evaluate=True,
predict=True,
device=torch.device('cuda:0')):
'''
使用训练好的net进行预测或评价
args：
net，训练好的RetinaNet对象；
dataloader，需要进行预测或评价的数据集的dataloader；
criterion，计算损失的函数，如果是None则不计算loss；
evaluate，如果是True则进行模型的评价（即计算mAP），如果
是False，则只进行预测或计算loss；
predict，是否进行预测，如果True则会返回预测框的类别和坐标，如果False，
则不会返回；
returns：
all_labels_pred, all_markers_pred，如果predict=True，则返回预测的每张图
片的预测框的标签和loc；
losses, cls_losses, loc_losses，如果criterion不是None，则得到其3个loss；
APs, mAP_score，如果evaluate=True，则返回每一类的AP值和其mAP；
'''
y_encoder = dataloader.dataset.y_encoder
y_encoder_mode = dataloader.dataset.out
assert (evaluate and y_encoder_mode in ['obj', 'all']) or not evaluate
assert (criterion is not None and y_encoder_mode in ['all', 'encode']) or \
criterion is None
print('Testing ...')
results = []
with torch.no_grad():
losses = 0.
all_labels_pred = [] # 预测的labels，
all_scores_pred = []
all_markers_pred = []
all_labels_true = []
all_markers_true = []
for batch in pb.progressbar(dataloader):
imgs = batch[0].to(device)
if y_encoder_mode in ['all', 'obj']:
labels_true, locs_true = batch[1:3]
if y_encoder_mode in ['all', 'encode']:
targets = batch[-1].to(device)
preds = net(imgs)
if criterion is not None:
loss = criterion(preds, targets)
losses += loss.item()
if predict or evaluate:
img_size = list(imgs.shape[-1:-3:-1])
for pred in preds:
res_c, res_s, res_l = y_encoder.decode(pred, img_size)
all_labels_pred.append(res_c)
all_scores_pred.append(res_s)
all_markers_pred.append(res_l)
if evaluate:
all_labels_true.extend(labels_true)
all_markers_true.extend(locs_true)
if predict:
results.append((all_labels_pred, all_markers_pred))
if criterion is not None:
losses = losses / len(dataloader.dataset)
results.append(losses)
if evaluate:
# 使用chain将两层的嵌套list变成一层，符合mAP函数的输出要求
APs, mAP_score = mAP(
all_labels_true,
all_markers_true,
all_labels_pred,
all_scores_pred,
all_markers_pred,
)
results.append((APs, mAP_score))
return tuple(results)
'A2', 'A3', 'A4', 'A5', 'A6', 'A7', 'B2', 'B3', 'B4', 'B5', 'B6', 'B7'
]
classes_Ra = np.array([0.05, 0.1, 0.2, 0.4, 0.8, 1.6])
# A_classes = ['A2', 'A3', 'A4', 'A5', 'A6', 'A7']
# B_classes = ['B2', 'B3', 'B4', 'B5', 'B6', 'B7']
img_per_class = 2200
img_names = list(imagenames(img_per_class))
nb_classes = 12
nb_channels = 1
test_ratio = 0.1
print("Loading data... ", flush=True)
# Loading Images
data = np.empty((img_per_class * nb_classes, *img_shape), dtype=np.float)
for i in progressbar(range(nb_classes)):
class_name = classes[i]
for j, img in enumerate(img_names):
imgpath = root + class_name + '/' + img
data[i * img_per_class + j, ...] = imread(imgpath)
data_full = data.reshape((-1, *img_shape_flat))
data_A, data_B = np.split(data_full, 2)
data_classes = np.split(data_full, nb_classes)
del data
# Creating labels
labels_full = np.arange(nb_classes)
labels_full = np.repeat(labels_full, img_per_class)
labels_half = np.arange(nb_classes / 2)
labels_merged = np.tile(labels_half, 2)
def transform(self, X: dt.Frame):
"""
Transform expects only one column that contains a file name
:param X: contains file names
:return: features created on signals contained in the files
"""
# First we want to make sure that:
# - X contains only 1 column
# - The column is text
# - The column contains file names
from progressbar import progressbar
import numba
@numba.jit(parallel=True, fastmath=True)
def get_rolling_min(seq):
window = 30
l = len(seq) - window
z = np.empty(l)
for i in numba.prange(l):
z[i] = np.min(seq[i:i + window])
return z
@numba.jit(parallel=True, fastmath=True)
def get_rolling_max(seq):
window = 30
l = len(seq) - window
z = np.empty(l)
for i in numba.prange(l):
z[i] = np.max(seq[i:i + window])
return z
def get_nb_events(file, levels):
sig = dt.fread(file).to_numpy()[:, 0]
a = get_rolling_min(sig)
b = get_rolling_max(sig)
z = np.log10(b - a + 1e-10)
return [np.sum(z[z > _level]) for _level in levels]
if X.shape[1] > 1:
return np.zeros(X.shape[0])
if isinstance(X, dt.Frame):
# Datatable can select features directly on type
if X[:, [str]].shape[1] == 0:
return np.zeros(X.shape[0])
files = X[:, [str]].to_numpy()[:, 0]
else:
if X[X.columns[0]].dtype != "object":
return np.zeros(X.shape[0])
files = X[X.columns[0]].values[:, 0]
# Now let's go through the files and create features
try:
# Here we are supposed to use numba so multi processing is not required
levels = np.arange(1.0, 1.2, 0.1)
ret_df = pd.DataFrame([
get_nb_events(file, levels)
# for file in files
for file in progressbar(files)
])
except ValueError as e:
err_msg = e.args[0]
if "file" in err_msg.lower() and "does not exist" in err_msg.lower(
):
print("Error in {} : {}".format(self.display_name, err_msg))
return np.zeros(X.shape[0])
# Use pandas instead of dt.Frame(features)
# Pending issue #9894
return ret_df.values
def transform(self, X: dt.Frame):
"""
Transform expects only one column that contains a file name
:param X: contains file names
:return: features created on signals contained in the files
"""
from progressbar import progressbar
# First we want to make sure that:
# - X contains only 1 column
# - The column is text
# - The column contains file names
# Make sure we have 1 column
if X.shape[1] > 1:
return np.zeros(X.shape[0])
# Extract file paths
if isinstance(X, dt.Frame):
# Datatable can select features directly on type
if X[:, [str]].shape[1] == 0:
return np.zeros(X.shape[0])
files = X[:, [str]].to_numpy()[:, 0]
else:
if X[X.columns[0]].dtype != "object":
return np.zeros(X.shape[0])
files = X[X.columns[0]].values[:, 0]
# Now go through the files and create features
try:
def processor(out, res):
# print(out)
# print(res, flush=True)
out.append(res[0])
# out[res[0]] = res[1]
num_tasks = X.shape[0]
pool_to_use = small_job_pool
pool = pool_to_use(logger=None,
processor=processor,
num_tasks=num_tasks)
# Features will be a list of dict
features = []
for i_f, file in enumerate(progressbar(files)):
# Create full path
full_path = file
# Send to pool
args = (i_f, full_path, self._mfcc_size)
kwargs = {}
pool.submit_tryget(None,
get_features,
args=args,
kwargs=kwargs,
out=features)
pool.finish()
features_df = pd.DataFrame(features)
self._output_feature_names = list(features_df.columns)
self._feature_desc = list(features_df.columns)
return features_df
except ValueError as e:
err_msg = e.args[0]
if "file" in err_msg.lower() and "does not exist" in err_msg.lower(
):
print("Error in {} : {}".format(self.display_name, err_msg))
return np.zeros(X.shape[0])
Plin = 10**(np.asarray(dset['input/PdB'][...]/10))
try:
obj = dset.create_dataset('SCA', shape = dset['objval'].shape, fillvalue = np.nan, dtype = dset['objval'].dtype)
popt = dset.create_dataset('SCA_xopt', shape = dset['objval'].shape + (4,), fillvalue = np.nan, dtype = dset['objval'].dtype)
obj2 = dset.create_dataset('SCAmax', shape = dset['objval'].shape, fillvalue = np.nan, dtype = dset['objval'].dtype)
popt2 = dset.create_dataset('SCAmax_xopt', shape = dset['objval'].shape + (4,), fillvalue = np.nan, dtype = dset['objval'].dtype)
except RuntimeError:
obj = dset['SCA']
popt = dset['SCA_xopt']
obj2 = dset['SCAmax']
popt2 = dset['SCAmax_xopt']
pt = None
for cidx, pidx in pb.progressbar(it.product(range(11907,20000), range(obj.shape[1])), widget = pb.ETA, max_value = (20000-11907)*obj.shape[1]):
h = np.asarray(dset['input/channel_to_noise_matched'][cidx], dtype = float)
p = Plin[pidx]
if pidx == 0:
pt = None
if pt is not None:
o1,p1 = SCA(h, mu, Pc, Pmax = p, pt = pt)
else:
o1 = -np.inf
o2,p2 = SCA(h, mu, Pc, Pmax = p, pt = np.full(4, p))
obj2[cidx,pidx] = o2
popt2[cidx,pidx,:] = p2
import cv2
import sys
from os.path import join
from os import walk, makedirs
from progressbar import progressbar
srcDir = sys.argv[1]
dstDir = sys.argv[2]
dirs = [join(root, dir) for root, dirs, files in walk(srcDir) for dir in dirs]
for dir in dirs:
newDir = join(dstDir, dir.replace(srcDir, ""))
makedirs(newDir, exist_ok=True)
imgFiles = [
join(root, file) for root, dirs, files in walk(srcDir) for file in files
if file.endswith(".png")
]
#print(dirs)
for imgFile in progressbar(imgFiles):
dstImgFile = join(dstDir, imgFile.replace(srcDir, ""))
img = cv2.imread(imgFile)
rescaledImg = cv2.resize(img,
dsize=(1280, 720),
interpolation=cv2.INTER_CUBIC)
cv2.imwrite(dstImgFile, rescaledImg)
def solve(self,
n_iteration=100,
save_likelihood=False,
save_parameter=False,
save_wav=False,
save_path="./",
interval_save_parameter=30,
mic_index=MIC_INDEX):
"""
Parameters:
save_likelihood: boolean
save likelihood and lower bound or not
save_parameter: boolean
save parameter or not
save_wav: boolean
save intermediate separated signal or not
save_path: str
directory for saving data
interval_save_parameter: int
interval of saving parameter
"""
self.n_iteration = n_iteration
self.initialize_PSD()
self.initialize_covarianceMatrix()
self.make_filename_suffix()
log_likelihood_array = []
for it in progressbar(range(self.n_iteration)):
self.update()
if save_parameter and ((it + 1) % interval_save_parameter
== 0) and ((it + 1) != self.n_iteration):
self.save_parameter(
save_path + "{}-parameter-{}-{}.pic".format(
self.method_name, self.filename_suffix, it + 1))
if save_wav and ((it + 1) % interval_save_parameter
== 0) and ((it + 1) != self.n_iteration):
self.separate_FastWienerFilter(mic_index=MIC_INDEX)
self.save_separated_signal(
save_path + "{}.wav".format(self.filename_suffix))
if save_likelihood and ((it + 1) % interval_save_parameter
== 0) and ((it + 1) != self.n_iteration):
log_likelihood_array.append(self.calculate_log_likelihood())
if save_parameter:
self.save_parameter(save_path + "{}-parameter-{}.pic".format(
self.method_name, self.filename_suffix))
if save_likelihood:
log_likelihood_array.append(self.calculate_log_likelihood())
pic.dump(
log_likelihood_array,
open(
save_path + "{}-likelihood-interval={}-{}.pic".format(
self.method_name, interval_save_parameter,
self.filename_suffix), "wb"))
self.separate_FastWienerFilter(mic_index=mic_index)
self.save_separated_signal(save_path +
"{}.wav".format(self.filename_suffix))
bar = progressbar.ProgressBar(max_value=len(local_time_global_ids))
with mp.Pool(processes = ncpus) as p:
for i, _ in enumerate( p.imap_unordered(save_one_file, local_time_global_ids), 1 ):
bar.update(i)
print('Generating main pvd file')
with open(main_filename,'w') as f:
f.write('<?xml version="1.0"?>\n')
f.write('<VTKFile type="Collection" version="0.1"\n')
f.write(' byte_order="LittleEndian"\n')
f.write(' compressor="vtkZLibDataCompressor">\n')
f.write(' <Collection>\n')
for i in progressbar.progressbar(range(len(times))):
filename = "pts_{:010d}.vtp".format(i)
if isfile(filename):
f.write(f'<DataSet timestep="{times[i]}" group="" part="0"\n')
f.write(f' file="{filename}"/>\n')
f.write(' </Collection>\n')
f.write('</VTKFile>\n')
def main():
parser = argparse.ArgumentParser(
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
# labels file
parser.add_argument('labels_file')
# en-cn file
parser.add_argument('en_cn_file')
# directory where contains labelme annotated json files
parser.add_argument('in_dir', help='input dir with annotated files')
# output directory for dataset
parser.add_argument('out_dir', help='output dataset directory')
args = parser.parse_args()
# build a chinese-english label convert dict
(cn2ens, en2cns) = en_cn_dict_build(args.en_cn_file)
# regex pattern(used in get image name)
pattern = re.compile(r"\d+") # e.g. use time as file name
# 1. Create Directories
# remove output dir manually when exists
if osp.exists(args.out_dir):
print('Output directory already exists:', args.out_dir)
quit(1)
# make voc format directories
os.makedirs(args.out_dir)
os.makedirs(osp.join(args.out_dir, 'JPEGImages'))
os.makedirs(osp.join(args.out_dir, 'Annotations'))
os.makedirs(osp.join(args.out_dir, 'AnnotationsVisualization'))
print('dataset dir is:', args.out_dir)
# 2. Get Label Information
# get labels and save it to dataset dir
class_names = []
for i, line in enumerate(
open(args.labels_file, 'r', encoding='UTF-8').readlines()):
class_id = i - 1 # starts with -1
class_name = line.strip()
if class_id == -1:
assert class_name == '__ignore__'
continue
elif class_id == 0:
assert class_name == '_background_'
class_names.append(class_name)
else:
class_names.append(cn2ens[class_name])
class_names = tuple(class_names)
print('class_names:', class_names)
out_class_names_file = osp.join(
args.out_dir, 'class_names.txt') # save labels in txt for information
with open(out_class_names_file, 'w', encoding='UTF-8') as f:
f.writelines('\n'.join(class_names))
print('Saved class_names:', out_class_names_file)
# 3. Process Every Json File
label_file_list = glob.glob(osp.join(args.in_dir, '*.json'))
for i in progressbar.progressbar(range(len(label_file_list))):
label_file = label_file_list[i]
# load json
# print('Generating dataset from:', label_file)
with open(label_file, 'r', encoding='UTF-8') as f:
data = json.load(f)
# regex get img name
filename = osp.splitext(osp.basename(label_file))[0]
base = pattern.findall(filename)[0]
# src image file
out_img_file = osp.join(args.out_dir, 'JPEGImages', base + '.jpg')
# annotation xml file
out_xml_file = osp.join(args.out_dir, 'Annotations', base + '.xml')
# visualize image file
out_viz_file = osp.join(args.out_dir, 'AnnotationsVisualization',
base + '_Viz.jpg')
# color annotated image file
out_colorize_file = osp.join(args.out_dir, 'AnnotationsVisualization',
base + '_ColorViz.jpg')
# labelme annotated file contains source image data(serialized)
# get img data as numpy array
imgdata = data['imageData']
img = utils.img_b64_to_arr(imgdata)
# also you can read image from img path too
# img_file = osp.join(osp.dirname(label_file), data['imagePath'])
# img = np.asarray(PIL.Image.open(img_file))
# save the origin image file
PIL.Image.fromarray(img).save(out_img_file)
# lxml is a wonderful xml tool
# generate voc format annotation file
maker = lxml.builder.ElementMaker()
xml = maker.annotation(
maker.folder(""), # folder name
maker.filename(base + '.jpg'), # img path
maker.source( # img source, doesn't matter
maker.database(""),
maker.annotation(""),
maker.image(""),
),
maker.size( # image size(height, width and channel)
maker.height(str(img.shape[0])),
maker.width(str(img.shape[1])),
maker.depth(str(img.shape[2])),
),
maker.segmented("0"), # if for segmentation
)
# two list for visualization
bboxes = []
labels = []
for shape in data['shapes']:
class_name = shape['label'] # object name in json file
class_id = class_names.index(
cn2ens[class_name]) # convert to class id
# box info from annotated points
xmin = shape['points'][0][0]
ymin = shape['points'][0][1]
xmax = shape['points'][2][0]
ymax = shape['points'][2][1]
bboxes.append([xmin, ymin, xmax, ymax])
labels.append(class_id)
#
xml.append(
maker.object( # object info
maker.name(
cn2ens[shape['label']]), # label name(in english)
maker.pose(""), # pose info, doesn't matter
maker.truncated("0"), # truncated info, doesn't matter
maker.difficult("0"), # diificulty, doesn't matter
maker.
bndbox( # bbox(up-left corner and bottom-right corner points)
maker.xmin(str(xmin)),
maker.ymin(str(ymin)),
maker.xmax(str(xmax)),
maker.ymax(str(ymax)),
),
))
# caption for visualize drawing
captions = [class_names[l] for l in labels]
viz = labelme.utils.draw_instances(img,
bboxes,
labels,
captions=captions)
# save it
PIL.Image.fromarray(viz).save(out_viz_file)
# another visualize format (colored mask in bbox)
# convert label to id
label_name_to_value = {'背景': 0}
for shape in sorted(data['shapes'], key=lambda x: x['label']):
label_name = shape['label']
if label_name in label_name_to_value:
label_value = label_name_to_value[label_name]
else:
label_value = len(label_name_to_value)
label_name_to_value[label_name] = label_value
# labelme's function
lbl = utils.shapes_to_label(img.shape, data['shapes'],
label_name_to_value)
label_names = [None] * (max(label_name_to_value.values()) + 1)
for name, value in label_name_to_value.items():
label_names[value] = cn2ens[name] # annotated in english
# labelme's function
lbl_viz = utils.draw_label(lbl, img, label_names)
# save img
PIL.Image.fromarray(lbl_viz).save(out_colorize_file)
# save voc annotation to xml file
with open(out_xml_file, 'wb') as f:
f.write(lxml.etree.tostring(xml, pretty_print=True))
def optimize(self, Xs, Ys, Ys_type=None, split_rate=0.7):
Ys = reformat_np_arr(Ys,
self.estimator.model_Ys_type,
from_np_arr_type=Ys_type)
dataset = Dataset()
dataset.add_data('Xs', Xs)
dataset.add_data('Ys', Ys)
train_set, test_set = dataset.split((split_rate, 1 - split_rate),
shuffle=False)
train_Xs, train_Ys = train_set.full_batch()
test_Xs, test_Ys = test_set.full_batch()
train_Ys = reformat_np_arr(train_Ys,
self.estimator.model_Ys_type,
from_np_arr_type=Ys_type)
test_Ys = reformat_np_arr(test_Ys,
self.estimator.model_Ys_type,
from_np_arr_type=Ys_type)
param_grid_size = self.param_grid_size
self.log.info(("optimize [%s], total %s's candidate estimator" %
(self.estimator, param_grid_size)))
self.result = []
class_ = self.estimator.__class__
gen_param = self.gen_param()
for _ in progressbar.progressbar(range(param_grid_size),
redirect_stdout=False):
try:
param = next(gen_param)
estimator = class_(**param)
estimator.fit(train_Xs, train_Ys)
train_score = estimator.score(train_Xs, train_Ys)
test_score = estimator.score(test_Xs, test_Ys)
predict = estimator.predict(test_Xs)
auc_score = sklearn.metrics.roc_auc_score(test_Ys, predict)
result = {
"train_score": train_score,
"test_score": test_score,
"param": param,
"auc_score": auc_score,
}
self.result += [result]
except KeyboardInterrupt as e:
log_error_trace(self.log.error, e)
raise KeyboardInterrupt
except BaseException as e:
self.log.warn(f'while optimize param {param} raise {e}')
self.result = sorted(self.result,
key=lambda a: (
-a["auc_score"],
-a["test_score"],
-a["train_score"],
))
self.best_param = self.result[0]["param"]
estimator = class_(**self.best_param)
estimator.fit(
train_Xs,
train_Ys,
)
return estimator
def train(net,
criterion,
optimizer,
dataloaders,
epoch,
device=torch.device('cuda:0'),
hist_args={}):
history = History(**hist_args)
for e in range(epoch):
epoch_loss = 0.
for phase in ['train', 'valid']:
y_encoder = dataloaders[phase].dataset.y_encoder
if phase == 'train':
net.train()
prefix = '%d, Training: ' % e
else:
net.eval()
prefix = '%d, Validation: ' % e
# validation时需要记录所有的真实标签和预测
epoch_label, epoch_loc = [], []
epoch_res_c, epoch_res_s, epoch_res_l = [], [], []
for batch in pb.progressbar(dataloaders[phase], prefix=prefix):
imgs, targets = batch[0].to(device), batch[-1].to(device)
with torch.set_grad_enabled(phase == 'train'):
preds = net(imgs)
loss = criterion(preds, targets)
if phase == 'train':
optimizer.zero_grad()
loss.backward()
optimizer.step()
else:
# 如果是eval，需要记录所有的预测，用于计算mAP
img_size = list(imgs.shape[-1:-3:-1])
epoch_label.extend(batch[1])
epoch_loc.extend(batch[2])
for pred in preds:
res_c, res_s, res_l = y_encoder.decode(
pred, img_size)
epoch_res_c.append(res_c)
epoch_res_s.append(res_s)
epoch_res_l.append(res_l)
with torch.no_grad():
epoch_loss += loss.item()
# 整个epoch的所有batches循环结束
with torch.no_grad():
num_batches = len(dataloaders[phase].dataset)
epoch_loss /= num_batches
if phase == 'train':
history.update_hist(loss=epoch_loss)
print('%d, %s, loss: %.4f' % (e, phase, epoch_loss))
else:
aps, map_score = mAP(epoch_label, epoch_loc, epoch_res_c,
epoch_res_s, epoch_res_l)
history.update_hist(val_loss=epoch_loss, mAP=map_score)
if map_score is np.nan:
map_score = 0
history.update_best(mAP=map_score,
loss=epoch_loss,
epoch=e,
model_wts=copy.deepcopy(
net.state_dict()))
print('%d, %s, loss: %.4f, mAP: %.4f' %
(e, phase, epoch_loss, map_score))
if 'valid' in dataloaders.keys():
net.load_state_dict(history.best[0]['model_wts'])
print('valid best loss: %.4f, best mAP: %.4f' %
(history.best[0]['loss'], history.best[0]['mAP']))
return history
def __init__(self, collection, resolution=None, margin=0.05, verbose=True):
"""Constructor of :class:`SaptialIndex` class
TODO: update documentation
Parameters
----------
features : bbox() + iterable
TrackCollection : on construit une grille
dont l’emprise est calculée sur la fonction getBBox
de TrackCollection et dans un deuxième temps on appelle
addSegment([c1,c2], [i,j]) pour chaque segment [c1,c2]
(localisé entre les points GPS i et i+1) de chaque trace j,
de la collection.
Network : on construit une grille (de taille 100 x 100) par défaut,
dont l’emprise est calculée sur celle du réseau, et on appelle
addSegment ([c1,c2], [i,j]) pour chaque segment [c1,c2]
(localisé entre les points GPS i et i+1) de chaque tronçon j,
du réseau.
resolution : tuple (xsize, ysize)
DESCRIPTION. The default is (100, 100).
Returns
-------
None.
"""
# Bbox only or collection
if isinstance(collection, Bbox):
bb = collection
else:
bb = collection.bbox()
bb = bb.copy()
bb.addMargin(margin)
(self.xmin, self.xmax, self.ymin, self.ymax) = bb.asTuple()
ax, ay = bb.getDimensions()
if resolution is None:
am = max(ax, ay)
r = am / 100
resolution = (int(ax / r), int(ay / r))
else:
r = resolution
resolution = (int(ax / r[0]), int(ay / r[1]))
self.collection = collection
# Keeps track of registered features
self.inventaire = set()
# Nombre de dalles par cote
self.csize = resolution[0]
self.lsize = resolution[1]
# print ('nb cellule', self.xsize * self.ysize)
# Tableau de collections de features appartenant a chaque dalle.
# Un feature peut appartenir a plusieurs dalles.
self.grid = []
for i in range(self.csize):
self.grid.append([])
for j in range(self.lsize):
self.grid[i].append([])
self.dX = ax / self.csize
self.dY = ay / self.lsize
# Calcul de la grille
if isinstance(collection, tuple):
self.collection = TrackCollection()
return
boucle = range(collection.size())
if verbose:
print(
"Building ["
+ str(self.csize)
+ " x "
+ str(self.lsize)
+ "] spatial index..."
)
boucle = progressbar.progressbar(boucle)
for num in boucle:
feature = collection[num]
# On récupere la trace
if isinstance(feature, Track):
self.addFeature(feature, num)
# On récupère l'arc du reseau qui est une trace
elif isinstance(feature, Edge):
self.addFeature(feature.geom, num)
def start(self):
#Open the image and blur it a little
beg = cv2.imread(self.look_path)
#print(self.look_path)
beg = cv2.GaussianBlur(beg, (self.blur_amount, self.blur_amount), 0)
avg_color = average_color(beg)
warnings.simplefilter(action='ignore', category=FutureWarning)
if self.starting_img == "":
blank = np.zeros((beg.shape[0], beg.shape[1], beg.shape[2]),
np.uint8)
cv2.rectangle(blank, (0, 0), (beg.shape[1], beg.shape[0]),
avg_color, -1)
else:
blank = self.starting_img
#cv2.imwrite(my_path + "poo.png", blank)
main_dots = []
for i in progressbar(range(self.number_of_dots)):
#progress(i, max=self.number_of_dots)
difference = cv2.absdiff(beg, blank)
if self.diff_blur_amount > 0:
difference = cv2.GaussianBlur(
difference, (self.diff_blur_amount, self.diff_blur_amount),
0)
(minVal, maxVal, minLoc,
maxLoc) = cv2.minMaxLoc(gray_img(difference))
pos = (beg[maxLoc[1], maxLoc[0]])
b, g, r = pos
dot = dot_series.Dot((maxLoc[1], maxLoc[0]), 1, (b, g, r))
changer = 0
inactive_count = 0
while inactive_count < 10:
if changer == 0:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].radius += 1
choises[1].radius += 5
choises[2].radius += 10
choises[3].radius = max(choises[2].radius - 1, 1)
choises[4].radius = max(choises[2].radius - 5, 1)
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
elif changer == 1:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].color = (min(choises[1].color[0] + 5,
255), choises[1].color[1],
choises[1].color[2])
choises[1].color = (min(choises[1].color[0] + 1,
255), choises[1].color[1],
choises[1].color[2])
choises[1].color = (max(choises[1].color[0] - 5,
0), choises[1].color[1],
choises[1].color[2])
choises[1].color = (max(choises[1].color[0] - 1,
0), choises[1].color[1],
choises[1].color[2])
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
elif changer == 2:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].color = (choises[1].color[0],
min(choises[1].color[1] + 5,
255), choises[1].color[2])
choises[1].color = (choises[1].color[0],
min(choises[1].color[1] + 1,
255), choises[1].color[2])
choises[1].color = (choises[1].color[0],
max(choises[1].color[1] - 5,
0), choises[1].color[2])
choises[1].color = (choises[1].color[0],
max(choises[1].color[1] - 1,
0), choises[1].color[2])
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
elif changer == 3:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].color = (choises[1].color[0],
choises[1].color[1],
min(choises[1].color[2] + 5, 255))
choises[1].color = (choises[1].color[0],
choises[1].color[1],
min(choises[1].color[2] + 1, 255))
choises[1].color = (choises[1].color[0],
choises[1].color[1],
max(choises[1].color[2] - 5, 0))
choises[1].color = (choises[1].color[0],
choises[1].color[1],
max(choises[1].color[2] - 1, 0))
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
elif changer == 4:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].position = (choises[1].position[0] + 1,
choises[1].position[1])
choises[2].position = (choises[2].position[0] - 1,
choises[2].position[1])
choises[3].position = (choises[3].position[0],
choises[3].position[1] + 1)
choises[4].position = (choises[4].position[0],
choises[4].position[1] - 1)
choises[5].position = (choises[1].position[0] + 5,
choises[1].position[1])
choises[6].position = (choises[2].position[0] - 5,
choises[2].position[1])
choises[7].position = (choises[3].position[0],
choises[3].position[1] + 5)
choises[8].position = (choises[4].position[0],
choises[4].position[1] - 5)
choises[9].position = (choises[1].position[0] + 1,
choises[1].position[1] + 1)
choises[10].position = (choises[2].position[0] - 1,
choises[2].position[1] + 1)
choises[11].position = (choises[3].position[0] + 1,
choises[3].position[1] - 1)
choises[12].position = (choises[4].position[0] - 1,
choises[4].position[1] - 1)
choises[13].position = (choises[1].position[0] + 5,
choises[1].position[1] + 5)
choises[14].position = (choises[2].position[0] - 5,
choises[2].position[1] + 5)
choises[15].position = (choises[3].position[0] + 5,
choises[3].position[1] - 5)
choises[16].position = (choises[4].position[0] - 5,
choises[4].position[1] - 5)
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
else:
choises = [
copy.deepcopy(dot),
copy.deepcopy(dot),
copy.deepcopy(dot)
]
choises[1].infront = True
choises[2].infront = False
dot, winner = find_winner(choises, dot, main_dots, beg,
blank)
if winner == 0:
inactive_count += 1
else:
inactive_count = 0
changer += 1
if changer == 6:
changer = 0
if dot.infront:
main_dots.append(dot)
else:
main_dots.insert(0, dot)
blank = plot_dots(blank, main_dots)
#difference = cv2.absdiff(beg, blank)
cv2.imwrite(self.final_path, blank)
"""
print(main_dots[0].position)
print(main_dots[0].radius)
print(main_dots[0].color)
cv2.imshow("beg", beg)
cv2.imshow("blank", blank)
cv2.imshow("diff", difference)
cv2.waitKey(0)
cv2.destroyAllWindows()
"""
#plot_dots(img, main_dots)
#progress(self.number_of_dots, max=self.number_of_dots)
print("")
return blank
def _calc_test_hvps_log_loss(criterion: Callable,
trained_model: nn.Module,
train_dataloader: DeviceDataLoader,
test_dataloader: DataLoader,
run_params: Mapping,
diff_wrt: Optional[torch.Tensor] = None,
show_progress: bool = False):
device = train_dataloader.device
diff_wrt = maybe(
diff_wrt, [p for p in trained_model.parameters() if p.requires_grad])
if run_params['use_gauss_newton']:
matmul_class: Union[Type[GNP], Type[HVP]] = GNP
damping = 0.001
else:
matmul_class = HVP
damping = 0.01
matmul = matmul_class(
calc_loss=lambda xs, target: criterion(trained_model(*xs), target),
parameters=diff_wrt,
data=train_dataloader,
data_len=len(train_dataloader),
damping=damping,
cache_batch=True)
cg = CG(matmul=matmul,
result_len=sum(p.numel() for p in diff_wrt),
max_iters=maybe(run_params['max_cg_iters'],
sum(p.numel() for p in diff_wrt)))
test_hvps: List[torch.Tensor] = []
iterator = progressbar(
test_dataloader) if show_progress else test_dataloader
for batch in iterator:
x_test, target = to_device(batch, device)
loss_at_x_test = criterion(trained_model(*x_test), target.squeeze())
grads = autograd.grad(loss_at_x_test, diff_wrt)
grad_at_z_test = collect(grads)
if len(test_hvps) != 0:
init = test_hvps[-1].detach().clone()
else:
init = torch.zeros_like(grad_at_z_test)
def _min(x, grad_at_z_test=grad_at_z_test):
x_tens = torch.tensor(
x, device=device) if not isinstance(x, torch.Tensor) else x
grad_tens = torch.tensor(
grad_at_z_test, device=device) if not isinstance(
grad_at_z_test, torch.Tensor) else grad_at_z_test
return np.array(0.5 * matmul(x_tens).dot(x_tens) -
grad_tens.dot(x_tens))
def _grad(x, grad_at_z_test=grad_at_z_test):
x_tens = torch.tensor(
x, device=device) if not isinstance(x, torch.Tensor) else x
return np.array(matmul(x_tens) - grad_at_z_test)
def _hess(x, p):
grad_tens = torch.tensor(
p, device=device) if not isinstance(p, torch.Tensor) else p
return np.array(matmul(grad_tens))
if getattr(run_params, 'use_scipy', False):
test_hvps.append(
torch.tensor(fmin_ncg(f=_min,
x0=init,
fprime=_grad,
fhess_p=_hess,
avextol=1e-8,
maxiter=100,
disp=False),
device=device))
else:
test_hvps.append(
cg.solve(grad_at_z_test,
test_hvps[-1] if len(test_hvps) != 0 else None))
matmul.clear_batch()
return torch.stack(test_hvps)
def run(self, processes=None):
all_nodes = list(self.network.A['physical'])
for i in progressbar.progressbar(range(self.tmin, self.tmax)):
self.run_chain(all_nodes, processes, len(all_nodes))
self.times.append(i)
def get_test_dataset_iterator(display_progress=False):
iterator = read_dataset_from_csv(test_dataset_path, columns=COLUMNS, chunk_size=TESTING_CHUNK_SIZE)
if display_progress:
iterator = progressbar(iterator, max_value=TESTING_CHUNKS_COUNT, max_error=False)
return get_dataset_iterator(iterator)
def add_task(task_name, dataset_name=None, local_path=None,
dataset_address=None, preproc_address=None,
include_predictors=None, exclude_predictors=None,
reingest=False, scan_length=1000,
dataset_summary=None, url=None, task_summary=None, **kwargs):
""" Adds a BIDS dataset task to the database.
Args:
task_name - task to add
dataset_name - overide dataset name
local_path - path to local bids dataset.
dataset_address - remote address of BIDS dataset.
preproc_address - remote address of preprocessed files.
include_predictors - set of predictors to ingest
exclude_predictors - set of predictors to exclude from ingestions
reingest - force reingesting even if dataset already exists
scan_length - default scan length in case it cant be found in image
dataset_summary - Dataset summary description,
url - Dataset external link,
task_summary - Task summary description,
kwargs - arguments to filter runs by
Output:
dataset model id
"""
cache.clear()
if dataset_address is not None and local_path is None:
local_path = install(
source=dataset_address,
path=(Path(
current_app.config['DATASET_DIR']) / dataset_name).as_posix()
).path
local_path = Path(local_path)
assert isfile(str(local_path / 'dataset_description.json'))
layout = BIDSLayout(str(local_path), derivatives=True)
if task_name not in layout.get_tasks():
raise ValueError("Task {} not found in dataset {}".format(
task_name, local_path))
dataset_name = dataset_name if dataset_name is not None \
else layout.description['Name']
# Get or create dataset model from mandatory arguments
dataset_model, new_ds = get_or_create(Dataset, name=dataset_name)
if new_ds:
dataset_model.description = layout.description
dataset_model.summary = dataset_summary
dataset_model.url = url
dataset_model.dataset_address = dataset_address
dataset_model.preproc_address = preproc_address
dataset_model.local_path = local_path.as_posix()
db.session.commit()
elif not reingest:
print("Dataset found, skipping ingestion...")
# return dataset_model.id
# Get or create task
task_model, new_task = get_or_create(
Task, name=task_name, dataset_id=dataset_model.id)
if new_task:
task_model.description = json.load(
(local_path / 'task-{}_bold.json'.format(task_name)).open())
task_model.summary = task_summary
task_model.TR = task_model.description['RepetitionTime']
db.session.commit()
stims_processed = {}
""" Parse every Run """
print("Parsing runs")
'''
all_runs = layout.get(task=task_name, suffix='bold', extensions='.nii.gz',
desc=None, **kwargs)
'''
# this worked one above didn't
all_runs = layout.get(task=task_name, suffix='bold', extensions='.nii.gz')
for img in progressbar(all_runs):
""" Extract Run information """
# Get entities
entities = {entity: getattr(img, entity)
for entity in ['subject', 'session', 'acquisition']
if entity in img.entities}
run_number = img.run if hasattr(img, 'run') else None
run_model, new = get_or_create(
Run, dataset_id=dataset_model.id, number=run_number,
task_id=task_model.id, **entities)
entities['task'] = task_model.name
if run_number:
run_number = str(run_number).zfill(2)
entities['run'] = run_number
# Get duration (helps w/ transformations)
if img.image is not None:
run_model.duration = img.image.shape[3] * \
img.image.header.get_zooms()[-1]
else:
run_model.duration = scan_length
# Put back as int
if 'run' in entities:
entities['run'] = int(entities['run'])
""" Extract Predictors"""
# Assert event files exist (for DataLad)
for e in layout.get_nearest(
img.path, suffix='events', all_=True, strict=False):
assert isfile(e)
collection = layout.get_collections(
'run', scan_length=run_model.duration, **entities)[0]
if 'stim_file' in collection.variables:
stims = collection.variables.pop('stim_file')
else:
stims = None
add_predictor_collection(
collection, dataset_model.id, run_model.id,
include=include_predictors,
exclude=exclude_predictors, TR=task_model.TR)
""" Ingest Stimuli """
if stims is not None:
for i, val in enumerate(stims.values):
stim_path = local_path / 'stimuli' / val
if val not in stims_processed:
try:
stim_hash = hash_stim(stim_path)
except OSError:
current_app.logger.debug(
'{} not found.'.format(stim_path))
continue
stims_processed[val] = stim_hash
else:
stim_hash = stims_processed[val]
stim_model, _ = add_stimulus(
stim_hash, path=stim_path, dataset_id=dataset_model.id)
# Get or create Run Stimulus association
runstim, _ = get_or_create(
RunStimulus, stimulus_id=stim_model.id,
run_id=run_model.id,
onset=stims.onset.tolist()[i],
duration=stims.duration.tolist()[i])
""" Add GroupPredictors """
print("Adding group predictors")
add_group_predictors(dataset_model.id, local_path / 'participants.tsv')
return dataset_model.id
print('Generating main pvd file')
with open(main_filename,'w') as f:
f.write('<?xml version="1.0"?>\n')
f.write('<VTKFile type="Collection" version="0.1"\n')
f.write(' byte_order="LittleEndian"\n')
f.write(' compressor="vtkZLibDataCompressor">\n')
f.write(' <Collection>\n')
filename = "pts_{:010d}.vtp".format(0)
if isfile(filename):
f.write(f'<DataSet timestep="0" group="" part="0"\n')
f.write(f' file="{filename}"/>\n')
for filenumber in progressbar.progressbar(filenumbers):
filename = "pts_{:010d}.vtp".format(filenumber)
if isfile(filename):
f.write(f'<DataSet timestep="{np.round(filenumber*timestep,10)}" group="" part="0"\n')
f.write(f' file="{filename}"/>\n')
f.write(' </Collection>\n')
f.write('</VTKFile>\n')
#%%
import random
import string
import pandas as pd
import progressbar
from fuzzywuzzy import process
from multiprocessing.dummy import Pool as ThreadPool
max_thread = 4
#%%
df = pd.DataFrame(index=range(15000), columns=['to_match'])
df_opt = pd.DataFrame(index=range(28000), columns=['option'])
for i, row in progressbar.progressbar(df.iterrows(), max_value=len(df)):
length = random.randint(5, 10)
df.at[i, 'to_match'] = ''.join(
random.choices(string.ascii_uppercase, k=length))
for i, row in progressbar.progressbar(df_opt.iterrows(),
max_value=len(df_opt)):
length = random.randint(5, 10)
df_opt.at[i, 'option'] = ''.join(
random.choices(string.ascii_uppercase, k=length))
df_opt = df_opt['option']
#%%
df['score'] = 'None'
df['match'] = 'None'
l_tpl_lev = []
def wait():
for i in progressbar.progressbar(range(1800)):
sleep(1)
import time import progressbar for i in progressbar.progressbar(range(100)): time.sleep(0.02)
def __init__(self, **kwargs):
import progressbar as pb
self.progressbarvar = pb.progressbar(**kwargs)
return None
def reset_ids(self, target_classes, import_objects,
import_objects_by_type_and_id, existing_classes,
map_treenodes=True, save=True):
"""Reset the ID of each import object to None so that a new object will
be created when the object is saved. At the same time an index is
created that allows per-type lookups of foreign key fields
"""
logger.info("Building foreign key update index")
# Build index for foreign key fields in models. For each type, map
# each foreign key name to a model class.
fk_index = defaultdict(dict)
for c in target_classes:
class_index = fk_index[c]
foreign_key_fields = [
f for f in c._meta.get_fields()
if f.is_relation
and f.many_to_one # ForeignKey instances
#if field.get_internal_type() == 'ForeignKey':
and f.related_model in target_classes
]
for field in foreign_key_fields:
# Get the database column name for this field
class_index[field.attname] = field.related_model
logger.info("Updating foreign keys to imported objects with new IDs")
all_classes = dict()
all_classes.update(existing_classes)
updated_fk_ids = 0
unchanged_fk_ids = 0
explicitly_created_summaries = 0
other_tasks = set(import_objects.keys()) - set(ordered_save_tasks)
# Iterate objects to import and respect dependency order
for object_type in ordered_save_tasks + list(other_tasks):
objects = import_objects.get(object_type)
if not objects:
# No objects of this object type are imported
continue
fk_fields = fk_index[object_type]
# No need to do rest if there are no foreign keys to change to begin
# with.
if len(fk_fields) == 0:
continue
imported_parent_nodes = []
bar_prefix = "- {}: ".format(object_type.__name__)
for deserialized_object in progressbar.progressbar(objects,
max_value=len(objects), redirect_stdout=True,
prefix=bar_prefix):
obj = deserialized_object.object
obj_type = type(obj)
for fk_field, fk_type in six.iteritems(fk_fields):
# Get import object with the former ID referenced in
# this field.
current_ref = getattr(obj, fk_field)
# Get updated model objects of the referenced type
imported_objects_by_id = import_objects_by_type_and_id[fk_type]
ref_obj = imported_objects_by_id.get(current_ref)
if ref_obj:
# Update foreign key reference to ID of newly saved
# object. Only for treenodes this is expected to result
# in not yet available data
if object_type == Treenode and fk_type == Treenode:
imported_parent_nodes.append((obj, current_ref))
elif ref_obj.id is None:
raise ValueError("The referenced {} object '{}' with import ID {} wasn't stored yet".format(
fk_type, str(ref_obj), current_ref))
setattr(obj, fk_field, ref_obj.id)
updated_fk_ids += 1
else:
unchanged_fk_ids += 1
# Save objects if they should either be imported or have change
# foreign key fields
if save and (updated_fk_ids or obj.id is None):
obj.save()
# Treenodes are special, because they can reference themselves. They
# need therefore a second iteration of reference updates after all
# treenodes have been saved and new IDs are available.
if map_treenodes and object_type == Treenode:
logger.info('Mapping parent IDs of treenodes to imported data')
imported_objects_by_id = import_objects_by_type_and_id[Treenode]
for obj, parent_id in progressbar.progressbar(imported_parent_nodes,
max_value=len(imported_parent_nodes),
redirect_stdout=True, prefix="- Mapping parent treenodes: "):
new_parent = imported_objects_by_id.get(parent_id)
if not new_parent:
raise ValueError("Could not find imported treenode {}".format(parent_id))
obj.parent_id = new_parent.id
if save:
obj.save()
# Update list of known classes after new classes have been saved
if object_type == Class:
for deserialized_object in objects:
obj = deserialized_object.object
all_classes[obj.class_name] = obj.id
# If skeleton class instances are created, make sure the skeleton
# summary table entries for the respective skeletons are there.
# Otherwise the ON CONFLICT claues of the summary update updates can
# be called multiple times. The alternative is to disable the
# trigger during import.
pre_create_summaries = False
if object_type == ClassInstance and pre_create_summaries:
skeleton_class_id = all_classes.get('skeleton')
for deserialized_object in objects:
obj = deserialized_object.object
if obj.class_column_id == skeleton_class_id:
r = SkeletonSummary.objects.get_or_create(project=self.target, skeleton_id=obj.id)
explicitly_created_summaries += 1
logger.info("".join(["{} foreign key references updated, {} did not ",
"require change, {} skeleton summaries were created"]).format(
updated_fk_ids, unchanged_fk_ids, explicitly_created_summaries))
coords = pandas.read_csv(coords_file, header=None, delim_whitespace=True)
print('Reading boundary codes')
codes = pandas.read_csv(boundary_label_file, header=None, delim_whitespace=True)
# In[9]:
print('Passing coordinates to vtk')
pts = vtk.vtkPoints()
pts.SetNumberOfPoints(coords.shape[0])
vv = coords.values
for i in progressbar.progressbar(range(coords.shape[0])):
pts.SetPoint(i, vv[i,1:])
# In[25]:
print('Passing boundaries to vtk')
boundaries = vtk.vtkCellArray()
with open(boundary_file,'r') as f:
for line in f:
data = line.strip().split()
import time
import progressbar
import numpy as np
for i in progressbar.progressbar(np.arange(100)):
time.sleep(0.02)
with progressbar.ProgressBar(max_value=10) as bar:
for i in range(10):
time.sleep(0.1)
bar.update(i)
# print('success_list: {}'.format(success_list))
# print('norm_success: {}'.format(norm_success))
ax.plot(alpha, norm_success, label='N: {}'.format(N))
if __name__ == "__main__":
# Run your code in order to study Perceptron training at least for the following parameter settings:
# N = 20, P = αN with α = 0.75, 1.0, 1.25, . . . 3.0, nD =50, n_max =100
n_max = 100
nD = 50
N_array = [5, 10, 15, 20, 50, 100]
# np.random.seed(0) # To make reproducible sets (if needed)
# determine the value of the fraction of successful runs as a function of alpha=P/N
alpha = np.arange(0.75, 3.25, 0.25)
fig = plt.figure()
ax = plt.subplot(111)
for N in progressbar.progressbar(N_array):
test_runs(n_max, nD, N, alpha, ax)
plt.xlabel(r'$\alpha = P/N$')
plt.ylabel(r'$Q_{l.s.}$')
ax.legend()
fig.savefig('Lab1/graphs/Q-alpha-graph.png')
plt.show()
line = f.readline().split()
nelement_types = int(line[0])
nelements = 0
for k in range(nelement_types):
line = f.readline().split()
nelements = nelements + int(line[1])
f.readline() #1.0 1.0
f.readline() #fensap input file
pts = vtk.vtkPoints()
pts.SetNumberOfPoints(npoints)
for i in progressbar(range(npoints)):
line = f.readline().split()
pts.SetPoint(i, float(line[0]), float(line[1]), float(line[2]))
cells = vtk.vtkCellArray()
celltypes = []
c = 0
with ProgressBar(max_value=nelements) as bar:
while True:
line = f.readline()
if line=='':
break
line = line.split()
nverts = len(line)
#! /usr/bin/env python3
import os, sys, timeit, progressbar
# Get command-line arguments
(F, N), T = map(int, sys.argv[1:3]), float(sys.argv[3])
# Generate a random file hierarchy
print(f'1. Generate {F} random files for testing:')
for f in progressbar.progressbar(range(0, F)):
os.system(f'mkdir -p test; dd if=/dev/urandom of=test/{f} bs=1 count=1024 > /dev/null 2>&1')
# Mesure 10 execution of a call to md5sumsum on our test dataset
print(f'2. Mesure {N} executions of our script:')
x = timeit.timeit("""
import os
os.system('./md5sumsum test')
""", number=N)
# Return an error exit code to Travis CI if our script is too slow
print(f'Our execution time is {x}s, our test goal was {T}s')
sys.exit(1 if x > T else 0)
wr.SetFileName(filename)
wr.Write()
bar = progressbar.ProgressBar(max_value=len(unique_times))
with mp.Pool(processes = ncpus) as p:
for i, _ in enumerate(p.imap_unordered(save_one_file, range(len(unique_times))), 1):
bar.update(i)
print('Generating main pvd file')
with open(main_filename,'w') as f:
f.write('<?xml version="1.0"?>\n')
f.write('<VTKFile type="Collection" version="0.1"\n')
f.write(' byte_order="LittleEndian"\n')
f.write(' compressor="vtkZLibDataCompressor">\n')
f.write(' <Collection>\n')
for i in progressbar.progressbar(range(len(unique_times))):
if empty_file[i]==0:
filename = "pts_{:010d}.vtp".format(i)
f.write(f'<DataSet timestep="{unique_times[i]}" group="" part="0"\n')
f.write(f' file="{filename}"/>\n')
f.write(' </Collection>\n')
f.write('</VTKFile>\n')
def perceptron_forest_study(data,
trees,
fracs,
num_seeds=5,
modes=["with", "without"],
filename=None):
"""
Generate an accuracy vs number of decision tree plot for a perceptron forest.
Parameters
----------
data : (pandas.DataFrame, pandas.DataFrame)
Input data for the learning algorithms. Tuple ordered by
data and classifications.
trees : numpy.ndarray
Array containing number of trees to try in perceptron forest.
fracs : numpy.ndarray
Array containing fraction of training data each tree should be trained
on.
num_seeds : int, optional
Number of random seeds to average results over. The default is 5.
modes: list(str), len 1-2
List of modes to run. Options are 'with' for running with data replacement
and 'without' for running without data replacment. The default is both
(['with', 'without']).
filename : str, optional
File to save plot to. The default is None.
Returns
-------
t_w : int
Number of trees that optimizes accuracy with replacement.
f_w : float
Data fraction that optimizes accuracy with replacement.
t_w0 : int
Number of trees that optimizes accuracy without replacement.
f_w0 : float
Data fraction that optimizes accuracy without replacement.
"""
# Define random seeds and parameters
np.random.seed(
0) # Set seed of random number generator for reproducability
seeds = np.random.randint(0, 1E9, num_seeds)
# Loop through number of trees in PF
means_w, means_wo = [], []
errs_w, errs_wo = [], []
print("\nPerceptron Forest Study\n")
for n in progressbar(trees):
for frac in fracs:
# Define hyperparams
# w means with replacement, wo means without replacement
hyperparams_w = {
"n_estimators": n,
"max_samples": frac,
"bootstrap": True
}
hyperparams_wo = {
"n_estimators": n,
"max_samples": frac,
"bootstrap": False
}
# Loop through seeds
mean_w, var_w = 0.0, 0.0
mean_wo, var_wo = 0.0, 0.0
for seed in seeds:
if "with" in modes:
# Init learner
hyperparams_w["random_state"] = seed
PF_w = Project(data, "perceptron_forest", hyperparams_w)
# Evaluate model accuracy
d_w = PF_w.evaluate_model_accuracy()
# Tally mean and std to averaged quantities over seed
mean_w += d_w[0][1]
var_w += d_w[0][2]**2
if "without" in modes:
hyperparams_wo["random_state"] = seed
PF_wo = Project(data, "perceptron_forest", hyperparams_wo)
d_wo = PF_wo.evaluate_model_accuracy()
mean_wo += d_wo[0][1]
var_wo += d_wo[0][2]**2
# Add to list
if "with" in modes:
means_w.append(mean_w / num_seeds)
errs_w.append(d_w[0][3] * np.sqrt(var_w) / num_seeds)
if "without" in modes:
means_wo.append(mean_wo / num_seeds)
errs_wo.append(d_wo[0][3] * np.sqrt(var_wo) / num_seeds)
if "with" in modes:
# Convert everything to np arrays
means_w = np.array(means_w)
means_wo = np.array(means_wo)
# Make sure error bars are in (0,1)
acc_w_upper = means_w + errs_w
acc_w_lower = means_w - errs_w
acc_w_upper[acc_w_upper > 1.0] = 1.0
acc_w_lower[acc_w_lower < 0.0] = 0.0
if "without" in modes:
errs_w = np.array(errs_w)
errs_wo = np.array(errs_wo)
acc_wo_upper = means_wo + errs_wo
acc_wo_lower = means_wo - errs_wo
acc_wo_upper[acc_wo_upper > 1.0] = 1.0
acc_wo_lower[acc_wo_lower < 0.0] = 0.0
# Make mesh of combination of fractions and trees
f, t = np.meshgrid(fracs, trees)
f = f.flatten()
t = t.flatten()
fig = plt.figure()
ax = fig.add_subplot(111, projection='3d')
ax.set_xlabel("Number of Decision Trees")
ax.set_ylabel("Data Fraction / Tree")
ax.set_zlabel("Average Mean Accuracy")
if "with" in modes:
ax.scatter(t,
f,
means_w,
marker="o",
s=2,
color="blue",
label="With Replacement")
# Add error bars
for i in range(t.size):
x = t[i]
y = f[i]
low_w = acc_w_lower[i]
high_w = acc_w_upper[i]
ax.plot([x, x], [y, y], [low_w, high_w], marker="_", color="blue")
if "without" in modes:
ax.scatter(t,
f,
means_wo,
marker="o",
s=2,
color="green",
label="Without Replacement")
# Add error bars
for i in range(t.size):
x = t[i]
y = f[i]
low_wo = acc_wo_lower[i]
high_wo = acc_wo_upper[i]
ax.plot([x, x], [y, y], [low_wo, high_wo],
marker="_",
color="green")
ax.legend(loc="best", markerscale=2)
ax.xaxis.set_major_locator(MaxNLocator(integer=True))
if filename is not None:
# Make sure directory exists
file_dir = filename[:filename.rfind("/")]
if "/" in filename and not os.path.isdir(file_dir):
os.makedirs(file_dir)
fig.savefig(filename, bbox_inches="tight")
filebase, extention = filename.rsplit(".", 1)
pl.dump(fig, open(filebase + ".fig", "wb"))
else:
plt.show()
best_params = []
# Determine optimal hyper-parameters for best accuracy
if "with" in modes:
best_acc_w = acc_w_lower.max()
best_match_w = acc_w_upper[acc_w_upper - best_acc_w > 0].min()
t_w, f_w = t[acc_w_upper == best_match_w].min(), f[acc_w_upper ==
best_match_w].min()
best_params.append(t_w)
best_params.append(f_w)
if "without" in modes:
best_acc_wo = acc_wo_lower.max()
best_match_wo = acc_wo_upper[acc_wo_upper - best_acc_wo > 0].min()
t_wo, f_wo = t[acc_wo_upper == best_match_wo].min(), f[
acc_wo_upper == best_match_wo].min()
best_params.append(t_wo)
best_params.append(f_wo)
return best_params
f.write(b'NONE000\0') #2
f.write(b'NONE000\0') #3
f.write(b'NONE000\0') #4
f.write(b'NONE000\0') #5
f.write(b'NONE000\0') #6
f.write(b'NONE000\0') #7
f.write(b'NONE000\0') #8
f.write(b'NONE000\0') #9
f.write(b'NONE000\0') #10
vector.astype('float64').tofile(f)
corrected_times = np.round(t20-t20[0], 15)
for i in progressbar.progressbar(range(t20.shape[0])):
t = t20[i]
rdr.SetTimeValue(t)
rdr.Update()
mesh = rdr.GetOutput().GetBlock(0)
veloc = mesh.GetPointData().GetArray(varname)
v = np.zeros((mesh.GetNumberOfPoints(),ndimensions), dtype = np.float64)
for j in range(veloc.GetNumberOfTuples()):
vv = veloc.GetTuple3(j)
v[j,:] = vv[:ndimensions]
filename = '{:s}-XFIEL.{:08d}.{:08d}.mpio.bin'.format(problem_name, field_id, i+1)
write_vector_mpio( os.path.join( outpath, filename), v, corrected_times[i] )
def buildSearchIndices(project_number, section, mode='index', chunk=None, max_age_days=None):
""" Builds search index for a project.
section must be one of:
'index' - create the index for the project if it does not exist
'mappings' - create mappings for the project if they do not exist
'media' - create documents for media
'states' - create documents for states
'localizations' - create documents for localizations
'treeleaves' - create documents for treeleaves
"""
project_name = Project.objects.get(pk=project_number).name
logger.info(f"Building search indices for project {project_number}: {project_name}")
if section == 'index':
# Create indices
logger.info("Building index...")
TatorSearch().create_index(project_number)
logger.info("Build index complete!")
return
if section == 'mappings':
# Create mappings
logger.info("Building mappings for media types...")
for type_ in progressbar(list(MediaType.objects.filter(project=project_number))):
TatorSearch().create_mapping(type_)
logger.info("Building mappings for localization types...")
for type_ in progressbar(list(LocalizationType.objects.filter(project=project_number))):
TatorSearch().create_mapping(type_)
logger.info("Building mappings for state types...")
for type_ in progressbar(list(StateType.objects.filter(project=project_number))):
TatorSearch().create_mapping(type_)
logger.info("Building mappings for leaf types...")
for type_ in progressbar(list(LeafType.objects.filter(project=project_number))):
TatorSearch().create_mapping(type_)
logger.info("Build mappings complete!")
return
class DeferredCall:
def __init__(self, qs):
self._qs = qs
def __call__(self):
for entity in self._qs.iterator():
for doc in TatorSearch().build_document(entity, mode):
yield doc
# Get queryset based on selected section.
logger.info(f"Building documents for {section}...")
qs = CLASS_MAPPING[section].objects.filter(project=project_number, meta__isnull=False)
# Apply max age filter.
if max_age_days:
min_modified = datetime.datetime.now() - datetime.timedelta(days=max_age_days)
qs = qs.filter(modified_datetime__gte=min_modified)
# Apply limit/offset if chunk parameter given.
if chunk is not None:
offset = INDEX_CHUNK_SIZE * chunk
qs = qs.order_by('id')[offset:offset+INDEX_CHUNK_SIZE]
batch_size = 500
count = 0
bar = ProgressBar(redirect_stderr=True, redirect_stdout=True)
dc = DeferredCall(qs)
total = qs.count()
bar.start(max_value=total)
for ok, result in streaming_bulk(TatorSearch().es, dc(),chunk_size=batch_size, raise_on_error=False):
action, result = result.popitem()
if not ok:
print(f"Failed to {action} document! {result}")
bar.update(min(count, total))
count += 1
if count > total:
print(f"Count exceeds list size by {total - count}")
bar.finish()
def _get_detections(generator,
model,
score_threshold=0.05,
max_detections=100,
save_path=None):
"""
Get the detections from the model using the generator.
The result is a list of lists such that the size is:
all_detections[num_images][num_classes] = detections[num_class_detections, 5]
Args:
generator: The generator used to run images through the model.
model: The model to run on the images.
score_threshold: The score confidence threshold to use.
max_detections: The maximum number of detections to use per image.
save_path: The path to save the images with visualized detections to.
Returns:
A list of lists containing the detections for each image in the generator.
"""
all_detections = [[
None for i in range(generator.num_classes()) if generator.has_label(i)
] for j in range(generator.size())]
for i in progressbar.progressbar(range(generator.size()),
prefix='Running network: '):
raw_image = generator.load_image(i)
image = generator.preprocess_image(raw_image.copy())
image, scale = generator.resize_image(image)
# run network
boxes, scores, labels = model.predict_on_batch(
np.expand_dims(image, axis=0))[:3]
# correct boxes for image scale
boxes /= scale
# select indices which have a score above the threshold
indices = np.where(scores[0, :] > score_threshold)[0]
# select those scores
scores = scores[0][indices]
# find the order with which to sort the scores
scores_sort = np.argsort(-scores)[:max_detections]
# select detections
# (n, 4)
image_boxes = boxes[0, indices[scores_sort], :]
# (n, )
image_scores = scores[scores_sort]
# (n, )
image_labels = labels[0, indices[scores_sort]]
# (n, 6)
image_detections = np.concatenate([
image_boxes,
np.expand_dims(image_scores, axis=1),
np.expand_dims(image_labels, axis=1)
],
axis=1)
if save_path is not None:
draw_annotations(raw_image,
generator.load_annotations(i),
label_to_name=generator.label_to_name)
draw_detections(raw_image,
image_boxes,
image_scores,
image_labels,
label_to_name=generator.label_to_name,
score_threshold=score_threshold)
cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image)
# copy detections to all_detections
for label in range(generator.num_classes()):
if not generator.has_label(label):
continue
all_detections[i][label] = image_detections[
image_detections[:, -1] == label, :-1]
return all_detections
def _get_detections(generator,
model,
score_threshold=0.05,
max_detections=100,
visualize=False):
"""
Get the detections from the model using the generator.
The result is a list of lists such that the size is:
all_detections[num_images][num_classes] = detections[num_class_detections, 5]
Args:
generator: The generator used to run images through the model.
model: The model to run on the images.
score_threshold: The score confidence threshold to use.
max_detections: The maximum number of detections to use per image.
save_path: The path to save the images with visualized detections to.
Returns:
A list of lists containing the detections for each image in the generator.
"""
all_detections = [[
None for i in range(generator.num_classes()) if generator.has_label(i)
] for j in range(generator.size())]
for i in progressbar.progressbar(range(generator.size()),
prefix='Running network: '):
image = generator.load_image(i)
src_image = image.copy()
h, w = image.shape[:2]
anchors = generator.anchors
image, scale, offset_h, offset_w = generator.preprocess_image(image)
# run network
boxes, scores, labels = model.predict_on_batch(
[np.expand_dims(image, axis=0),
np.expand_dims(anchors, axis=0)])
boxes[..., [0, 2]] = boxes[..., [0, 2]] - offset_w
boxes[..., [1, 3]] = boxes[..., [1, 3]] - offset_h
boxes /= scale
boxes[:, :, 0] = np.clip(boxes[:, :, 0], 0, w - 1)
boxes[:, :, 1] = np.clip(boxes[:, :, 1], 0, h - 1)
boxes[:, :, 2] = np.clip(boxes[:, :, 2], 0, w - 1)
boxes[:, :, 3] = np.clip(boxes[:, :, 3], 0, h - 1)
# select indices which have a score above the threshold
indices = np.where(scores[0, :] > score_threshold)[0]
# select those scores
scores = scores[0][indices]
# find the order with which to sort the scores
scores_sort = np.argsort(-scores)[:max_detections]
# select detections
# (n, 4)
image_boxes = boxes[0, indices[scores_sort], :]
# (n, )
image_scores = scores[scores_sort]
# (n, )
image_labels = labels[0, indices[scores_sort]]
# (n, 6)
detections = np.concatenate([
image_boxes,
np.expand_dims(image_scores, axis=1),
np.expand_dims(image_labels, axis=1)
],
axis=1)
if visualize:
draw_annotations(src_image,
generator.load_annotations(i),
label_to_name=generator.label_to_name)
draw_detections(src_image,
detections[:5, :4],
detections[:5, 4],
detections[:5, 5].astype(np.int32),
label_to_name=generator.label_to_name,
score_threshold=score_threshold)
# cv2.imwrite(os.path.join(save_path, '{}.png'.format(i)), raw_image)
cv2.namedWindow('{}'.format(i), cv2.WINDOW_NORMAL)
cv2.imshow('{}'.format(i), src_image)
cv2.waitKey(0)
# copy detections to all_detections
for class_id in range(generator.num_classes()):
all_detections[i][class_id] = detections[detections[:, -1] ==
class_id, :-1]
return all_detections
def import_data(self):
""" Imports data from a file and overrides its properties, if wanted.
This method also deactivates auto commit (if it is activated)
temporary.
"""
cursor = connection.cursor()
# Defer all constraint checks
cursor.execute('SET CONSTRAINTS ALL DEFERRED')
# Drop summary table trigger to make insertion faster
cursor.execute("""
DROP TRIGGER on_edit_treenode_update_summary_and_edges ON treenode;
DROP TRIGGER on_insert_treenode_update_summary_and_edges ON treenode;
DROP TRIGGER on_delete_treenode_update_summary_and_edges ON treenode;
""")
# Get all existing users so that we can map them basedon their username.
mapped_user_ids = set()
mapped_user_target_ids = set()
# Map data types to lists of object of the respective type
import_data = defaultdict(list)
n_objects = 0
# Read the file and sort by type
logger.info("Loading data from {}".format(self.source))
with open(self.source, "r") as data:
loaded_data = serializers.deserialize(self.format, data)
for deserialized_object in progressbar.progressbar(loaded_data,
max_value=progressbar.UnknownLength, redirect_stdout=True):
obj = deserialized_object.object
import_data[type(obj)].append(deserialized_object)
n_objects += 1
if n_objects == 0:
raise CommandError("Nothing to import, no importable data found")
created_users = dict()
if import_data.get(User):
import_users = dict((u.object.id, u) for u in import_data.get(User))
logger.info("Found {} referenceable users in import data".format(len(import_users)))
else:
import_users = dict()
logger.info("Found no referenceable users in import data")
# Get CATMAID model classes, which are the ones we want to allow
# optional modification of user, project and ID fields.
app = apps.get_app_config('catmaid')
user_updatable_classes = set(app.get_models())
logger.info("Adjusting {} import objects to target database".format(n_objects))
# Needed for name uniquness of classes, class_instances and relations
existing_classes = dict(Class.objects.filter(project_id=self.target.id) \
.values_list('class_name', 'id'))
existing_relations = dict(Relation.objects.filter(project_id=self.target.id) \
.values_list('relation_name', 'id'))
existing_class_instances = dict(ClassInstance.objects.filter(project_id=self.target.id) \
.values_list('name', 'id'))
existing_concept_ids = set(Concept.objects.all().values_list('id', flat=True))
# Find classes for neurons and skeletons in import data
if Class in import_data:
allowed_duplicate_classes = tuple(c.object.id
for c in import_data.get(Class)
if c.object.class_name in ('neuron', 'skeleton'))
else:
allowed_duplicate_classes = tuple()
n_reused = 0
n_moved = 0
append_only = not self.preserve_ids
need_separate_import = []
objects_to_save = defaultdict(list)
import_objects_by_type_and_id = defaultdict(dict)
for object_type, import_objects in six.iteritems(import_data):
# Allow user reference updates in CATMAID objects
if object_type not in user_updatable_classes:
need_separate_import.append(object_type)
continue
# Stores in append-only mode import IDs and links them to the
# respective objects. This is needed, to update foreign keys to this
# ID when it is replaced with a new ID.
objects_by_id = import_objects_by_type_and_id[object_type]
is_class = object_type == Class
is_relation = object_type == Relation
is_class_instance = object_type == ClassInstance
# CATMAID model objects are inspected for user fields
for deserialized_object in import_objects:
obj = deserialized_object.object
# Semantic data like classes and class instances are expected to
# be unique with respect to their names. Existing objects with
# the same ID will get a new ID even if --preserve-ids is set.
existing_obj_id = None
concept_id_exists = obj.id in existing_concept_ids
if is_class:
existing_obj_id = existing_classes.get(obj.class_name)
if is_relation:
existing_obj_id = existing_relations.get(obj.relation_name)
if is_class_instance:
existing_obj_id = existing_class_instances.get(obj.name)
# Neurons (class instances of class "neuron" and "skeleton")
# are a special case. There can be multiple neurons with
# the same name, something that is not allowed in other
# cases. In this particular case, however, class instance
# reuse is not wanted.
if existing_obj_id and obj.class_column_id in allowed_duplicate_classes:
existing_obj_id = None
concept_id_exists = False
if existing_obj_id is not None:
# Add mapping so that existing references to it can be
# updated. The object itself is not marked for saving,
# because it exists already.
current_id = obj.id
objects_by_id[current_id] = obj
obj.id = existing_obj_id
n_reused += 1
continue
# If there is already an known object with the ID of the object
# we are importing at the moment and the current model is a
# class, relation or class_instance, then the imported object
# will get a new ID, even with --preservie-ids set. We reuse
# these types.
if concept_id_exists:
current_id = obj.id
objects_by_id[current_id] = obj
obj.id = None
n_moved += 1
# Replace existing data if requested
self.override_fields(obj)
# Map users based on username, optionally create unmapped users.
self.map_or_create_users(obj, import_users, mapped_user_ids,
mapped_user_target_ids, created_users)
# Remove pre-defined ID and keep track of updated IDs in
# append-only mode (default).
if append_only:
current_id = obj.id
objects_by_id[current_id] = obj
# By setting id to None, Django will create a new object and
# set the new ID.
obj.id = None
# Remember for saving
objects_to_save[object_type].append(deserialized_object)
if len(created_users) > 0:
logger.info("Created {} new users: {}".format(len(created_users),
", ".join(sorted(created_users.keys()))))
else:
logger.info("No unmapped users imported")
# Finally save all objects. Make sure they are saved in order:
logger.info("Storing {} database objects including {} moved objects, reusing additional {} existing objects" \
.format(n_objects - n_reused, n_moved, n_reused))
# In append-only mode, the foreign keys to objects with changed IDs have
# to be updated. In preserve-ids mode only IDs to classes and relations
# will be updated. Saving model objects after an update of referenced
# keys is only needed in append-only mode.
self.reset_ids(user_updatable_classes, objects_to_save,
import_objects_by_type_and_id, existing_classes)
other_tasks = set(objects_to_save.keys()) - set(ordered_save_tasks)
for object_type in ordered_save_tasks + list(other_tasks):
objects = objects_to_save.get(object_type)
if objects:
logger.info("- Importing objects of type " + object_type.__name__)
for deserialized_object in progressbar.progressbar(objects,
max_value=len(objects), redirect_stdout=True):
deserialized_object.save()
logger.info("- Importing all other objects")
for other_model in progressbar.progressbar(need_separate_import,
max_value=len(need_separate_import), redirect_stdout=True):
other_objects = import_data[other_model]
if other_model == User:
# If user model objects are imported and users were mapped, ask
# user if alrady mapped users should be skipped during import.
# We don't need to take of newly created users, because they are
# only created if no model is found. Therefore all other model
# objects can be imported.
if mapped_user_target_ids:
mapped_usernames = set(self.user_id_map.get(u) for u in mapped_user_target_ids)
import_usernames = set(import_users.keys())
not_imported_usernames = import_usernames - mapped_usernames
already_imported_usernames = import_usernames - not_imported_usernames
if already_imported_usernames:
print("The following usernames are mapped to " +
"existing users, but the import data " +
"also contains objects for these users: " +
", ".join(already_imported_usernames))
ignore_users = ask_yes_no("Skip those users in input "
"data and don't import them? [y/n]")
if ignore_users:
logger.info("Won't import mapped users: " +
", ".join(already_imported_usernames))
other_objects = [u for u in other_objects \
if u.object.username not in already_imported_usernames]
else:
logger.info("Will import all listed users in import data")
for deserialized_object in other_objects:
if deserialized_object.object.username in created_users.keys():
deserialized_object.save()
# Reset counters to current maximum IDs
cursor.execute('''
SELECT setval('concept_id_seq', coalesce(max("id"), 1), max("id") IS NOT null)
FROM concept;
SELECT setval('location_id_seq', coalesce(max("id"), 1), max("id") IS NOT null)
FROM location;
SELECT setval('auth_user_id_seq', coalesce(max("id"), 1), max("id") IS NOT null)
FROM auth_user;
''')
cursor.execute("""
CREATE TRIGGER on_insert_treenode_update_summary_and_edges
AFTER INSERT ON treenode
REFERENCING NEW TABLE as inserted_treenode
FOR EACH STATEMENT EXECUTE PROCEDURE on_insert_treenode_update_summary_and_edges();
CREATE TRIGGER on_edit_treenode_update_summary_and_edges
AFTER UPDATE ON treenode
REFERENCING NEW TABLE as new_treenode OLD TABLE as old_treenode
FOR EACH STATEMENT EXECUTE PROCEDURE on_edit_treenode_update_summary_and_edges();
CREATE TRIGGER on_delete_treenode_update_summary_and_edges
AFTER DELETE ON treenode
REFERENCING OLD TABLE as deleted_treenode
FOR EACH STATEMENT EXECUTE PROCEDURE on_delete_treenode_update_summary_and_edges();
""")
logger.info("Updating edge tables")
rebuild_edge_tables(log=lambda msg: logger.info(msg))
logger.info("Updated skeleton summary tables")
cursor.execute("""
DELETE FROM catmaid_skeleton_summary;
SELECT refresh_skeleton_summary_table();
""")
def evaluate_coco(generator, model, threshold=0.05):
""" Use the pycocotools to evaluate a COCO model on a dataset.
Args
generator : The generator for generating the evaluation data.
model : The model to evaluate.
threshold : The score threshold to use.
"""
# start collecting results
results = []
image_ids = []
for index in progressbar.progressbar(range(generator.size()), prefix='COCO evaluation: '):
image = generator.load_image(index)
image = generator.preprocess_image(image)
image, scale = generator.resize_image(image)
if keras.backend.image_data_format() == 'channels_first':
image = image.transpose((2, 0, 1))
# run network
boxes, scores, labels = model.predict_on_batch(np.expand_dims(image, axis=0))
# correct boxes for image scale
boxes /= scale
# change to (x, y, w, h) (MS COCO standard)
boxes[:, :, 2] -= boxes[:, :, 0]
boxes[:, :, 3] -= boxes[:, :, 1]
# compute predicted labels and scores
for box, score, label in zip(boxes[0], scores[0], labels[0]):
# scores are sorted, so we can break
if score < threshold:
break
# append detection for each positively labeled class
image_result = {
'image_id' : generator.image_ids[index],
'category_id' : generator.label_to_coco_label(label),
'score' : float(score),
'bbox' : box.tolist(),
}
# append detection to results
results.append(image_result)
# append image to list of processed images
image_ids.append(generator.image_ids[index])
if not len(results):
return
# write output
json.dump(results, open('{}_bbox_results.json'.format(generator.set_name), 'w'), indent=4)
json.dump(image_ids, open('{}_processed_image_ids.json'.format(generator.set_name), 'w'), indent=4)
# load results in COCO evaluation tool
coco_true = generator.coco
coco_pred = coco_true.loadRes('{}_bbox_results.json'.format(generator.set_name))
# run COCO evaluation
coco_eval = COCOeval(coco_true, coco_pred, 'bbox')
coco_eval.params.imgIds = image_ids
coco_eval.evaluate()
coco_eval.accumulate()
coco_eval.summarize()
return coco_eval.stats
