def getMembers():
membersFromFile = readFromFile()
members = []
for member in membersFromFile:
members.append({'id': member['id'], 'score': member['score']})
return members
associated_label_global = Dict.empty(key_type=types.int64,
value_type=types.int64)
undetermined_global = set()
neighbor_label_dict = dict()
#timing
start_time_pickleload = time.time()
# load sets by iterating over all folders (expected to be small time consumption)
for bz in range(param.z_start, param.z_start + param.n_blocks_z):
for by in range(param.y_start, param.y_start + param.n_blocks_y):
for bx in range(param.x_start, param.x_start + param.n_blocks_x):
output_folder = blockFolderPath(param.folder_path, bz, by, bx)
neighbor_label_set_border_global = readFromFile(
"neighbor_label_set_border_global", output_folder, "")
associated_label_local = readFromFile("associated_label_local",
output_folder, "")
undetermined_local = readFromFile("undetermined_local",
output_folder, "")
neighbor_label_dict_reduced_local = readFromFile(
"neighbor_label_dict_reduced", output_folder, "")
if -2 in undetermined_local:
print(bz, by, bx)
print("undetermined")
if -2 in neighbor_label_dict_reduced_local.keys():
print(bz, by, bx)
print("neighbor label dict")
def execStep3(inplist):
bz = inplist[0]
by = inplist[1]
print("STEP 3: running for bz,by: " + str(bz) + ", " + str(by))
for bx in range(param.x_start, param.x_start + param.n_blocks_x):
# STEP 3
start_time_total = time.time()
block_number = (bz) * (param.y_start + param.n_blocks_y) * (
param.x_start + param.n_blocks_x) + by * (param.x_start +
param.n_blocks_x) + bx
label_start = -1 * block_number * param.max_labels_block - 1
label_end = label_start - param.max_labels_block
start_time_readpickle = time.time()
associated_label_global = readFromFile("associated_label_global",
param.folder_path, "")
time_readpickle = time.time() - start_time_readpickle
start_time_cutdict = time.time()
associated_label_block = {
key: value
for key, value in associated_label_global.items()
if (key > label_end and key <= label_start)
}
associated_label = Dict.empty(key_type=types.int64,
value_type=types.int64)
associated_label.update(associated_label_block)
time_cutdict = time.time() - start_time_cutdict
start_time_fillWholes = time.time()
output_folder = param.folder_path + "/z" + str(bz).zfill(
4) + "y" + str(by).zfill(4) + "x" + str(bx).zfill(4) + "/"
fillWholes(output_path=output_folder,
associated_label=associated_label,
bz=bz,
by=by,
bx=bx)
time_fillWholes = time.time() - start_time_fillWholes
time_total = time.time() - start_time_total
g = open(
param.total_time_filepath + "-" + str(bz).zfill(4) + "z-" +
str(by).zfill(4) + "y-" + str(bx).zfill(4) + "x.txt", "a+")
g.write(format(time_total, '.4f') + "\n")
g.close()
g = open(param.step03_info_filepath, "a+")
g.write("bz/by/bx," + str(bz).zfill(4) + "," + str(by).zfill(4) + "," +
str(bx).zfill(4) + ","
"total_time," + format(time_total, '.4f') + "," +
"pickleload_time," + format(time_readpickle, '.4f') + "," +
"cutdict_time," + format(time_cutdict, '.4f') + "," +
"fillWholes_time," + format(time_fillWholes, '.4f') + "\n")
g.close()
return 1
def execStep2B(inpList):
# input list is dummy argument
print("STEP 2B: running")
start_time_total = time.time()
# final step
neighbor_label_set_border_global_combined = set()
associated_label_global = Dict.empty(key_type=types.int64,
value_type=types.int64)
undetermined_global = set()
neighbor_label_dict = dict()
#timing
start_time_pickleload = time.time()
# load sets by iterating over all folders (expected to be small time consumption)
for bz in range(param.z_start, param.z_start + param.n_blocks_z):
for by in range(param.y_start, param.y_start + param.n_blocks_y):
for bx in range(param.x_start, param.x_start + param.n_blocks_x):
output_folder = blockFolderPath(param.folder_path, bz, by, bx)
neighbor_label_set_border_global = readFromFile(
"neighbor_label_set_border_global", output_folder, "")
associated_label_local = readFromFile("associated_label_local",
output_folder, "")
undetermined_local = readFromFile("undetermined_local",
output_folder, "")
neighbor_label_dict_reduced_local = readFromFile(
"neighbor_label_dict_reduced", output_folder, "")
neighbor_label_set_border_global_combined = neighbor_label_set_border_global_combined.union(
neighbor_label_set_border_global)
associated_label_global.update(associated_label_local)
undetermined_global = undetermined_global.union(
undetermined_local)
neighbor_label_dict.update(neighbor_label_dict_reduced_local)
del neighbor_label_set_border_global, associated_label_local, undetermined_local
len_label_set_border = len(neighbor_label_set_border_global_combined)
time_pickleload = time.time() - start_time_pickleload
start_time_findAssocLabelGlobal = time.time()
#compute associated label global
neighbor_label_dict = writeNeighborLabelDict(
neighbor_label_dict=neighbor_label_dict,
neighbor_label_set=neighbor_label_set_border_global_combined.copy())
associated_label_global, undetermined_global, isHole, isNotHole = findAssociatedLabels(
neighbor_label_dict, undetermined_global, associated_label_global)
associated_label_global = setUndeterminedtoNonHole(
undetermined_global.copy(), associated_label_global)
if param.compute_statistics:
hole_components = isHole
n_Holes = len(isHole)
n_NotHoles = len(isNotHole) + len(undetermined_global)
time_findAssocLabelGlobal = time.time() - start_time_findAssocLabelGlobal
start_time_writepickle = time.time()
dumpNumbaDictToFile(associated_label_global, "associated_label_global",
param.folder_path, "")
time_writepickle = time.time() - start_time_writepickle
time_total = time.time() - start_time_total
len_associated_label_global = len(associated_label_global)
g = open(param.step02B_info_filepath, "a+")
g.write("total_time," + format(time_total, '.4f') + "," +
"pickleload_time," + format(time_pickleload, '.4f') + "," +
"findAssocLabelGlobal_time," +
format(time_findAssocLabelGlobal, '.4f') + "," +
"writepickle_time," + format(time_writepickle, '.4f') + "," +
"len_label_set_border," + str(len_label_set_border).zfill(16) +
"," + "len_associated_label_global," +
str(len_associated_label_global).zfill(16) + "," + "n_Holes," +
str(n_Holes).zfill(16) + "," + "n_NotHoles," +
str(n_NotHoles).zfill(16) + "\n")
g.close()
g = open(param.total_time_filepath + "-step_2B.txt", "a+")
g.write(format(time_total, '.4f') + "\n")
g.close()
if param.compute_statistics:
g = open(param.hole_components_filepath + "-global.txt", "a+")
for entry in hole_components:
g.write(str(int(entry)).zfill(25) + "\n")
g.close()
return 1
bz = int(sys.argv[1])
by = int(sys.argv[2])
for bx in range(param.x_start, param.x_start + param.n_blocks_x):
# STEP 3
start_time_total = time.time()
block_number = (bz) * (param.y_start + param.n_blocks_y) * (
param.x_start + param.n_blocks_x) + by * (param.x_start +
param.n_blocks_x) + bx
label_start = -1 * block_number * param.max_labels_block - 1
label_end = label_start - param.max_labels_block
start_time_readpickle = time.time()
associated_label_global = readFromFile("associated_label_global",
param.folder_path, "")
time_readpickle = time.time() - start_time_readpickle
start_time_cutdict = time.time()
associated_label_block = {
key: value
for key, value in associated_label_global.items()
if (key > label_end and key <= label_start)
}
associated_label = Dict.empty(key_type=types.int64, value_type=types.int64)
associated_label.update(associated_label_block)
time_cutdict = time.time() - start_time_cutdict
start_time_fillWholes = time.time()
output_folder = param.folder_path + "/z" + str(bz).zfill(4) + "y" + str(
by).zfill(4) + "x" + str(bx).zfill(4) + "/"