def get_output_parameters(observable_df: pd.DataFrame,
sbml_model: libsbml.Model) -> List[str]:
"""Get output parameters
Returns IDs of parameters used in observable and noise formulas that are
not defined in the SBML model.
Arguments:
observable_df: PEtab observable table
sbml_model: SBML model
Returns:
List of output parameter IDs
"""
formulas = list(observable_df[OBSERVABLE_FORMULA])
if NOISE_FORMULA in observable_df:
formulas.extend(observable_df[NOISE_FORMULA])
output_parameters = OrderedDict()
for formula in formulas:
free_syms = sorted(sp.sympify(formula).free_symbols,
key=lambda symbol: symbol.name)
for free_sym in free_syms:
sym = str(free_sym)
if sbml_model.getElementBySId(sym) is None and sym != 'time':
output_parameters[sym] = None
return list(output_parameters.keys())
async def do_menu(self, ctx, c):
emoji_to_embed = OrderedDict()
for idx, image_info in enumerate(c['images']):
emoji = char_to_emoji(str(idx))
emoji_to_embed[emoji] = make_card_embed(c, idx)
starting_menu_emoji = list(emoji_to_embed.keys())[0]
return await self._do_menu(ctx, starting_menu_emoji, emoji_to_embed)
def parse(self, input_features):
if 'bounding_boxes_{}'.format(self.disease_annotation) not in input_features:
Add_Bounding_Box = Add_Bounding_Box_Indexes([self.disease_annotation], add_to_dictionary=False)
input_features = Add_Bounding_Box.parse(input_features)
if 'bounding_boxes_{}'.format(self.disease_annotation) in input_features:
bounding_boxes = input_features['bounding_boxes_{}'.format(self.disease_annotation)]
voxel_volumes = input_features['voxel_volumes_{}'.format(self.disease_annotation)]
del input_features['voxel_volumes_{}'.format(self.disease_annotation)]
del input_features['bounding_boxes_{}'.format(self.disease_annotation)]
image_base = input_features['image']
annotation_base = input_features['annotation']
out_features = OrderedDict()
for cube_index, [box, voxels] in enumerate(zip(bounding_boxes, voxel_volumes)):
if voxels < self.min_voxel_volume or voxels > self.max_voxels:
continue
z_start, z_stop, r_start, r_stop, c_start, c_stop = add_bounding_box_to_dict(box, return_indexes=True)
box_size = [z_stop - z_start, r_stop - r_start, c_stop - c_start]
remainders = np.asarray([self.cube_size[i] - box_size[i] % self.cube_size[i]
if box_size[i] % self.cube_size[i] != 0 else 0 for i in range(3)])
z_start, z_stop, r_start, r_stop, c_start, c_stop = expand_box_indexes(z_start, z_stop, r_start, r_stop,
c_start, c_stop,
annotation_shape=
annotation_base.shape,
bounding_box_expansion=
remainders//2+1)
image = image_base[z_start:z_stop, r_start:r_stop, c_start:c_stop]
annotation = annotation_base[z_start:z_stop, r_start:r_stop, c_start:c_stop]
stack_image, stack_annotation = [image[None,...]], [annotation[None,...]]
for axis in range(3):
output_images = []
output_annotations = []
for i in stack_image:
split = i.shape[axis+1] // self.cube_size[axis]
if split > 1:
output_images += np.array_split(i, split, axis=axis+1)
else:
output_images += [i]
for i in stack_annotation:
split = i.shape[axis+1] // self.cube_size[axis]
if split > 1:
output_annotations += np.array_split(i, split, axis=axis+1)
else:
output_annotations += [i]
stack_image = output_images
stack_annotation = output_annotations
for box_index, [image_cube, annotation_cube] in enumerate(zip(stack_image, stack_annotation)):
temp_feature = OrderedDict()
image_cube, annotation_cube = image_cube[0], annotation_cube[0]
temp_feature['image'] = image_cube[:self.cube_size[0]]
temp_feature['annotation'] = annotation_cube[:self.cube_size[0]]
for key in input_features: # Bring along anything else we care about
if key not in temp_feature.keys():
temp_feature[key] = input_features[key]
out_features['Disease_Box_{}_{}'.format(cube_index, box_index)] = temp_feature
input_features = out_features
return input_features
return input_features
def f_rec(n):
dict = OrderedDict({0: 0, 1: 1, 2: 2})
if n in dict.keys():
return dict[n]
else:
for i in range(3, n + 1):
current_value = dict[i - 1] + dict[i - 3]
dict.popitem(last=False)
dict[i] = current_value
return dict[n]
class CppCompleter(CompleterBase):
# temp file for racer input
live_template_file = os.path.join(util.get_resources_dir(),
"live_templates_cpp.xml")
# TODO: Support clang completer
def setCompletionPrefix(self, text: str):
temp_file_name = util.get_temp_file("cpp")
with codecs.open(temp_file_name, "w", "utf-8") as f:
f.write(self.parent.toPlainText())
src_line_num = str(self.parent.textCursor().blockNumber() + 1)
src_char_num = str(self.parent.textCursor().columnNumber())
try:
out = subprocess.check_output(
# read all header is too slow
# "clang -fsyntax-only -Xclang -code-completion-at=%s:%s:%s %s"
"clang -cc1 -fsyntax-only -code-completion-at=%s:%s:%s %s" %
(temp_file_name, src_line_num, src_char_num, temp_file_name),
shell=True,
).decode()
except subprocess.CalledProcessError as e:
out = e.output.decode()
self.candidates_dict = OrderedDict()
for line in out.split("\n"):
if line.startswith("COMPLETION:"):
cand = line.split(" ")[1]
if text not in cand:
continue
if cand not in self.ng_words:
self.candidates_dict[cand] = -1
for live_template in self.live_templates:
if live_template.name.startswith(text):
self.candidates_dict[
live_template.template] = live_template.rpos
if len(self.candidates_dict
) >= 10 or text in self.candidates_dict.keys():
self.candidates_dict = {}
self.setModel(QtCore.QStringListModel(self.candidates_dict.keys()))
super().setCompletionPrefix(text)
def commentCountDistribution(streamComments):
commentCount = OrderedDict()
for user in streamComments:
count = len(streamComments[user])
if count not in commentCount:
commentCount[count] = []
commentCount[count].append(user)
commentDistribution = {}
for count in sorted(commentCount.keys()):
commentDistribution[count] = len(commentCount[count])
return commentDistribution
def get_features(image_path, annotation_path, image_processors=None, record_writer=None):
features = OrderedDict()
features['image_path'] = image_path
features['annotation_path'] = annotation_path
if image_processors is not None:
for image_processor in image_processors:
features, _ = down_dictionary(features, OrderedDict(), 0)
for key in features.keys():
features[key] = image_processor.parse(features[key])
features, _ = down_dictionary(features, OrderedDict(), 0)
record_writer.parse(features)
def parse(self, input_features):
out_features = OrderedDict()
start_chop = 0
image_base = input_features['image']
annotation_base = input_features['annotation']
image_path = input_features['image_path']
spacing = input_features['spacing']
z_images_base, rows, cols = image_base.shape
if self.max_rows != np.inf:
rows = min([rows,self.max_rows])
if self.max_cols != np.inf:
cols = min([cols,self.max_cols])
image_base, annotation_base = image_base[:, :rows, :cols], annotation_base[:, :rows, :cols]
step = min([self.max_z, z_images_base])
for index in range(z_images_base // step + 1):
image_features = OrderedDict()
if start_chop >= z_images_base:
continue
image = image_base[start_chop:start_chop + step, ...]
annotation = annotation_base[start_chop:start_chop + step, ...]
if image.shape[0] < max([step, self.min_z]):
if self.mirror_small_bits:
while image.shape[0] < max([step, self.min_z]):
mirror_image = np.flip(image, axis=0)
mirror_annotation = np.flip(annotation, axis=0)
image = np.concatenate([image, mirror_image], axis=0)
annotation = np.concatenate([annotation, mirror_annotation], axis=0)
image = image[:max([step, self.min_z])]
annotation = annotation[:max([step, self.min_z])]
start, stop = get_start_stop(annotation, extension=0)
image_features['image_path'] = image_path
image_features['image'] = image
image_features['annotation'] = annotation
image_features['start'] = start
image_features['stop'] = stop
image_features['z_images'] = image.shape[0]
image_features['rows'] = image.shape[1]
image_features['cols'] = image.shape[2]
image_features['spacing'] = spacing
for key in input_features.keys():
if key not in image_features.keys():
image_features[key] = input_features[key] # Pass along all other keys.. be careful
out_features['Image_{}'.format(index)] = image_features
input_features = out_features
return input_features
def createFasta(input_file, append_file, order):
fastaInDict = dict()
orderDictSc = OrderedDict()
with open(input_file) as FASTAIN, open(append_file,
"a") as APP, open(order) as ORD:
fastaParse = SeqIO.parse(FASTAIN, "fasta")
fastaOutDict = defaultdict(list)
for fastaSeq in fastaParse:
s = str(fastaSeq.seq)
idFasta = fastaSeq.id
fastaInDict[idFasta] = s
keyMap = {
"Smic.scaffold9__1420062__1920061":
"Smic.scaffold9__1420062__2138115",
"Smic.scaffold236__1__500000": "Smic.scaffold236__1__795886",
"Smic.scaffold338__1__500000": "Smic.scaffold338__1__646490",
"Smic.scaffold458__1__500000": "Smic.scaffold458__1__544999"
}
for line in ORD:
line = line.rstrip("\n")
val = line.split("\t")
k = val[0]
if (k in keyMap):
k = keyMap[k]
seq = fastaInDict[k][0:500000]
else:
seq = fastaInDict[k]
fastaOutDict[val[1]].append(seq)
orderDictSc[val[1]] = len(seq)
for keys in orderDictSc.keys():
chrom = "cluster" + keys
print("Writing Chromosome " + str(chrom))
APP.write(">" + chrom + "\n")
APP.write("".join(fastaOutDict[keys]))
APP.write("\n")
class App:
def __init__(self):
self.root = Tk()
self.block_size = IntVar()
self.current_column_step = IntVar()
self.block_size.set(3)
self.isDecode = False
"""Увеличиаем размер блока"""
def update_block_size(self):
self.block_size.set(self.spin.get())
"""Выделяем блок мышкой"""
def markArea(self, event):
# Если есть таблица какая-нибудь
if (self.isDecode and event.x > 30):
self.firstCanvas.delete("hello")
# Надо выделить область по которой кликнули мышкой
column_size = 270 // self.block_size.get()
# определяем в какой мы колонке
our_column = event.x // column_size
# выделяем
self.firstCanvas.create_rectangle(40 + our_column * column_size,
25,
column_size +
column_size * our_column + 20,
30 + 15 * len(self.table[0]),
outline="#ff0000",
tags="hello")
self.marked = our_column
# Передаем данные полученные из частотного анализа
self.draw_data(self.marked)
if (our_column < self.block_size.get()):
# передаем в ползунок
self.current_column_step.set(self.steps[self.marked])
""""""
def peredecode(self, event):
if (self.isDecode and self.marked < self.block_size.get()):
self.steps[self.marked] = self.current_column_step.get()
# Теперь перерисовываем
self.create_message(self.steps, self.table)
def draw_data(self, item):
self.fq.delete("all")
column = 0
j = 0
for (i, letter) in enumerate(self.super_dicts[item].keys()):
if (10 + 15 * j > 350):
column += 1
j = 0
self.fq.create_text(20 + 30 * column,
10 + 15 * j,
text=letter + ": " +
str(self.super_dicts[item].get(letter)))
j += 1
def makeWidgets(self):
self.firstFrame = Frame(self.root)
self.firstFrame.pack(side=LEFT, anchor="nw", padx=10, pady=10)
Label(self.firstFrame, text="Таблица для исходного сообщения").pack()
self.firstAlterFrame = Frame(self.firstFrame)
self.firstAlterFrame.pack()
self.firstCanvas = Canvas(self.firstAlterFrame,
width=300,
height=500,
bg="#ffffff")
self.firstCanvas.bind("<Button-1>", self.markArea)
self.firstCanvas.pack(side=LEFT)
self.firstCanvasScroll = Scrollbar(self.firstAlterFrame,
orient='vertical',
command=self.firstCanvas.yview)
self.firstCanvasScroll.pack(side=RIGHT, fill=Y, expand=True)
self.firstCanvas.configure(yscrollcommand=self.firstCanvasScroll.set)
Label(self.firstFrame, text="Введите исходное сообщение").pack(pady=10)
self.firstText = Text(self.firstFrame, width=37, height=10)
self.firstText.pack()
self.secondFrame = Frame(self.root)
self.secondFrame.pack(side=LEFT, anchor="n", padx=10, pady=10)
Label(self.secondFrame, text="Таблица для конечного сообщения").pack()
self.secondAlterFrame = Frame(self.secondFrame)
self.secondAlterFrame.pack()
self.secondCanvas = Canvas(self.secondAlterFrame,
width=300,
height=500,
bg="#ffffff")
self.secondCanvas.pack(side=LEFT)
self.secondCanvasScroll = Scrollbar(self.secondAlterFrame,
orient='vertical',
command=self.secondCanvas.yview)
self.secondCanvasScroll.pack(side=RIGHT, fill=Y, expand=True)
self.secondCanvas.configure(yscrollcommand=self.secondCanvasScroll.set)
Label(self.secondFrame, text="Конечный результат").pack(pady=10)
self.secondText = Text(self.secondFrame, width=37, height=10)
self.secondText.pack()
self.lastFrame = Frame(self.root)
self.lastFrame.pack(side=LEFT, anchor="n", padx=10, pady=10)
Label(self.lastFrame, text="Панель управления").pack()
Label(self.lastFrame, text="Размер блока").pack(pady=10)
self.spin = Spinbox(self.lastFrame,
from_=0,
textvariable=self.block_size,
to=10,
command=self.update_block_size)
self.spin.pack()
Label(self.lastFrame, text="Частотный анализ").pack(pady=10)
self.fq = Canvas(self.lastFrame, width=150, height=400, bg="#ffffff")
self.fq.pack()
# Наш ползунок
Label(self.lastFrame, text="Ручная корректировка").pack(pady=10)
self.scale = Scale(self.lastFrame,
from_=0,
to=33,
length=150,
tickinterval=6,
orient='horizontal',
showvalue=YES,
variable=self.current_column_step,
command=self.peredecode)
self.scale.pack()
Button(self.lastFrame,
text="Расшифровать!",
width=15,
height=5,
command=self.decode).pack(pady=20)
"""Инициализирукм гуй"""
def start(self):
self.root.title("Лабораторная работа номер 2")
# self.root.geometry("900x500")
self.makeWidgets()
self.root.mainloop()
def decode(self):
self.isDecode = True
self.super_dicts = []
# Берем сообщение из текстового блока и загоняем в столбец по размеру блока
self.table = ['' for i in range(self.block_size.get())]
message = self.firstText.get("1.0", 'end-1c')
for (i, letter) in enumerate(message):
self.table[i % self.block_size.get()] += letter
self.draw(self.firstCanvas, self.table)
# Теперь мы должны сделать словари с частотными анализами для каждой колонки ох, говорю я
self.steps = [
self.analis(self.table[i]) for i in range(self.block_size.get())
]
# Так, у нас массив сдвигов, нам надо каждый столбец сдвинуть на это число
# Нам нужен не только массив сдвигов, но и словари с частоными анализами
# Теперь берем каждую строку и сдвигаем на сдвиги, но это все в отдельной функции
self.create_message(self.steps, self.table)
"""Собираем исходное сообщение"""
def create_message(self, steps, table):
self.alterTable = ['' for i in table]
for i in range(len(table)):
self.alterTable[i] = App.caesar(table[i], 33 - steps[i])
self.draw(self.secondCanvas, self.alterTable)
out = ''
for i in range(len(self.alterTable[0])):
for j in range(self.block_size.get()):
try:
out += self.alterTable[j][i]
except IndexError:
out += ''
self.secondText.delete('1.0', "end")
self.secondText.insert('1.0', out)
"""Функция для сдвига по шифру цезаря Можно конечно в одну строчку через bytes.translate, но этот вариант мне больше нравится"""
def caesar(message, step):
alpha = [
' ', 'а', 'б', 'в', 'г', 'д', 'е', 'ж', 'з', 'и', 'й', 'к', 'л',
'м', 'н', 'о', 'п', 'р', 'с', 'т', 'у', 'ф', 'х', 'ц', 'ч', 'ш',
'щ', 'ъ', 'ы', 'ь', 'э', 'ю', 'я'
]
res = []
for item in range(len(alpha)):
if item + step >= len(alpha):
res += alpha[item + step - len(alpha)]
elif item + step < 0:
res += alpha[len(alpha) - item + step]
else:
res += alpha[item + step]
wow = list(zip(alpha, res))
msq = ''
for letter in message:
for item in wow:
if letter == item[0]:
msq += item[1]
break
else:
msq += letter
return msq
"""Анализ частотностит символов"""
def analis(self, column):
alphabet = " абвгдежзийклмнопрстуфхцчшщъыьэюя"
self.dicts = {letter: 0 for letter in alphabet}
dict_step = {letter: i for (i, letter) in enumerate(alphabet)}
for i in column:
self.dicts[i] += 1
self.dicts = OrderedDict(
sorted(self.dicts.items(), key=lambda t: -t[1]))
self.super_dicts.append(self.dicts)
azaza = list(self.dicts.keys())
return dict_step.get(azaza[0])
"""Заполнянм кавас столбиками с буквами"""
def draw(self, obj, table):
# наша исходная таблица готова, нам необходимо теперь заполнить канвас
obj.delete("all")
obj.create_line(30, 0, 30, 30 + len(table[0]) * 15, fill="#000000")
obj.create_line(30, 20, 300, 20, fill="#000000")
# Делим оставшееся пространство для того что бы положить туда столбцы
column_size = 270 // self.block_size.get()
# теперь в цикле раставляем цифры и сообщение
for i in range(self.block_size.get()):
obj.create_text(column_size / 1.2 + column_size * i,
10,
text=str(i + 1))
for (j, letter) in enumerate(table[i]):
obj.create_text(column_size / 1.2 + column_size * i,
30 + j * 15,
text=letter)
# Нумерация строк
for i in range(len(table[0])):
obj.create_text(10, 30 + i * 15, text=str(i + 1))
obj.configure(scrollregion=obj.bbox("all"))
def build_compute_landscape(self, world_map):
'''
# Using the world_map.json config file, build
# a dict self.gpu_landscape like this:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# Also sets
# o self.master_hostname, the hostname
# running the one process that coordinates all others.
# o self.WORLD_SIZE, number of GPUs used across all machines
# o self.my_gpus, the number of GPUs on this machine
:param world_map:
:type world_map:
:return: information about how many GPUs are
on each node
:rtype: OrderedDict
'''
if not self.hostname in world_map.keys():
raise ConfigError(
f"World map does not contain an entry for this machine {self.hostname}"
)
# World size is the number of training script processes,
# which is equal to number of GPUs used on all participating
# machines combined:
# Number of GPUs across all machines:
self.WORLD_SIZE = 0
self.master_hostname = None
# Go through the world map, machine (a.k.a. node)
# one at a time, in alpha order of the machine
# names to ensure all copies of this script
# come to the same conclusions about ranks
# Build gpu_landscape:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# The structure is an OrderedDict(), containing
# machines alphabetically by name. This discipline
# is required so that all copies of this launch script
# (one copy per machine) arrive at the same ordering of
# GPUs:
gpu_landscape = OrderedDict({})
for machine_name in sorted(world_map.keys()):
# Get dict of info about the machine:
machine_info = world_map[machine_name]
try:
machine_gpus = machine_info['gpus']
except KeyError:
print(
"World map must include a 'gpus' entry; the value may be 0"
)
gpu_landscape[machine_name] = {}
gpu_landscape[machine_name]['num_gpus'] = machine_gpus
# List of GPU numbers to use is optional
# in world_maps:
machine_gpus_to_use = machine_info.get('devices', None)
if machine_gpus_to_use is None:
# Use all GPUs on that machine:
machine_gpus_to_use = list(range(machine_gpus))
gpu_landscape[machine_name]['gpu_device_ids'] = machine_gpus_to_use
# Accept all kinds of affirmatives as values:
# for identification of the master node entry:
is_master_node = machine_info.get('master', False) \
in [1, 'True', 'true', 'Yes', 'yes']
if is_master_node:
self.master_hostname = machine_name
if machine_name == self.hostname:
self.am_master_node = True
try:
self.MASTER_ADDR = socket.gethostbyname(machine_name)
except socket.gaierror:
# For machines that have no
# findable IP address:
self.MASTER_ADDR = '127.0.0.1'
self.WORLD_SIZE += machine_gpus
# Go through the machine enries in gpu_landscape, and
# assign rank ranges to each. Must start with
# the master node, b/c it must start with rank 0.
# For the master node, it is possible that it has
# no GPUs
master_info = gpu_landscape[self.master_hostname]
master_info['rank_range'] = list(range(master_info['num_gpus']))
master_info['start_rank'] = 0
if len(master_info['rank_range']) == 0:
# Master only has a GPU:
master_info['rank_range'] = [0]
# Start assigning more ranks after
# the GPUs of the master:
running_rank = master_info['rank_range'][-1] + 1
for machine_name in gpu_landscape.keys():
if machine_name == self.master_hostname:
# We already did the master node
continue
mach_info = gpu_landscape[machine_name]
mach_info['start_rank'] = running_rank
num_gpus = mach_info['num_gpus']
range_bound = running_rank + (num_gpus if num_gpus > 0 else 1)
mach_info['rank_range'] = list(range(running_rank, range_bound))
running_rank += (num_gpus if num_gpus > 0 else 1)
self.my_gpus = gpu_landscape[self.hostname]['num_gpus']
self.gpu_landscape = gpu_landscape
return gpu_landscape
class ControlGUIClient(object):
'''
Controller GUI class
'''
def __init__(self, port, controller):
'''
Builds the GUI, connects it to the server (thread). The GUI is just another client of
the service.
'''
global guiClient
guiClient = self
self.logger = logging.getLogger(__name__)
self.port = port
self.controller = controller
self.context = controller.context
GObject.threads_init()
self.builder = Gtk.Builder()
riaps_folder = os.getenv('RIAPSHOME', './')
try:
self.builder.add_from_file(
join(riaps_folder, "etc/riaps-ctrl.glade")) # GUI construction
except RuntimeError:
self.logger.error('Cannot find GUI configuration file')
raise
self.builder.connect_signals({
"onDeleteWindow": self.on_Quit,
"onConsoleEntryActivate": self.on_ConsoleEntry,
"onSelectApplication": self.on_SelectApplication,
"onSelectDeployment": self.on_SelectDeployment,
"onFolderEntryActivate": self.on_folderEntryActivate,
"onResetAll": self.on_resetAll,
"onHaltAll": self.on_haltAll,
"onQuit": self.on_Quit,
"onLoadApplication": self.on_loadApplication,
"onViewApplication": self.on_viewApplication,
"onLogChanged": self.on_LogChanged
})
#keyFile = controller.keyFile
#certFile = controller.certFile
self.socket = self.context.socket(zmq.PULL)
self.socket.bind(self.controller.endpoint)
GLib.io_add_watch(self.socket.fileno(), 1, GLib.IO_IN,
self.on_serverMessage)
# self.conn = rpyc.connect(self.controller.hostAddress, port) # Local connection to the service
# GLib.io_add_watch(self.conn, 1, GLib.IO_IN, self.bg_server) # Register the callback with the service
# self.conn.root.login("*gui*", self.on_serverMessage) # Log in to the service
self.mainWindow = self.builder.get_object("window1")
self.messages = self.builder.get_object("messageTextBuffer")
self.logWindow = self.builder.get_object("scrolledwindow1")
self.consoleIn = self.builder.get_object("consoleEntryBuffer")
self.appNameEntry = self.builder.get_object("appNameEntry")
self.deplNameEntry = self.builder.get_object("deplNameEntry")
self.folderEntry = self.builder.get_object("folderEntry")
#self.launchButton = self.builder.get_object("launchButton")
#self.launchButton.set_sensitive(False)
#self.stopButton = self.builder.get_object("stopButton")
#self.stopButton.set_sensitive(False)
#self.removeButton = self.builder.get_object("removeButton")
#self.removeButton.set_sensitive(False)
self.appLaunched = False
self.appDownLoaded = False
'''
Status Table Additions
'''
self.cellTextPlaceHolder = ' '
self.column_cur_size = 12
self.row_cur_size = 16
self.appToLoad = None
self.appSelected = None
self.gridScrollWindow = self.builder.get_object('scrolledwindow2')
self.gridTable = Gtk.Grid()
self.gridScrollWindow.add_with_viewport(self.gridTable)
self.nodeIDDict = OrderedDict()
self.appStatusDict = OrderedDict()
self.init_GridTable()
self.mainWindow.show_all()
def run(self):
self.messages.insert(self.messages.get_end_iter(), " " * 256 + "\n")
Gtk.main()
# def bg_server(self, source=None, cond=None):
# '''
# Check if there is something pending from the server thread. Called by the main GUI loop
# '''
# if self.conn:
# self.conn.poll_all()
# return True
# else:
# return False
def log(self, text, prompt='> '):
global guiLock
with guiLock:
end = self.messages.get_end_iter()
text = prompt + text + '\n'
self.messages.insert(end, text)
self.updateStatus(text)
def on_LogChanged(self, *_args):
with guiLock:
adj = self.logWindow.get_vadjustment()
upper, page = adj.get_upper(), adj.get_page_size()
adj.set_value(upper - page)
def on_serverMessage(self, _channel=None, _cond=None):
'''
Callback used by the service thread(s): it prints a log message.
'''
while True:
try:
text = self.socket.recv_pyobj(flags=zmq.NOBLOCK)
self.log(text)
except zmq.error.ZMQError:
break
return True
def isIPaddress(self, addr):
try:
socket.inet_aton(addr)
return True
except socket.error:
return False
def getIPaddress(self, hName):
if self.isIPaddress(hName):
return hName
else:
try:
ipAddr = socket.gethostbyname(hName)
return ipAddr
except socket.error:
return hName
def on_ConsoleEntry(self, *args):
'''
Called when the console entry receives an 'activate' event
'''
global guiLock
fabcmd = self.consoleIn.get_text()
if len(fabcmd) == 0: fabcmd = "help"
fcmd = "fab"
fflag = "-f"
fpath = self.controller.fabModule
hosts = self.controller.getClients()
tPath = None
if len(hosts) == 0:
self.log('? No hosts connected - using default')
cmd = str.join(' ', (fcmd, fflag, fpath, fabcmd))
else:
cHost = self.getIPaddress(self.controller.nodeName)
hNames = [
self.getIPaddress(socket.getfqdn(host)) for host in hosts
]
hConf = {'RIAPS': {'nodes': hNames, 'control': cHost}}
fAppsFolder = ""
if cHost in hNames:
appsFolder = os.getenv('riapsApps', None)
fAppsFolder = "--set RIAPSAPPS=%s" % appsFolder if appsFolder else ""
_drop, tPath = tempfile.mkstemp(text=True)
with open(tPath, "w") as tFd:
toml.dump(hConf, tFd)
fhostsFile = ("--set hostsFile=" + tPath)
cmd = str.join(
' ', (fcmd, fflag, fpath, fabcmd, fhostsFile, fAppsFolder))
self.log(cmd)
proc = subprocess.run(shlex.split(cmd),
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
resp = proc.stdout.decode('utf-8')
if tPath: os.unlink(tPath)
# print(resp)
# self.log(resp,': ')
for line in resp.split('\n'):
if len(line) > 0:
self.log(line, ': ')
self.consoleIn.delete_text(0, -1)
def selectFile(self, title, patterns):
'''
File selection dialog
'''
self.fcd = Gtk.FileChooserDialog(
"Select " + str(title), None, Gtk.FileChooserAction.OPEN,
(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, Gtk.STOCK_OPEN,
Gtk.ResponseType.OK))
for pattern in patterns:
filterR = Gtk.FileFilter()
filterR.set_name("%s" % pattern)
filterR.add_pattern(pattern)
self.fcd.add_filter(filterR)
filterA = Gtk.FileFilter()
filterA.set_name("All files")
filterA.add_pattern("*")
self.fcd.add_filter(filterA)
self.fcd.set_transient_for(self.mainWindow)
self.response = self.fcd.run()
fileName = None
if self.response == Gtk.ResponseType.OK:
fileName = self.fcd.get_filename()
self.fcd.destroy()
return fileName
def selectFolder(self, title):
'''
Folder selection dialog
'''
self.fcd = Gtk.FileChooserDialog(
"Select " + str(title), None, Gtk.FileChooserAction.SELECT_FOLDER,
(Gtk.STOCK_CANCEL, Gtk.ResponseType.CANCEL, "Select",
Gtk.ResponseType.OK))
self.fcd.set_transient_for(self.mainWindow)
self.response = self.fcd.run()
folderName = None
if self.response == Gtk.ResponseType.OK:
folderName = self.fcd.get_filename()
self.fcd.destroy()
return folderName
def isAppOK(self):
aName = self.appNameEntry.get_text()
dName = self.deplNameEntry.get_text()
return (aName != None and aName != '' and dName != None
and dName != '')
def on_SelectApplication(self, *args):
'''
App selection. Sets the app entry and calls the controller to compile the app model.
'''
fileName = self.selectFile("application model", ["*.riaps", "*.json"])
if fileName != None:
self.appNameEntry.set_text(os.path.basename(fileName))
self.controller.compileApplication(fileName,
self.folderEntry.get_text())
#if self.isAppOK():
# self.launchButton.set_sensitive(True)
# self.removeButton.set_sensitive(True)
def clearApplication(self):
'''
Clears the app entry.
'''
self.appNameEntry.set_text('')
def on_SelectDeployment(self, *args):
'''
Deployment selection. Sets the deployment entry and calls the controller
to compile the deployment model.
'''
fileName = self.selectFile("deployment", ["*.depl", "*.json"])
if fileName != None:
self.deplNameEntry.set_text(os.path.basename(fileName))
self.appToLoad = self.controller.compileDeployment(fileName)
#if self.isAppOK():
# self.launchButton.set_sensitive(True)
# self.removeButton.set_sensitive(True)
def clearDeployment(self):
'''
Clears the deployment entry
'''
self.deplNameEntry.set_text('')
def on_folderEntryActivate(self, *args):
'''
App folder selection. Called when the folder entry or the folder button is activated.
'''
folderName = self.selectFolder("application directory")
if folderName != None:
self.folderEntry.set_text(folderName)
self.controller.setAppFolder(folderName)
def on_haltAll(self, *args):
'''
Reset and halt all connected clients. Deplos maybe restarted automatically.
'''
self.controller.cleanAll()
self.controller.killAll()
def on_resetAll(self, *args):
'''
Clean all connected deplos (stop/remove apps)
'''
self.controller.cleanAll()
def on_Quit(self, *args):
'''
Quit the app. Forces a return from the GUI loop
'''
# self.conn.close()
self.socket.close()
Gtk.main_quit()
"""
Begin Status Table Additions
"""
def updateStatus(self, text):
'''
Server message parser and dispatcher. Based on the log message received, updates the status grid gui.
To be deprecated when the server is updated to call directly the status grid gui update functions.
'''
statusList = text.split(' ')
if statusList[0] == ('>'):
ip = re.findall(r'[0-9]+(?:\.[0-9]+){3}', text)
#if len(ip) == 0:
# return
ip = ''.join(ip)
if statusList[1] == ('+'): # node connected
self.update_node_connected_status(ip)
return
elif statusList[1] == '-': # node disconnected
self.update_node_disconnected_status(ip)
return
elif statusList[1] == 'R': # remove status
self.update_remove_status(statusList[2])
return
elif statusList[1] == 'H': # halt status
self.update_halt_status(statusList[2], statusList[3])
return
elif statusList[1] == 'L': # launch status
self.update_launch_status(statusList)
""" Grid table update functions - to be used as entry points for status grid gui updates """
def update_node_connected_status(self, ip):
'''
A new node connected to the controller
'''
self.node_connected(ip)
def update_node_disconnected_status(self, ip):
'''
A node disconnected from the controller
'''
self.node_disconnected(ip)
def update_remove_status(self, app):
'''
An app has been removed
'''
# '> R DistributedAverager
self.remove_app(app)
def update_halt_status(self, node, app):
'''
An app has been halted on a node
'''
#'> H 129.59.105.70 DistributedAverager Averager'
self.halt_app(node, app)
def update_launch_status(self, info):
'''
An app has been launched on a node
'''
# '> L 129.59.105.70 DistributedAverager Averager []'
node = info[2]
app = info[3]
actor = info[4]
self.launch_app(node, app, actor)
def update_node_apps(self, clientName, data):
'''
Update the gui with list of apps running on the client
'''
global guiLock
with guiLock:
if not data: return
for item in data:
appName, actors = item[0], item[1]
self.add_app(appName)
for actorName in actors:
self.launch_app(clientName, appName, actorName)
self.controller.addToLaunchList(clientName, appName,
actorName)
""" Status grid gui update functions - these are the actual update functions """
def node_connected(self, ip):
'''
A node connected to the controller
'''
self.nodeIDDict[ip] = True
num_nodes = len(self.nodeIDDict)
num_node_cols = self.column_cur_size - 1
if num_nodes > num_node_cols:
for i in range(self.column_cur_size, num_nodes + 1):
self.add_table_column(i)
self.column_cur_size = self.column_cur_size + 1
keys = list(self.nodeIDDict.keys())
col_id = keys.index(ip) + 1
cell = self.gridTable.get_child_at(col_id, 0)
if cell is not None:
self.modify_text_cell_color(cell, 'black', 'white')
self.modify_text_cell_text(cell, ip)
self.gridTable.show_all()
def node_disconnected(self, ip):
'''
A node disconnected from the controller
'''
if ip not in self.nodeIDDict:
return
self.nodeIDDict[ip] = False
col_keys = list(self.nodeIDDict.keys())
col_idx = col_keys.index(ip) + 1
c_cell = self.gridTable.get_child_at(col_idx, 0)
if c_cell is not None:
self.modify_text_cell_color(c_cell, 'black', 'black')
self.modify_text_cell_text(c_cell, '')
# modify data - reset the data at a particular (col_idx)
for row_idx, key in enumerate(self.appStatusDict, 1):
self.appStatusDict[key][col_idx] = ''
r_cell = self.gridTable.get_child_at(col_idx, row_idx)
if r_cell is not None:
self.modify_text_cell_color(r_cell, 'white')
self.modify_text_cell_text(r_cell,
self.cellTextPlaceHolder)
self.gridTable.show_all()
def remove_app(self, app):
'''
An app has been removed from the system
'''
# modify gui
index = list(self.appStatusDict.keys()).index(app) + 1
self.remove_table_row(index)
self.row_cur_size = self.row_cur_size - 1
# modify data
del self.appStatusDict[app]
self.create_table_row(self.row_cur_size, self.column_cur_size)
self.row_cur_size = self.row_cur_size + 1
self.gridTable.show_all()
def halt_app(self, node, app):
'''
An app has been halted
'''
if node not in self.nodeIDDict:
return
if self.nodeIDDict[node] is True:
col_map = list(self.nodeIDDict.keys())
col_id = col_map.index(node) + 1
row_map = list(self.appStatusDict.keys())
row_id = row_map.index(app) + 1
cell = self.gridTable.get_child_at(col_id, row_id)
self.modify_text_cell_color(cell, 'grey')
def launch_app(self, node, app, actor):
'''
An app has been launched
'''
if node not in self.nodeIDDict:
return
col_map = list(self.nodeIDDict.keys())
col_id = col_map.index(node) + 1
row_map = list(self.appStatusDict.keys())
row_id = row_map.index(app) + 1
existing_actor = self.appStatusDict[app][col_id - 1]
if existing_actor == 'None' or existing_actor == '':
self.appStatusDict[app][col_id - 1] = actor
elif actor not in existing_actor:
self.appStatusDict[app][col_id - 1] = existing_actor + '\n' + actor
cell = self.gridTable.get_child_at(col_id, row_id)
if cell is None:
self.gridTable.attach(
self.create_table_cell(self.appStatusDict[app][col_id - 1],
'black', 'lime'), col_id, row_id, 1, 1)
else:
self.modify_text_cell_color(cell, 'lime', 'black')
child_list = cell.get_children()
child_list[0].set_label(self.appStatusDict[app][col_id - 1])
self.gridTable.show_all() # not sure if it's necessary here
def add_app(self, app):
if app in self.appStatusDict:
return
self.appStatusDict[app] = [''] * self.column_cur_size
num_apps = len(self.appStatusDict)
num_app_rows = self.row_cur_size - 1
if (num_apps > num_app_rows):
for i in range(self.row_cur_size, num_apps + 1):
self.create_table_row(i, self.column_cur_size)
self.row_cur_size = self.row_cur_size + 1
cell = self.gridTable.get_child_at(0, len(self.appStatusDict))
cell.destroy()
self.gridTable.attach(self.create_app_menu_button(app), 0,
len(self.appStatusDict), 1, 1)
self.gridScrollWindow.show_all()
def on_loadApplication(self, widget):
'''
Load the selected application onto to the network
'''
# add a row in the table for the application
if self.appToLoad is None:
return
self.add_app(self.appToLoad)
self.clearApplication()
self.clearDeployment()
self.appToLoad = None
def on_viewApplication(self, widget):
'''
View the selected application as to be deployed
'''
model = self.appNameEntry.get_text()
deplo = self.deplNameEntry.get_text()
try:
fileName = gviz(model, deplo) + '.dot'
subprocess.Popen(['xdot', fileName])
except:
pass
''' Event handlers for widgets(buttons etc) '''
def on_show_app_ctrl_options(self, widget):
# set selected app name
self.appSelected = widget.get_label()
widget.get_popup().show_all()
def on_launch_app_press(self, widget):
appSelected = self.appSelected
self.appSelected = ''
self.controller.launchByName(appSelected)
def on_stop_app_press(self, widget):
appSelected = self.appSelected
self.appSelected = ''
self.controller.haltByName(appSelected)
def on_remove_app_press(self, widget):
appSelected = self.appSelected
self.appSelected = ''
self.controller.removeAppByName(appSelected)
''' Initialize grid table with blank cells '''
def init_GridTable(self):
# Add 1st button
for c in range(self.row_cur_size):
self.create_table_row(c, self.column_cur_size)
def create_table_row(self, row_index, col_length):
for i in range(col_length):
cell = self.create_table_cell(self.cellTextPlaceHolder)
self.gridTable.attach(cell, i, row_index, 1, 1)
if row_index == 0:
self.modify_text_cell_color(cell, 'black', 'white')
if i == 0:
self.modify_text_cell_text(cell, "App \\ Node")
''' Utility functions '''
def create_app_menu_button(self, text):
menu_button = Gtk.MenuButton(text)
menu_button.connect('pressed', self.on_show_app_ctrl_options)
menu = Gtk.Menu()
menu_button.set_popup(menu)
item = Gtk.ImageMenuItem.new_from_stock(Gtk.STOCK_MEDIA_PLAY)
item.get_child().set_text('Launch')
item.connect('activate', self.on_launch_app_press)
menu.append(item)
item = Gtk.ImageMenuItem.new_from_stock(Gtk.STOCK_MEDIA_STOP)
item.connect('activate', self.on_stop_app_press)
menu.append(item)
menu.append(Gtk.SeparatorMenuItem())
item = Gtk.ImageMenuItem.new_from_stock(Gtk.STOCK_REMOVE)
item.connect('activate', self.on_remove_app_press)
menu.append(item)
return menu_button
def create_table_cell(self, text, fg_color='blue', bg_color='white'):
try:
frame_box = Gtk.Frame()
frame_box.modify_fg(Gtk.StateType.NORMAL,
Gdk.color_parse(fg_color))
frame_box.modify_bg(Gtk.StateType.NORMAL,
Gdk.color_parse(bg_color))
text_label = Gtk.Label()
text_content = '<b>' + ''.join(text) + '</b>'
text_label.set_markup(text_content)
text_label.set_justify(Gtk.Justification.CENTER)
frame_box.add(text_label)
return frame_box
except Exception as e:
message = str(e)
return None
def add_table_column(self, col_index):
for i in range(self.row_cur_size):
self.gridTable.attach(
self.create_table_cell(self.cellTextPlaceHolder, 'black'),
col_index, i, 1, 1)
self.gridTable.show_all()
def remove_table_row(self, index):
self.gridTable.remove_row(index)
''' Cell helper functions'''
def modify_text_cell_color(self, cell, bg='', fg=''):
if cell is not None:
if bg != '':
cell.modify_bg(Gtk.StateType.NORMAL, Gdk.color_parse(bg))
if fg != '':
cell.modify_fg(Gtk.StateType.NORMAL, Gdk.color_parse(fg))
def modify_text_cell_text(self, cell, text=''):
if cell is not None:
children = cell.get_children()
if len(children) > 0:
children[0].set_label(text)
def set_cell_bg_color(self, text):
if 'Actor' in text:
bg_color = 'lime'
if 'Stopped' in text:
bg_color = 'grey'
return bg_color
'''
'''
Created on 16-Aug-2019
@author: Sanjay Ghosh
'''
from _collections import OrderedDict
mydict = OrderedDict()
T = int(input())
for loop in range(T):
myarr = input().split()
key = " ".join(myarr[0:len(myarr) - 1])
value = int(" ".join(myarr[len(myarr) - 1:]))
if key not in mydict:
mydict[key] = value
else:
mydict[key] = mydict[key] + value
for key in mydict.keys():
print(key + " " + str(mydict[key]))
#deque
q = deque([1, 2, 3])
q.append('a')
q.append('abc')
q.appendleft('first')
oFirst = q.pop()
print(oFirst)
#defaultdict
d = defaultdict(lambda: None)
d['key1'] = 'abc'
print(d['key1'])
print(d['key2'])
#orderdict
od = OrderedDict([('b', 1), ('a', 2), ('c', 3)])
print(od)
print(od.keys())
#Counter
c = Counter()
for ch in "programming":
c[ch] = c[ch] + 1
print(c)
print(c.elements())
print(c.most_common(3))
print(sum(c.values()))
c1 = Counter({'g': 2, 'm': 2, 'r': 2, 'a': 1, 'i': 1, 'o': 1, 'n': 1, 'p': 1})
print(c1)
def import_model(sbml_model: Union[str, 'libsbml.Model'],
condition_table: Optional[Union[str, pd.DataFrame]] = None,
observable_table: Optional[Union[str, pd.DataFrame]] = None,
measurement_table: Optional[Union[str, pd.DataFrame]] = None,
model_name: Optional[str] = None,
model_output_dir: Optional[str] = None,
verbose: Optional[Union[bool, int]] = True,
allow_reinit_fixpar_initcond: bool = True,
**kwargs) -> None:
"""
Create AMICI model from PEtab problem
:param sbml_model:
PEtab SBML model or SBML file name.
:param condition_table:
PEtab condition table. If provided, parameters from there will be
turned into AMICI constant parameters (i.e. parameters w.r.t. which
no sensitivities will be computed).
:param observable_table:
PEtab observable table.
:param measurement_table:
PEtab measurement table.
:param model_name:
Name of the generated model. If model file name was provided,
this defaults to the file name without extension, otherwise
the SBML model ID will be used.
:param model_output_dir:
Directory to write the model code to. Will be created if doesn't
exist. Defaults to current directory.
:param verbose:
Print/log extra information.
:param allow_reinit_fixpar_initcond:
See :class:`amici.ode_export.ODEExporter`. Must be enabled if initial
states are to be reset after preequilibration.
:param kwargs:
Additional keyword arguments to be passed to
:meth:`amici.sbml_import.SbmlImporter.sbml2amici`.
"""
set_log_level(logger, verbose)
logger.info(f"Importing model ...")
# Get PEtab tables
observable_df = petab.get_observable_df(observable_table)
# to determine fixed parameters
condition_df = petab.get_condition_df(condition_table)
if observable_df is None:
raise NotImplementedError("PEtab import without observables table "
"is currently not supported.")
# Model name from SBML ID or filename
if model_name is None:
if isinstance(sbml_model, libsbml.Model):
model_name = sbml_model.getId()
else:
model_name = os.path.splitext(os.path.split(sbml_model)[-1])[0]
if model_output_dir is None:
model_output_dir = os.path.join(os.getcwd(), model_name)
logger.info(f"Model name is '{model_name}'. "
f"Writing model code to '{model_output_dir}'.")
# Load model
if isinstance(sbml_model, str):
# from file
sbml_reader = libsbml.SBMLReader()
sbml_doc = sbml_reader.readSBMLFromFile(sbml_model)
sbml_model = sbml_doc.getModel()
else:
# Create a copy, because it will be modified by SbmlImporter
sbml_doc = sbml_model.getSBMLDocument().clone()
sbml_model = sbml_doc.getModel()
show_model_info(sbml_model)
sbml_importer = amici.SbmlImporter(sbml_model)
sbml_model = sbml_importer.sbml
if observable_df is not None:
observables, noise_distrs, sigmas = \
get_observation_model(observable_df)
logger.info(f'Observables: {len(observables)}')
logger.info(f'Sigmas: {len(sigmas)}')
if not len(sigmas) == len(observables):
raise AssertionError(
f'Number of provided observables ({len(observables)}) and sigmas '
f'({len(sigmas)}) do not match.')
# TODO: adding extra output parameters is currently not supported,
# so we add any output parameters to the SBML model.
# this should be changed to something more elegant
# <BeginWorkAround>
formulas = chain((val['formula'] for val in observables.values()),
sigmas.values())
output_parameters = OrderedDict()
for formula in formulas:
# we want reproducible parameter ordering upon repeated import
free_syms = sorted(sp.sympify(formula).free_symbols,
key=lambda symbol: symbol.name)
for free_sym in free_syms:
sym = str(free_sym)
if sbml_model.getElementBySId(sym) is None:
output_parameters[sym] = None
logger.debug(f"Adding output parameters to model: {output_parameters}")
for par in output_parameters.keys():
petab.add_global_parameter(sbml_model, par)
# <EndWorkAround>
# TODO: to parameterize initial states or compartment sizes, we currently
# need initial assignments. if they occur in the condition table, we
# create a new parameter initial_${startOrCompartmentID}.
# feels dirty and should be changed (see also #924)
# <BeginWorkAround>
initial_states = [
col for col in condition_df if sbml_model.getSpecies(col) is not None
]
initial_sizes = [
col for col in condition_df
if sbml_model.getCompartment(col) is not None
]
fixed_parameters = []
if len(initial_states) or len(initial_sizes):
# add preequilibration indicator variable
# NOTE: would only be required if we actually have preequilibration
# adding it anyways. can be optimized-out later
if sbml_model.getParameter(PREEQ_INDICATOR_ID) is not None:
raise AssertionError("Model already has a parameter with ID "
f"{PREEQ_INDICATOR_ID}. Cannot handle "
"species and compartments in condition table "
"then.")
indicator = sbml_model.createParameter()
indicator.setId(PREEQ_INDICATOR_ID)
indicator.setName(PREEQ_INDICATOR_ID)
# Can only reset parameters after preequilibration if they are fixed.
fixed_parameters.append(PREEQ_INDICATOR_ID)
for assignee_id in initial_sizes + initial_states:
init_par_id_preeq = f"initial_{assignee_id}_preeq"
init_par_id_sim = f"initial_{assignee_id}_sim"
for init_par_id in [init_par_id_preeq, init_par_id_sim]:
if sbml_model.getElementBySId(init_par_id) is not None:
raise ValueError(
"Cannot create parameter for initial assignment "
f"for {assignee_id} because an entity named "
f"{init_par_id} exists already in the model.")
init_par = sbml_model.createParameter()
init_par.setId(init_par_id)
init_par.setName(init_par_id)
assignment = sbml_model.createInitialAssignment()
assignment.setSymbol(assignee_id)
formula = f'{PREEQ_INDICATOR_ID} * {init_par_id_preeq} '\
f'+ (1 - {PREEQ_INDICATOR_ID}) * {init_par_id_sim}'
math_ast = libsbml.parseL3Formula(formula)
assignment.setMath(math_ast)
# <EndWorkAround>
fixed_parameters.extend(
get_fixed_parameters(sbml_model=sbml_model, condition_df=condition_df))
logger.debug(f"Fixed parameters are {fixed_parameters}")
logger.info(f"Overall fixed parameters: {len(fixed_parameters)}")
logger.info(
"Variable parameters: " +
str(len(sbml_model.getListOfParameters()) - len(fixed_parameters)))
# Create Python module from SBML model
sbml_importer.sbml2amici(
model_name=model_name,
output_dir=model_output_dir,
observables=observables,
constant_parameters=fixed_parameters,
sigmas=sigmas,
allow_reinit_fixpar_initcond=allow_reinit_fixpar_initcond,
noise_distributions=noise_distrs,
verbose=verbose,
**kwargs)
class TrainScriptLauncher:
#------------------------------------
# Constructor
#-------------------
# Use distributed torch default port:
COMM_PORT = '5678'
def __init__(self, unittesting=False):
self.hostname = socket.getfqdn()
if unittesting:
# Let unittests create an instance
# and call individual methods:
return
# Logging to console during launch:
self.log = LoggingService()
# Convenience: directory of this
# script, and project root directory
curr_dir = Path(__file__).parent
proj_root = curr_dir.joinpath('../..').resolve()
self.curr_dir = str(curr_dir)
self.proj_root = str(proj_root)
args_parser = BirdsTrainingArgumentsParser(
formatter_class=BirdsTrainingArgumentsParser.
BlankLinesHelpFormatter,
description="PyTorch distributed training launch "
"helper to spawn multiple distributed "
"birds_train_parallel.py processes")
all_args = args_parser.parse_args()
# Separate the args for this launch script
# from the args destined for the copies of
# the train script:
self.launch_args = all_args['launch_args']
self.script_args = all_args['script_args']
# Build the gpu_landscape dict:
self.gather_world_layout(self.launch_args)
self.GPUS_USED_THIS_MACHINE = self.gpu_landscape[
self.hostname]['num_gpus']
#------------------------------------
# gather_world_layout
#-------------------
def gather_world_layout(self, launch_args):
'''
# Compute a unique number for each GPU within
# the group of nodes (machines). Starting with
# the master node's first GPU as 0 (if master node
# has a GPU.
# The resulting GPU layout is assigned to
# variable gpu_landscape:
:param launch_args:
:type launch_args:
'''
try:
config_file = launch_args['config']
if not os.path.exists(config_file):
raise ConfigError(
f"Configuration file {config_file} that was provided as command line arg does not exist."
)
except KeyError:
raise RuntimeError(
"Error: launch args must include a config file. See config.cfg.Example in project root"
)
self.config = DottableConfigParser(config_file)
# Ensure that the launch_args contains
# the path to the training script. It
# will be there if provided on the cmd line.
# But it may instead be under Path:train_script
# in the configuration:
try:
self.launch_args['training_script']
except KeyError:
# The training script was not specified
# on the command line. Is it in the config
# file:
try:
self.launch_args['training_script'] = self.config.getpath(
'Paths', 'train_script', relative_to=self.curr_dict)
except KeyError:
raise ValueError(
"No training script specified on command line or in config file"
)
try:
self.world_map_path = self.config.getpath(
'Paths', 'world_map', relative_to=self.curr_dir)
except KeyError:
raise RuntimeError(
f"Could not find entry for 'world_map' in config file {config_file}"
)
self.world_map = self.read_world_map(self.world_map_path)
# Ensure that this machine has an
# entry in the world_map:
try:
# Get this machine's info (sub)dict:
_my_world_info = self.world_map[self.hostname]
except KeyError:
raise ConfigError(
f"World map file does not contain entry for this machine ({self.hostname})"
)
self.compute_landscape = {}
# Whether or not machine running this
# code is the master node:
self.am_master_node = False
# Build gpu_landscape, which maps
# machine names to the rank range
# that they occupy via the number of
# their GPUs
#
# {machine_name1 : [1],
# machine_name2 : [0],
# machine_name3 : [1,2,3],
self.gpu_landscape = self.build_compute_landscape(self.world_map)
if self.master_hostname is None:
raise ConfigError(
f'No master machine in {self.world_map_path}; one entry needs to be "master" : 1'
)
# Common pytorch port is either in the config file,
# or we use the pytorch default
self.MASTER_PORT = self.config.getint('Parallelism', 'master_port',
self.COMM_PORT)
# Handle special case: no GPUs anywere, and
# we are on node 0: in that case start a single
# copy of the training script. If it is written
# properly, it will detect the absence of a GPU,
# and use the CPU. This happens during debugging
# on a laptop:
if self.WORLD_SIZE == 0 and self.am_master_node:
self.WORLD_SIZE += 1
# If trying to launch on a node without GPUs,
# when GPUs are available elsewhere, refuse to
# start the script (is this needed?):
if not TESTING:
if self.my_gpus == 0 and self.WORLD_SIZE > 0:
raise RuntimeError(
"This machine does not have any GPU, but others do; training script not started."
)
#------------------------------------
# launch_scripts
#-------------------
def launch_scripts(self):
'''
Launch (possibly) multiple copies of
the training script. Use world_map.json
to know how many, and which GPUs this
machine is to use.
Each copy is told:
o MASTER_ADDR # Where to reach the coordinating process
o MASTER_PORT # Corresponding port
o RANK # The copy's sequence number, which is
# Unique across all participating machines
o LOCAL_RANK # Which of this machine's GPU to use (0-origin)
o WORLD_SIZE # How many GPUs are used on all machines together
o GPUS_USED_THIS_MACHINE # Number of GPUs *used* on this
# machine, according to the world_map.
'''
# Compute a unique number for each GPU within
# the group of nodes (machines). Starting with
# the master node's first GPU as 0 (if master node
# has a GPU.
# The resulting GPU layout is assigned to
# variable gpu_landscape:
#
# {<machine_name> :
# This machine's range of ranks:
rank_range = self.gpu_landscape[self.hostname]['rank_range']
this_machine_gpu_ids = self.gpu_landscape[
self.hostname]['gpu_device_ids']
min_rank_this_machine = self.gpu_landscape[self.hostname]['start_rank']
local_rank = 0
# Map from process object to rank (for debug msgs):
self.who_is_who = OrderedDict()
for rank in rank_range:
cmd = self.training_script_start_cmd(
rank, len(this_machine_gpu_ids), local_rank,
min_rank_this_machine, self.launch_args, self.script_args)
# Copy stdin, and give the copy to the subprocess.
# This enables the subprocess to ask user whether
# to save training state in case of a cnt-C:
newstdin = os.fdopen(os.dup(sys.stdin.fileno()))
# Spawn one training script.
process = subprocess.Popen(
cmd,
stdin=newstdin,
stdout=None, # Script inherits this launch
stderr=None # ... script's stdout/stderr
)
self.who_is_who[process] = rank
local_rank += 1
if not self.launch_args['quiet']:
print(
f"Node {self.hostname} {os.path.basename(sys.argv[0])}: Num processes launched: {len(self.who_is_who)}"
)
if self.am_master_node:
print(f"Awaiting {self.WORLD_SIZE} process(es) to finish...")
else:
print(f"Awaiting {self.my_gpus} process(es) to finish...")
failed_processes = []
try:
for process in self.who_is_who.keys():
process.wait()
if process.returncode != 0:
failed_processes.append(process)
continue
except KeyboardInterrupt:
# Gently kill the training scripts:
self.handle_cnt_c()
pass # See which processes get the interrupt
num_failed = len(failed_processes)
if num_failed > 0:
print(f"Number of failed training scripts: {num_failed}")
for failed_process in failed_processes:
train_script = self.launch_args['training_script']
script_rank = self.who_is_who[failed_process]
msg = (
f"Training script {train_script} (rank {script_rank}) encountered error(s); see logfile"
)
print(msg)
#------------------------------------
# training_script_start_cmd
#-------------------
def training_script_start_cmd(self, rank, gpus_used_this_machine,
local_rank, min_rank_this_machine,
launch_args, script_args):
'''
From provided information, creates a legal
command string for starting the training script.
:param rank: rank of the script; i.e. it's process' place
in the sequence of all train script processes
across all machines
:type rank: int
:param gpus_used_this_machine: number of GPU devices to
be used, according to the world_map; may be less than
number of available GPUs
:type gpus_used_this_machine: int
:param local_rank: index into the local sequence of GPUs
for for the GPU that the script is to use
:type local_rank: int
:param min_rank_this_machine: the lowest of the ranks among
the training scripts on this machine
:type min_rank_this_machine: int
:param launch_args: command line arguments intended for the
launch script, as opposed to being destined for the
train script
:type launch_args: {str : Any}
:param script_args: additional args for the train script
:type script_args: {str : Any}
'''
# Build the shell command line,
# starting with 'python -u':
cmd = [sys.executable, "-u"]
cmd.append(launch_args['training_script'])
# Add the args for the script that were
# in the command line:
for arg_name in script_args.keys():
script_arg_val = script_args[arg_name]
if script_arg_val is None or arg_name == 'config':
# Skip over non-specified CLI args:
continue
cmd.append(f"--{arg_name}={script_args[arg_name]}")
# Add the 'secret' args that tell the training
# script all the communication parameters:
cmd.extend([
f"--MASTER_ADDR={self.MASTER_ADDR}",
f"--MASTER_PORT={self.MASTER_PORT}", f"--RANK={rank}",
f"--LOCAL_RANK={local_rank}",
f"--MIN_RANK_THIS_MACHINE={min_rank_this_machine}",
f"--WORLD_SIZE={self.WORLD_SIZE}",
f"--GPUS_USED_THIS_MACHINE={gpus_used_this_machine}"
])
# Finally, the obligatory non-option arg
# to the training script: the configuration
# file:
config_file_name = script_args['config']
cmd.append(config_file_name)
self.log.debug(f"****** Launch: the cmd is {cmd}")
return cmd
#------------------------------------
# read_world_map
#-------------------
def read_world_map(self, path):
'''
Read the JSON5 world map file, and
return a corresponding dict. JSON5
allows something like:
/*
This is a block comment.
Notice the lacking quote
chars around the keys below.
The are optional in JSON5
*/
{quintus.stanford.edu : {
"master" : Yes
"gpus" : 2
},
quatro.stanford.edu : {
"gpus" : 2,
"devices" : [1,2]
}
}
BUT: JSON5 gets angry at dots in the
keys.
So we first read the file, and try to find
the machine names. We temporarily replace
them with an acceptable marker, and then
convert back.
:param path: path to world map file
:type path: string
'''
dot_substitute = '___'
try:
# Read all the world map file lines:
with open(path, 'r') as world_map_fd:
tmp_world_map = world_map_fd.readlines()
except IOError as e:
raise IOError(f"World map file at {path} not found") from e
# Replace occurrences of '.' with dot_substitute:
new_text = []
for line in tmp_world_map:
new_text.append(line.replace('.', dot_substitute))
# ... and make one string from all the lines:
json_str = '\n'.join(new_text)
try:
# Hopefully, JSON5 will eat it now:
world_map_almost = json5.loads(json_str)
except JSONError as e:
raise JSONError(
f"World map file at {path} contains bad JSON") from e
# Need to fix all the dot substitutions.
# At this point the data structure is
# { <machine_name> : {spec_attr1 : val1,
# spec_attr2 : val2,
# }
# }
# Fix the machine names first:
mach_names_fixed = [
machine_name.replace(dot_substitute, '.')
for machine_name in world_map_almost.keys()
]
machine_specs_fixed = []
# Now dig into each of the nested machine spec
# dicts, and fix attrs and values there:
for spec in world_map_almost.values():
# Spec is a dict nested inside the outer one:
spec_fixed = {
key.replace(dot_substitute, '.'): val.replace(
dot_substitute, '.') if isinstance(val, str) else val
for key, val in spec.items()
}
machine_specs_fixed.append(spec_fixed)
# Put it all together:
world_map = {
machine_name: spec_dict
for machine_name, spec_dict in zip(mach_names_fixed,
machine_specs_fixed)
}
return world_map
#------------------------------------
# build_compute_landscape
#-------------------
def build_compute_landscape(self, world_map):
'''
# Using the world_map.json config file, build
# a dict self.gpu_landscape like this:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# Also sets
# o self.master_hostname, the hostname
# running the one process that coordinates all others.
# o self.WORLD_SIZE, number of GPUs used across all machines
# o self.my_gpus, the number of GPUs on this machine
:param world_map:
:type world_map:
:return: information about how many GPUs are
on each node
:rtype: OrderedDict
'''
if not self.hostname in world_map.keys():
raise ConfigError(
f"World map does not contain an entry for this machine {self.hostname}"
)
# World size is the number of training script processes,
# which is equal to number of GPUs used on all participating
# machines combined:
# Number of GPUs across all machines:
self.WORLD_SIZE = 0
self.master_hostname = None
# Go through the world map, machine (a.k.a. node)
# one at a time, in alpha order of the machine
# names to ensure all copies of this script
# come to the same conclusions about ranks
# Build gpu_landscape:
#
# {'machine_name1' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# {'machine_name2' : {'start_rank' : <int>,
# 'num_gpus' : <int>,
# 'gpu_device_ids': [<int>,<int>,...]
# }
#
# The structure is an OrderedDict(), containing
# machines alphabetically by name. This discipline
# is required so that all copies of this launch script
# (one copy per machine) arrive at the same ordering of
# GPUs:
gpu_landscape = OrderedDict({})
for machine_name in sorted(world_map.keys()):
# Get dict of info about the machine:
machine_info = world_map[machine_name]
try:
machine_gpus = machine_info['gpus']
except KeyError:
print(
"World map must include a 'gpus' entry; the value may be 0"
)
gpu_landscape[machine_name] = {}
gpu_landscape[machine_name]['num_gpus'] = machine_gpus
# List of GPU numbers to use is optional
# in world_maps:
machine_gpus_to_use = machine_info.get('devices', None)
if machine_gpus_to_use is None:
# Use all GPUs on that machine:
machine_gpus_to_use = list(range(machine_gpus))
gpu_landscape[machine_name]['gpu_device_ids'] = machine_gpus_to_use
# Accept all kinds of affirmatives as values:
# for identification of the master node entry:
is_master_node = machine_info.get('master', False) \
in [1, 'True', 'true', 'Yes', 'yes']
if is_master_node:
self.master_hostname = machine_name
if machine_name == self.hostname:
self.am_master_node = True
try:
self.MASTER_ADDR = socket.gethostbyname(machine_name)
except socket.gaierror:
# For machines that have no
# findable IP address:
self.MASTER_ADDR = '127.0.0.1'
self.WORLD_SIZE += machine_gpus
# Go through the machine enries in gpu_landscape, and
# assign rank ranges to each. Must start with
# the master node, b/c it must start with rank 0.
# For the master node, it is possible that it has
# no GPUs
master_info = gpu_landscape[self.master_hostname]
master_info['rank_range'] = list(range(master_info['num_gpus']))
master_info['start_rank'] = 0
if len(master_info['rank_range']) == 0:
# Master only has a GPU:
master_info['rank_range'] = [0]
# Start assigning more ranks after
# the GPUs of the master:
running_rank = master_info['rank_range'][-1] + 1
for machine_name in gpu_landscape.keys():
if machine_name == self.master_hostname:
# We already did the master node
continue
mach_info = gpu_landscape[machine_name]
mach_info['start_rank'] = running_rank
num_gpus = mach_info['num_gpus']
range_bound = running_rank + (num_gpus if num_gpus > 0 else 1)
mach_info['rank_range'] = list(range(running_rank, range_bound))
running_rank += (num_gpus if num_gpus > 0 else 1)
self.my_gpus = gpu_landscape[self.hostname]['num_gpus']
self.gpu_landscape = gpu_landscape
return gpu_landscape
#------------------------------------
# handle_cnt_c
#-------------------
def handle_cnt_c(self):
'''
Given a list of process instances,
Send SIGINT (cnt-C) to them:
:param procs:
:type procs:
'''
# Line processes up, highest rank first,
# master process last:
procs_terminate = sorted([proc for proc in self.who_is_who.keys()],
key=lambda obj: self.who_is_who[obj],
reverse=True)
for process in procs_terminate:
# If process is no longer running,
# forget about it:
if process.poll is not None:
# Process dead:
continue
process.send_signal(signal.SIGTERM)
process.wait()
def create_atlas(animal, create):
fileLocationManager = FileLocationManager(animal)
atlas_name = 'atlasV7'
THUMBNAIL_DIR = os.path.join(ROOT_DIR, animal, 'preps', 'CH1', 'thumbnail')
ATLAS_PATH = os.path.join(DATA_PATH, 'atlas_data', atlas_name)
ORIGIN_PATH = os.path.join(ATLAS_PATH, 'origin')
VOLUME_PATH = os.path.join(ATLAS_PATH, 'structure')
OUTPUT_DIR = os.path.join(fileLocationManager.neuroglancer_data, 'atlas')
if os.path.exists(OUTPUT_DIR):
shutil.rmtree(OUTPUT_DIR)
os.makedirs(OUTPUT_DIR, exist_ok=True)
origin_files = sorted(os.listdir(ORIGIN_PATH))
volume_files = sorted(os.listdir(VOLUME_PATH))
sqlController = SqlController(animal)
resolution = sqlController.scan_run.resolution
surface_threshold = 0.8
SCALE = (10 / resolution)
structure_volume_origin = {}
for volume_filename, origin_filename in zip(volume_files, origin_files):
structure = os.path.splitext(volume_filename)[0]
if structure not in origin_filename:
print(structure, origin_filename)
break
color = get_structure_number(structure.replace('_L', '').replace('_R', ''))
origin = np.loadtxt(os.path.join(ORIGIN_PATH, origin_filename))
volume = np.load(os.path.join(VOLUME_PATH, volume_filename))
volume = np.rot90(volume, axes=(0, 1))
volume = np.flip(volume, axis=0)
volume[volume > surface_threshold] = color
volume = volume.astype(np.uint8)
structure_volume_origin[structure] = (volume, origin)
col_length = sqlController.scan_run.width/SCALE
row_length = sqlController.scan_run.height/SCALE
z_length = len(os.listdir(THUMBNAIL_DIR))
atlasV7_volume = np.zeros(( int(row_length), int(col_length), z_length), dtype=np.uint8)
print('atlas volume shape', atlasV7_volume.shape)
##### actual data for both sets of points, pixel coordinates
centers = OrderedDict(MD589_centers)
centers_list = []
for value in centers.values():
centers_list.append((value[1]/SCALE, value[0]/SCALE, value[2]))
COM = np.array(centers_list)
atlas_com_centers = OrderedDict()
atlas_all_centers = {}
for structure, (volume, origin) in sorted(structure_volume_origin.items()):
midcol, midrow, midz = origin
row_start = midrow + row_length / 2
col_start = midcol + col_length / 2
z_start = midz / 2 + z_length / 2
row_end = row_start + volume.shape[0]
col_end = col_start + volume.shape[1]
z_end = z_start + (volume.shape[2] + 1) / 2
midcol = (col_end + col_start) / 2
midrow = (row_end + row_start) / 2
midz = (z_end + z_start) / 2
if structure in centers.keys():
atlas_com_centers[structure] = [midrow, midcol, midz]
atlas_all_centers[structure] = [midrow, midcol, midz]
ATLAS_centers = OrderedDict(atlas_com_centers)
ATLAS = np.array(list(ATLAS_centers.values()))
#### both sets of data are scaled to stack of DK52
pprint(COM)
pprint(ATLAS)
#####Transform to auto align
r_auto, t_auto = align_point_sets(ATLAS.T, COM.T)
rotationpath = os.path.join(ATLAS_PATH, f'atlas2{animal}.rotation.npy')
np.save(rotationpath, r_auto)
translatepath = os.path.join(ATLAS_PATH, f'atlas2{animal}.translation.npy')
np.save(translatepath, t_auto)
# Litao, look at the start and end for these structures, the x and y look good
# but the z (section) is off
debug = True
for structure, (volume, origin) in sorted(structure_volume_origin.items()):
print(str(structure).ljust(7),end=": ")
source_point = np.array(atlas_all_centers[structure]) # get adjusted x,y,z from above loop
results = (r_auto @ source_point + t_auto.T).reshape(1,3) # transform to fit
x = results[0][1] # new x
y = results[0][0] # new y
z = results[0][2] # z
x = x - volume.shape[0]/2
y = y - volume.shape[1]/2
x_start = int( round(x))
y_start = int( round(y))
z_start = int(z - volume.shape[2]/4)
x_end = int( round(x_start + volume.shape[0]))
y_end = int( round(y_start + volume.shape[1]))
z_end = int( round(z_start + (volume.shape[2] + 1) // 2))
if debug:
#print('volume shape', volume.shape, end=" ")
print('COM row',
str(int(y)).rjust(4),
'mid col',
str(int(x)).rjust(4),
'mid z',
str(int(z)).rjust(4),
end=" ")
print('Row range',
str(y_start).rjust(4),
str(y_end).rjust(4),
'col range',
str(x_start).rjust(4),
str(x_end).rjust(4),
'z range',
str(z_start).rjust(4),
str(z_end).rjust(4),
end=" ")
if structure in centers.keys():
xo,yo,zo = MD589_centers[structure]
print('COM off by:',
round(x*SCALE - xo, 2),
round(y*SCALE - yo, 2),
round(z - zo, 2),
end=" ")
z_indices = [z for z in range(volume.shape[2]) if z % 2 == 0]
volume = volume[:, :, z_indices]
volume = np.swapaxes(volume, 0, 1)
try:
atlasV7_volume[y_start:y_end, x_start:x_end, z_start:z_end] += volume
except ValueError as ve:
print('Bad fit', end=" ")
print()
resolution = int(resolution * 1000 * SCALE)
print('Shape of downsampled atlas volume', atlasV7_volume.shape)
print('Resolution at', resolution)
if create:
atlasV7_volume = np.rot90(atlasV7_volume, axes=(0, 1))
atlasV7_volume = np.fliplr(atlasV7_volume)
atlasV7_volume = np.flipud(atlasV7_volume)
atlasV7_volume = np.fliplr(atlasV7_volume)
offset = [0,0,0]
ng = NumpyToNeuroglancer(atlasV7_volume, [resolution, resolution, 20000], offset=offset)
ng.init_precomputed(OUTPUT_DIR)
ng.add_segment_properties(get_segment_properties())
ng.add_downsampled_volumes()
ng.add_segmentation_mesh()
#outpath = os.path.join(ATLAS_PATH, f'{atlas_name}.tif')
#io.imsave(outpath, atlasV7_volume.astype(np.uint8))
end = timer()
print(f'Finito! Program took {end - start} seconds')
from _collections import OrderedDict
# print("Before:\n")
# od = OrderedDict()
# od['a'] = 1
# od['b'] = 2
# od['c'] = 3
# od['d'] = 4
# for key, value in od.items():
# print(key, value)
#
# print("\nAfter:\n")
# od['c'] = 5
# for key, value in od.items():
# print(key, value)
input_string = 'aabbccddef'
ordered_dict = OrderedDict()
for input in input_string:
keys = ordered_dict.keys()
if input not in keys:
ordered_dict[input] = 1
else:
ordered_dict[input] += 1
out_string = ""
for key, value in ordered_dict.items():
out_string = out_string + str(key) + str(value)
print(out_string)
def _report_textual_results(self, tally_coll, res_dir):
'''
Give a sequence of tallies with results
from a series of batches, create long
outputs, and inputs lists from all tallies
Computes information retrieval type values:
precision (macro/micro/weighted/by-class)
recall (macro/micro/weighted/by-class)
f1 (macro/micro/weighted/by-class)
acuracy
balanced_accuracy
Combines these results into a Pandas series,
and writes them to a csv file. That file is constructed
from the passed-in res_dir, appended with 'ir_results.csv'.
Finally, constructs Github flavored tables from the
above results, and posts them to the 'text' tab of
tensorboard.
Returns the results measures Series
:param tally_coll: collect of tallies from batches
:type tally_coll: ResultCollection
:param res_dir: directory where all .csv and other
result files are to be written
:type res_dir: str
:return results of information retrieval-like measures
:rtype: pandas.Series
'''
all_preds = []
all_labels = []
for tally in tally_coll.tallies(phase=LearningPhase.TESTING):
all_preds.extend(tally.preds)
all_labels.extend(tally.labels)
res = OrderedDict({})
res['prec_macro'] = precision_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['prec_micro'] = precision_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['prec_weighted'] = precision_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['prec_by_class'] = precision_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['recall_macro'] = recall_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['recall_micro'] = recall_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['recall_weighted'] = recall_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['recall_by_class'] = recall_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['f1_macro'] = f1_score(all_labels,
all_preds,
average='macro',
zero_division=0)
res['f1_micro'] = f1_score(all_labels,
all_preds,
average='micro',
zero_division=0)
res['f1_weighted'] = f1_score(all_labels,
all_preds,
average='weighted',
zero_division=0)
res['f1_by_class'] = f1_score(all_labels,
all_preds,
average=None,
zero_division=0)
res['accuracy'] = accuracy_score(all_labels, all_preds)
res['balanced_accuracy'] = balanced_accuracy_score(
all_labels, all_preds)
res_series = pd.Series(list(res.values()), index=list(res.keys()))
# Write information retrieval type results
# to a one-line .csv file, using pandas Series
# as convenient intermediary:
res_csv_path = os.path.join(res_dir, 'ir_results.csv')
res_series.to_csv(res_csv_path)
res_rnd = {}
for meas_nm, meas_val in res.items():
# Measure results are either floats (precision, recall, etc.),
# or np arrays (e.g. precision-per-class). For both
# cases, round each measure to one digit:
res_rnd[meas_nm] = round(meas_val,1) if type(meas_val) == float \
else meas_val.round(1)
ir_measures_skel = {
'col_header': ['precision', 'recall', 'f1'],
'row_labels': ['macro', 'micro', 'weighted'],
'rows': [[
res_rnd['prec_macro'], res_rnd['recall_macro'],
res_rnd['f1_macro']
],
[
res_rnd['prec_micro'], res_rnd['recall_micro'],
res_rnd['f1_micro']
],
[
res_rnd['prec_weighted'], res_rnd['recall_weighted'],
res_rnd['f1_weighted']
]]
}
ir_per_class_rows = [[
prec_class, recall_class, f1_class
] for prec_class, recall_class, f1_class in zip(
res_rnd['prec_by_class'], res_rnd['recall_by_class'],
res_rnd['f1_by_class'])]
ir_by_class_skel = {
'col_header': ['precision', 'recall', 'f1'],
'row_labels': self.class_names,
'rows': ir_per_class_rows
}
accuracy_skel = {
'col_header': ['accuracy', 'balanced_accuracy'],
'row_labels': ['Overall'],
'rows': [[res_rnd['accuracy'], res_rnd['balanced_accuracy']]]
}
ir_measures_tbl = GithubTableMaker.make_table(ir_measures_skel,
sep_lines=False)
ir_by_class_tbl = GithubTableMaker.make_table(ir_by_class_skel,
sep_lines=False)
accuracy_tbl = GithubTableMaker.make_table(accuracy_skel,
sep_lines=False)
# Write the markup tables to Tensorboard:
self.writer.add_text('Information retrieval measures',
ir_measures_tbl,
global_step=0)
self.writer.add_text('Per class measures',
ir_by_class_tbl,
global_step=0)
self.writer.add_text('Accuracy', accuracy_tbl, global_step=0)
return res_series
class Transect(sm.CustomObject):
base_type = "transect"
type = "transect"
datum = "top of face"
def __init__(self, **kwargs):
super(Transect, self).__init__()
self._locs = OrderedDict()
self.name = kwargs.get("name", None)
start = kwargs.get("start", (0, 0)) # coords (lat, long)
end = kwargs.get("end", (0, 0))
self.s_coords = Coords(lat=start[0], lon=start[1])
self.e_coords = Coords(lat=end[0], lon=end[1])
self.ug_values = []
self.ug_xs = []
self.h_face = kwargs.get("h_face", None)
self.av_ground_slope = kwargs.get("av_ground_slope", None)
self._extra_class_inputs = [
"locs", "start", "end", "ug_values", "ug_xs", "h_face",
"av_ground_slope", "datum"
]
self.inputs = self.inputs + self._extra_class_inputs
def add_cpt_by_coords(self, cpt, coords, **kwargs):
esp = kwargs.get("esp", None)
loc = Loc(cpt=cpt, name=cpt.file_name, esp=esp)
loc.coords = coords
return self.add_loc_by_coords(coords, loc)
def add_cpt(self, cpt, x, **kwargs):
offset = kwargs.get("offset", None)
off_dir = kwargs.get("off_dir", "-")
esp = kwargs.get("esp", None)
loc = Loc(cpt=cpt,
name=cpt.file_name,
offset=offset,
off_dir=off_dir,
esp=esp)
return self.add_loc(x, loc)
def get_cpt_names(self):
_cpts = []
for x in self.locs:
_cpts.append(self.locs[x].cpt_file_name)
return _cpts
def set_ids(self):
for i, loc_name in enumerate(self.locs):
self.locs[loc_name].id = i + 1
if self.locs[loc_name].soil_profile is not None:
self.locs[loc_name].soil_profile.id = i + 1
def to_dict(self, extra=(), **kwargs):
outputs = OrderedDict()
skip_list = ["locs"]
if hasattr(self, "inputs"):
full_inputs = list(self.inputs) + list(extra)
else:
full_inputs = list(extra)
for item in full_inputs:
if item not in skip_list:
value = self.__getattribute__(item)
outputs[item] = sf.collect_serial_value(value)
return outputs
def add_to_dict(self, models_dict, **kwargs):
if self.base_type not in models_dict:
models_dict[self.base_type] = OrderedDict()
outputs = self.to_dict(**kwargs)
models_dict[self.base_type][self.unique_hash] = outputs
for loc_num in self.locs:
self.locs[loc_num].add_to_dict(
models_dict,
parent_dict=models_dict[self.base_type][self.unique_hash])
def reset_cpt_folder_paths(self, folder_path):
for loc_name in self.locs:
self.locs[loc_name].cpt_folder_path = folder_path
@property
def tran_line(self):
try:
from liquepy.spatial.map_coords import Line
return Line(self.s_coords, self.e_coords)
except ImportError as e:
warnings.warn('Need to import spatial packages', stacklevel=3)
warnings.warn(e, stacklevel=3)
return None
@property
def x_end(self):
return self.tran_line.dist
@property
def locs(self):
return self._locs
def add_loc(self, x: float, loc):
loc.x = x
self._locs[x] = loc
self._sort_locs()
return self._locs[x]
def add_loc_by_coords(self, coords, loc):
from liquepy.spatial import map_coords
if not sum(self.start) or not sum(self.end):
raise ValueError("start and end coordinates must be set")
loc.x = map_coords.calc_proj_line_dist(self.tran_line, coords)
loc.offset = map_coords.calc_line_offset(self.tran_line, coords)
loc.off_dir = map_coords.calc_line_off_dir(self.tran_line, coords)
self._locs[loc.x] = loc
self._sort_locs()
return self._locs[loc.x]
@locs.setter
def locs(self, locs):
for loc_id in locs:
loc_dist = locs[loc_id]["x"]
self.locs[loc_dist] = Loc()
sm.add_to_obj(self.locs[loc_dist], locs[loc_id])
def _sort_locs(self):
"""
Sort the locs by distance.
:return:
"""
self._locs = OrderedDict(sorted(self._locs.items(),
key=lambda t: t[0]))
def get_loc_by_name(self, name):
for x in self.locs:
if self.locs[x].name == name:
return self.locs[x]
def get_loc_by_dist(self, dist):
return self.locs[dist]
def loc(self, index):
index = int(index)
if index == 0:
raise KeyError("index=%i, but must be 1 or greater." % index)
return list(self._locs.values())[index - 1]
def remove_loc(self, loc_int):
key = list(self._locs.keys())[loc_int - 1]
del self._locs[key]
def replace_loc(self, loc_int, soil):
key = list(self._locs.keys())[loc_int - 1]
self._locs[key] = soil
@property
def start(self):
return self.s_coords.as_tuple
@property
def end(self):
return self.e_coords.as_tuple
@start.setter
def start(self, values):
self.s_coords = Coords(lat=values[0], lon=values[1])
@end.setter
def end(self, values):
self.e_coords = Coords(lat=values[0], lon=values[1])
def build_experience_buffer(grid, Vx_rzns, Vy_rzns, paths, sampling_interval,
num_of_paths, num_actions):
exp_buffer_all_trajs = []
for k in range(num_of_paths):
exp_buffer_kth_traj = []
Vxt = Vx_rzns[k, :, :]
Vyt = Vy_rzns[k, :, :]
trajectory = paths[0, k]
# append starting point to traj
coord_traj = [(trajectory[0][0], trajectory[0][1])]
s_i, s_j = compute_cell(grid, trajectory[0])
state_traj = [(s_i, s_j)]
# make dictionary states mapping to coords. and choose middle coord to append to traj
traj_dict = OrderedDict()
for j in range(0, len(trajectory)):
s_i, s_j = compute_cell(grid, trajectory[j])
s = (s_i, s_j)
c = (trajectory[j][0], trajectory[j][1])
if not traj_dict.get(s):
traj_dict[s] = [c]
else:
traj_dict[s].append(c)
keys = list(traj_dict.keys())
keys.remove(keys[0]) #remove first and last keys (states).
keys.remove(keys[-1]) #They are appended separately
for s in keys:
state_traj.append(s)
l = len(traj_dict[s])
coord_traj.append(traj_dict[s][int(l // 2)])
coord_traj.append((trajectory[-1][0], trajectory[-1][1]))
s_i, s_j = compute_cell(grid, trajectory[-1])
state_traj.append((s_i, s_j))
state_traj.reverse()
coord_traj.reverse()
#build buffer
print("check warning: ", k)
# print("s1, p1, p2, Vxt, Vyt")
for i in range(len(state_traj) - 1):
s1 = state_traj[i + 1]
s2 = state_traj[i]
# t ,m,n=s1
m, n = s1
p1 = coord_traj[i + 1]
p2 = coord_traj[i]
"""COMMENTING THIS STATEMENT BELOW"""
# if (s1[1],s1[2])!=(s2[1],s2[2]):
#vx=Vxt[t,i,j]
vx = Vxt[m, n]
vy = Vyt[m, n]
# print(s1,p1,p2, vx, vy)
a1 = Calculate_action(s1, p1, p2, vx, vy, grid)
r1 = grid.move_exact(a1, vx, vy)
exp_buffer_kth_traj.append([s1, a1, r1, s2])
#append kth-traj-list to master list
exp_buffer_all_trajs.append(exp_buffer_kth_traj)
return exp_buffer_all_trajs