def main():
parser = argparse.ArgumentParser()
parser.add_argument("inputfile",
help="yaml file which should be processed")
# optionales argument
parser.add_argument("-c",
"--convert-to-owl-rdf",
help="convert to owl rdf format",
action="store_true")
parser.add_argument(
"-o",
"--output",
help=
"optional path of output (otherwise it is derived from the input path)"
)
parser.add_argument("-i", "--interactive", help="start interactive shell")
parser.add_argument("-f",
"--force",
help="override security preventions",
action="store_true")
args = parser.parse_args()
if args.convert_to_owl_rdf:
convert_to_owl_rdf(args)
elif args.interactive:
om = ypo.OntologyManager(args.inputfile)
IPS()
else:
print("nothing to do")
def _enable_reproducible_pickle_repr_for_expr(expr):
"""
Convert all attributes which are dicts to OrderedDict. (See motivation above).
Store the original objects for later recovery.
:param expr:
:return: None
"""
all_dicts = _find_dicts_in_obj(expr)
for dictname in all_dicts:
if dictname in blacklisted_dict_names:
continue
thedict = getattr(expr, dictname)
# account for attributes which have been converted earlier
if isinstance(thedict, OrderedDict):
continue
assert isinstance(thedict, dict)
try:
newordereddict = _dict_to_ordered_dict(thedict)
except ValueError:
IPS()
raise SystemExit
try:
setattr(expr, dictname, newordereddict)
except AttributeError as aerr:
pass
else:
replaced_attributes[expr] = (dictname, thedict)
def get_project_READMEmd(marker_a=None, marker_b=None):
"""
Return the content of README.md from the root directory of this project
(optionally return only the text between the two marker-strings)
:return:
"""
basepath = os.path.dirname(os.path.abspath(__file__))
project_root = os.path.dirname(basepath)
fpath = os.path.join(project_root, "README.md")
with open(fpath, "r") as txt_file:
txt = txt_file.read()
if marker_a is None:
assert marker_b is None
return txt
else:
assert marker_b is not None
try:
idx1 = txt.index(marker_a) + len(marker_a)
idx2 = txt.index(marker_b)
except ValueError:
IPS()
return txt
return txt[idx1:idx2]
def test_start_ips(self):
from . import forms
form = forms.SignUpForm()
if 0:
IPS()
else:
pass
def test_start_ips(self):
"""
Pseudo-testcase for interactive experiments with IPython shell.
:return:
"""
# python manage.py test sober.tests.ViewTests.test_start_ips
x = 0
if x:
# noinspection PyUnusedLocal
IPS()
def smith_step(A, t, var):
# erste Spalte (Index: j), die nicht komplett 0 ist
# j soll größer als Schrittzähler sein
nr, nc = A.shape
row_op_list = []
cols = st.col_split(A)
# erste nicht verschwindende Spalte finden
for j, c in enumerate(cols):
if j < t:
continue
if not c == c*0:
break
# Eintrag mit Index t soll ungleich 0 sein, ggf. Zeilen tauschen
if c[t] == 0:
i, elt = first_nonzero_element(c)
ro = row_swap(nr, t, i)
c = ro*c
row_op_list.append(ro)
col = c.expand()
while True:
new_col, L0 = smith_column_step(col, t, var)
if L0 == sp.eye(nr):
# nothing has changed
break
row_op_list.append(L0)
col = new_col
# jetzt teilt col[t] alle Einträge in col
# Probe in der nächsten Schleife
col.simplify()
col = col.expand()
for i,a in enumerate(col):
if i == t:
continue
if not sp.simplify(sp.gcd(a, col[t]) - col[t]) == 0:
IPS()
raise ValueError, "col[t] should divide all entries in col"
quotient = sp.simplify(a/col[t])
if a == 0:
continue
# eliminiere a
ro = row_op(nr, i, t, -1, quotient)
row_op_list.append(ro)
return row_op_list
def feedback_factory(vf_f, vf_g, xx, clcp_coeffs):
n = len(xx)
assert len(clcp_coeffs) == n + 1
assert len(vf_f) == n
assert len(vf_g) == n
assert clcp_coeffs[-1] == 1
# prevent datatype problems:
clcp_coeffs = st.to_np(clcp_coeffs)
# calculate the relevant covector_fields
# 1. extended nonlinear controllability matrix
Qext = st.nl_cont_matrix(vf_f, vf_g, xx, n_extra_cols=0)
QQinv = Qext.inverse_ADJ()
w_i = QQinv[-1, :]
omega_symb_list = [w_i]
t0 = time.time()
for i in range(1, n + 1):
w_i = st.lie_deriv_covf(w_i, vf_f, xx)
omega_symb_list.append(w_i)
print(i, t0 - time.time())
# dieser schritt dauert ca. 1 min
# ggf. sinnvoll: Konvertierung in c-code
# stack all 1-forms together
omega_matrix = sp.Matrix(omega_symb_list)
IPS()
omega_matrix_func = sp2c.convert_to_c(xx,
omega_matrix,
cfilepath="omega.c")
# noinspection PyPep8Naming
def feedback(xx_ref):
omega_matrix = omega_matrix_func(*xx_ref)
# iterate row-wise over the matrix
feedback_gain = st.to_np(
sum([rho_i * w for (rho_i, w) in zip(clcp_coeffs, omega_matrix)]))
return feedback_gain
# now return that fabricated function
return feedback
def func2(q1, q2):
"""
to demonstrate debugging on exception
"""
a = q1 / q2
if q2 == 5:
z = 100
IPS() # start embedded ipython shell in the local scope
# -> explore global namespace
return a
return res
jdc = JS_Diagram_Converter()
def convert(fname, source):
jdc.convert_diagram(source)
d_path = os.path.join(download_path, "diagram.svg")
dst_path = os.path.join(".", fname.replace(".md", ".svg"))
for i in range(10):
time.sleep(1)
try:
shutil.move(d_path, dst_path)
except FileNotFoundError:
continue
else:
break
res = list(get_all_src().items())
# res = list(get_all_src().items()); convert(*res[0])
IPS()
jdc.quit()
def leven(self):
"""
This method is an implementation of the Levenberg-Marquardt-Method
to solve nonlinear least squares problems.
For more information see: :ref:`levenberg_marquardt`
"""
i = 0
x = self.x0 ##:: guess_value
eye = scp.sparse.identity(len(
self.x0)) ##:: diagonal matrix, value: 1.0, danwei
# this is interesting for debugging:
n_spln_prts = self.masterobject.eqs.trajectories.n_parts_x
self.mu = 1e-4
# borders for convergence-control
b0 = 0.2
b1 = 0.8
rho = 0.0
reltol = self.reltol
# set self.W and its inverse
self.set_weights()
# Winv = scp.sparse.csr_matrix(np.diag(1.0/np.diag(self.W.toarray())))
Fx = self.W.dot(self.F(x))
# for particle swarm approach (dbg, obsolete)
def nF(z):
return norm(self.F(z))
# measure the time for the LM-Algorithm
T_start = time.time()
break_outer_loop = False
while not break_outer_loop:
i += 1
DFx = self.W.dot(self.DF(x))
DFx = scp.sparse.csr_matrix(DFx)
if np.any(np.isnan(DFx.toarray())):
msg = "Invalid start guess (leads to nan in Jacobian)"
self.log_warn(msg)
raise NanError(msg)
break_inner_loop = False
count_inner = 0
while not break_inner_loop:
#: left side of equation, J'J+mu^2*I, Matrix.T=inv(Matrix)
A = DFx.T.dot(DFx) + self.mu**2 * eye
#: right side of equation, J'f, (f=Fx)
b = DFx.T.dot(Fx)
s = -scp.sparse.linalg.spsolve(A, b) #: h
# !! dbg / investigation code
if 0:
C = self.F(x, info=True)
n_states, n_points = C.X.shape
if self.masterobject.dyn_sys.n_pconstraints == 1:
dX = np.row_stack(
(C.dX.reshape(-1, n_states).T, [0] * n_points))
ff = Fx[:-n_points].reshape(-1, n_states).T
else:
dX = C.dX.reshape(-1, n_states).T
ff = Fx.reshape(-1, n_states).T
i = 0
r = C.ff(C.X[:, i:i + 1], C.U[:, i:i + 1],
C.P[:, i:i + 1]) - dX[:, i:i + 1]
# drop penalty values
plt.plot(abs(ff.T))
plt.title(
u"Fehler der refsol-Startschätzung: in Randbereichen am stärksten"
)
# Fazit: ggf die Veränderung der Parameter stärker wichten, wo die Fehler groß sind
# plt.figure()
# plt.plot(s)
plt.show()
IPS()
ll = zip(abs(s), self.masterobject.eqs.all_free_parameters)
ll.sort()
# sehen, welche Parmeter sich wie stark verändern...
# IPS()
# Note Fx is organized as follows: all penalty values are at the end
xs = x + np.array(s).flatten()
Fxs = self.W.dot(self.F(xs))
if any(np.isnan(Fxs)):
# this might be caused by too small mu
msg = "Invalid start guess (leads to nan in Function)"
self.log_warn(msg)
raise NanError(msg)
normFx = norm(Fx)
normFxs = norm(Fxs)
R1 = (normFx - normFxs)
R2 = (normFx - (norm(Fx + DFx.dot(s))))
rho = R1 / R2
# Note: bigger mu means less progress but
# "more regular" conditions
if R1 < 0 or R2 < 0:
# the step was too big -> residuum would be increasing
self.mu *= 2
rho = 0.0 # ensure another iteration
# self.debug("increasing res. R1=%f, R2=%f, dismiss solution" % (R1, R2))
elif (rho <= b0):
self.mu *= 2
elif (rho >= b1):
self.mu *= 0.5
# -> if b0 < rho < b1 : leave mu unchanged
self.log_debug(" rho= %f mu= %f, |s|^2=%f" %
(rho, self.mu, norm(s)))
if np.isnan(rho):
# this should might be caused by large values for xs
# but it should have been catched above
self.log_warn("rho = nan (should not happen)")
# IPS()
raise NanError()
if rho < 0:
self.log_warn("rho < 0 (should not happen)")
if interfaceserver.has_message(
interfaceserver.messages.lmshell_inner):
self.log_debug("lm: inner loop shell")
IPS()
locally_stuck_flag = False
if self.mu > 100:
if abs(R1 / normFx) < reltol:
locally_stuck_flag = True
else:
# unexpexted situation
IPS()
# if the system more or less behaves linearly
break_inner_loop = rho > b0 or locally_stuck_flag
count_inner += 1
Fx = Fxs # F(x+h) -> Fx_new
x = xs # x+h -> x_new
# store for possible future usage
self.x0 = xs
# rho = 0.0
self.res_old = self.res
self.res = normFx
# save value for graphics etc
self.res_list.append(self.res)
self.mu_list.append(self.mu)
self.ntries_list.append(count_inner)
if i > 1 and self.res > self.res_old:
self.log_warn("res_old > res (should not happen)")
spaces = " " * 20
msg = "sp=%d LM_it=%d k=%f %s res=%f"
self.log_debug(msg % (n_spln_prts, i, xs[-1], spaces, self.res))
self.cond_abs_tol = self.res <= self.tol
if self.res > 1:
self.cond_rel_tol = abs(self.res -
self.res_old) / self.res <= reltol
else:
self.cond_rel_tol = abs(self.res - self.res_old) <= reltol
self.cond_num_steps = i >= self.maxIt
if interfaceserver.has_message(
interfaceserver.messages.lmshell_outer):
self.log_debug("lm: outer loop shell")
mo = self.masterobject
sx1 = mo.eqs.trajectories.splines['x1']
IPS()
if interfaceserver.has_message(interfaceserver.messages.run_ivp):
self.cond_external_interrupt = True
if interfaceserver.has_message(
interfaceserver.messages.plot_reslist):
plt.plot(self.res_list)
plt.ylim(min(self.res_list), np.percentile(self.res_list, 80))
# plt.figure()
# plt.plot(self.ntries_list)
plt.show()
if interfaceserver.has_message(interfaceserver.messages.change_w):
self.log_info("start lm again with chaged weights")
self.set_weights("random")
return self.leven()
if interfaceserver.has_message(interfaceserver.messages.change_x):
self.log_debug("lm: change x")
dx = (np.random.rand(len(x)) * 0.5 - 1) * 0.1 * np.abs(x)
x2 = x + dx
self.log_debug("lm: alternative value: %s" % norm(self.F(x2)))
self.x0 = x2
self.log_info("start lm again")
return self.leven()
# IPS()
break_outer_loop = self.cond_abs_tol or self.cond_rel_tol \
or self.cond_num_steps or self.cond_external_interrupt
self.log_break_reasons(break_outer_loop)
if break_outer_loop:
pass
# IPS()
# LM Algorithm finished
T_LM = time.time() - T_start
self.avg_LM_time = T_LM / i
# Note: if self.cond_num_steps == True, the LM-Algorithm was stopped
# due to maximum number of iterations
# -> it might be worth to continue
self.sol = x
def load_ontology(self, startdir,
entity_list: List[models.GenericEntity]) -> None:
"""
load the yml file of the ontology and create instances based on entity_list
:param startdir:
:param entity_list: list of ackrep entities
:return:
"""
if isinstance(self.OM, ypo.OntologyManager):
# Nothing to do
return
assert len(models.ProblemSpecification.objects.all()
) > 0, "no ProblemSpecification found"
assert len(entity_list) > 0, "empty entity_list"
path = os.path.join(startdir, "ontology", "ocse-prototype-01.owl.yml")
self.OM = ypo.OntologyManager(path, world=ypo.owl2.World())
mapping = {}
for cls in self.OM.n.ACKREP_Entity.subclasses():
mapping[cls.name.replace("ACKREP_", "")] = cls
for e in entity_list:
cls = mapping.get(type(e).__name__)
if cls:
# instantiation of owlready classes has side effects -> instances are tracked
# noinspection PyUnusedLocal
instance = cls(has_entity_key=e.key, name=e.name)
tag_list = util.smart_parse(e.tag_list)
assert isinstance(
tag_list,
list), f"unexpexted type of e.tag_list: {type(tag_list)}"
for tag in tag_list:
if tag.startswith("ocse:"):
ocse_concept_name = tag.replace("ocse:", "")
# see yamlpyowl doc (README) wrt proxy_individuals
proxy_individual_name = f"i{ocse_concept_name}"
res = self.OM.onto.search(
iri=f"*{proxy_individual_name}")
if not len(res) == 1:
msg = f"Unknown tag: {tag}. Maybe a spelling error?"
raise NameError(msg)
proxy_individual = res[0]
instance.has_ontology_based_tag.append(
proxy_individual)
# IPS(e.key == "M4PDA")
else:
print("unknown entity type:", e)
if len(list(self.OM.n.ACKREP_ProblemSolution.instances())) == 0:
msg = "Instances of ACKREP_ProblemSolution are missing. This is unexpected."
IPS()
raise ValueError(msg)
for ocse_entity in self.OM.n.OCSE_Entity.instances():
self.ocse_entity_mapping[ocse_entity.name] = ocse_entity
self.generate_bottom_up_tag_relations()
optimizer = torch.optim.Adam(net.parameters(), lr=0.01)
loss_func = nn.MSELoss()
# do the training
loss_over_epoch = []
Nepochs = 2000
for t in range(Nepochs):
if t % int(Nepochs/100) == 0:
print(t)
prediction = net(X_train) # input x and predict based on x
IPS()
loss = loss_func(prediction, Y_train) # must be (1. nn output, 2. target)
loss_over_epoch.append(loss.data.numpy())
optimizer.zero_grad() # clear gradients for next train
loss.backward() # backpropagation, compute gradients
optimizer.step() # apply gradients
plt.figure()
plt.semilogy(loss_over_epoch)
# plt.ioff()
plt.show()
# -> explore global namespace
return a
# ST() # start tracicing start command line debugger here
# presse h <Enter> for help
# n <Enter> to execute next line
# q <Enter> to quit
# create some objects to play around with
x = [0, 1, 2, 3, 4]
y = sin(3.14)
z = func1(0) - 5
s = "teststring"
b1 = func2(1.0, 2.0) # nothing happens
#b2 = func2(1.0, 0) # ZeroDivisionError -> start interactive debugger
IPS() # start embedded ipython shell on top level scope
# -> explore global namespace (e.g. type s.<TAB> or s.lower?)
# type CTRL-D to exit
b2 = func2(1.0, 5) # start IPS inside func2
# not the difference e.g. the local value of z
def main():
argparser = argparse.ArgumentParser()
argparser.add_argument("--key",
help="print a random key and exit",
action="store_true")
argparser.add_argument("-cs",
"--check-solution",
metavar="metadatafile",
help="check solution (specified by metadata file)")
argparser.add_argument("--check-all-solutions",
help="check all solutions (may take some time)",
action="store_true")
argparser.add_argument("-n",
"--new",
help="interactively create new entity",
action="store_true")
argparser.add_argument("-l",
"--load-repo-to-db",
help="load repo to database",
metavar="path")
argparser.add_argument("-e",
"--extend",
help="extend database with repo",
metavar="path")
argparser.add_argument(
"--qq",
help="create new metada.yml based on interactive questionnaire",
action="store_true")
# for development only
argparser.add_argument(
"--dd",
help="start interactive IPython shell for debugging",
action="store_true")
argparser.add_argument("--md",
help="shortcut for `-m metadata.yml`",
action="store_true")
argparser.add_argument(
"-m",
"--metadata",
help="process metadata in yaml syntax (.yml file). ")
args = argparser.parse_args()
if args.new:
create_new_entity()
elif args.dd:
IPS()
elif args.load_repo_to_db:
startdir = args.load_repo_to_db
core.load_repo_to_db(startdir)
print(bgreen("Done"))
elif args.extend:
startdir = args.extend
core.extend_db(startdir)
print(bgreen("Done"))
elif args.qq:
entity = dialoge_entity_type()
field_values = dialoge_field_values(entity)
core.convert_dict_to_yaml(field_values, target_path="./metadata.yml")
return
elif args.check_solution:
metadatapath = args.check_solution
check_solution(metadatapath)
elif args.check_all_solutions:
check_all_solutions()
elif args.metadata or args.md:
if args.md:
args.metadata = "metadata.yml"
data = core.get_metadata_from_file(args.metadata)
print(f"\n {bgreen('content of '+args.metadata)}\n")
pprint.pprint(data, indent=1)
print("")
return
elif args.key:
print("Random entity-key: ", core.gen_random_entity_key())
return
else:
print("This is the ackrep_core command line tool\n")
argparser.print_help()
# debug: where does the contradiction occur:
for i, (restr, indiv) in enumerate(restriction_tuples):
print(f"\n\n{i+2}\n")
om.add_restriction_to_entity(restr, indiv)
if debug:
om.sync_reasoner(infer_property_values=True)
om.sync_reasoner(infer_property_values=True)
# this should finally run:
task_complete = True
if task_complete:
om.sync_reasoner(infer_property_values=True)
assert n.Spaniard.owns == n.dog
assert n.Englishman.owns == n.snails
assert n.Japanese.owns == n.zebra
assert n.Norwegian.owns == n.fox
assert n.Ukrainian.owns == n.horse
assert n.Norwegian.lives_in == n.house_1
assert n.Englishman.lives_in == n.house_3
assert n.Japanese.lives_in == n.house_5
assert n.Ukrainian.lives_in == n.house_2
assert n.Spaniard.lives_in == n.house_4
IPS(
) # start interactive shell in namespace (should be run in a terminal window, not inside pycharm, spyder, ...)
def chdir(self,
path,
target_spec: Literal["remote", "local", "both"] = "both",
tolerate_error=False):
"""
The following works on uberspace:
c.chdir("etc")
c.chdir("~")
c.chdir("$HOME")
:param path:
:param target_spec:
:param tolerate_error:
:return:
"""
if path is None:
self.dir = None
return
assert len(path) > 0
# handle relative paths
if path[0] not in ("/", "~", "$"):
# path is a relative directory
if self.dir is None:
# this should prevent too hazardous invocations
msg = "Relative path cannot be the first path specification"
raise ValueError(msg)
pwd_res = self.run("pwd",
hide=True,
warn=True,
target_spec=target_spec)
assert pwd_res.exited == 0
abs_path = f"{pwd_res.stdout.strip()}/{path}"
else:
# !! handle the cases of $RELATIVE_PATH and $UNDEFINED (however, not so important)
abs_path = path
old_path = self.dir
self.dir = abs_path
cmd = "pwd"
res = self.run(cmd, hide=True, warn=True, target_spec=target_spec)
pwd_txt = res.stdout.strip()
if res.exited != 0:
print(
bred(f"Could not change directory. Error message: {res.stderr}"
))
self.dir = old_path
# assure they have the last component in common
# the rest might differ due to symlinks and relative paths
elif not pwd_txt.endswith(os.path.split(path)[1]):
if not tolerate_error and not path.startswith(
"~") and not path.startswith("$"):
print(
bred(
f"Could not change directory. `pwd`-result: {res.stdout}"
))
IPS(print_tb=-1)
self.dir = old_path
res = EContainer(exited=1, old_res=res)
return res
