def getTests(cache, qname):
"""
Expand the tests of this node.
"""
INFO("sys.getTests(%s)" % (qname))
return generic.getRelated(cache,
qname,
"sys:hasProperty",
restriction="dev:Test")
def show_variable(node, args=None):
"""
Show the 'variable' view of the 'soft' category.
"""
INFO("soft.show_variable(%s)" % node['qname'])
node["type"] = None
node["points_to_type"] = None
node["initial_value"] = None
node["address"] = None
node["value"] = None
results = QUERY("""
SELECT DISTINCT ?type ?pointsToType ?initialValue ?address ?link ?value
WHERE {
OPTIONAL { %s soft:hasType | (^soft:isTypeOf) ?type . } .
OPTIONAL { %s soft:pointsToType ?pointsToType .} .
OPTIONAL { %s soft:hasInitialValue ?initialValue . } .
OPTIONAL { %s soft:hasAddress ?address . } .
OPTIONAL { %s expr:hasValue | expr:hasNumericValue ?value . } .
}
""" % ((node["qname"], ) * 5))
for type, pointsToType, initialValue, address, link, value in results:
if type is not None:
typeQName = URI_TO_QNAME(type.toPython())
node["type"] = typeQName
generic.getDefaultNode(node.cache, typeQName).show("soft", "type")
if pointsToType is not None:
pointsToTypeQName = URI_TO_QNAME(pointsToType.toPython())
node["points_to_type"] = pointsToTypeQName
generic.getDefaultNode(node.cache,
pointsToTypeQName).show("soft", "type")
if initialValue is not None:
node["initial_value"] = initialValue.toPython()
if address is not None:
node["address"] = address.toPython()
if value is not None:
node["value"] = getExpressionString(value)
node.expand("soft", "variable")
for memberOf in node["member_of"]:
node.cache[memberOf].show("soft")
for l in node["links"]:
node.cache[l].show("soft")
for qualifier in node["qualifiers"]:
node.cache[qualifier].show("soft")
def show_struct(node, args=None):
"""
Show the 'struct' view of the 'soft' category.
"""
INFO("soft.show_struct(%s)" % node['qname'])
show_type(node)
node.expand("soft", "struct")
for attr in node["attributes"]:
node.cache[attr].show("soft")
def getLocalVariables(cache, qname):
"""
Expand the IEC 61131-3 local variables of a node.
"""
INFO("soft.getLocalVariables(%s)" % (qname))
return generic.getRelated(
cache,
qname,
"iec61131:hasLocalVariable",
remove="(soft:extends+)/iec61131:hasLocalVariable")
def fillNumber(node, optional=False):
"""
Fill out the number of a container item (according to the containers ontology).
"""
INFO(" Fill number of %s")
node["number"] = None
results = QUERY("""
SELECT DISTINCT ?number
WHERE {
%s (cont:isItemOf|(^cont:hasItem))/cont:hasNumber ?number .
}
""" %node["qname"])
for (number,) in results:
node["number"] = int(number.toPython())
INFO(" --> %s" %node["number"])
def show_enum(node, args=None):
"""
Show the 'enum' view of the 'soft' category.
"""
INFO("soft.show_enum(%s)" % node['qname'])
show_type(node)
node.expand("soft", "enum")
for ns in node["items"]:
node.cache[ns].show("soft", "enum_item")
def getMethods(cache, qname):
"""
Expand the IEC 61131-3 methods of a node.
"""
INFO("soft.getMethods(%s)" % (qname))
return generic.getRelated(
cache,
qname,
"iec61131:hasMethod | iec61131:hasMethodInstance",
remove=
"(soft:extends+)/(iec61131:hasMethod | iec61131:hasMethodInstance)")
def show_test(node, args=None):
"""
Show the 'test' view of the 'sys' category.
"""
INFO("sys.show_test(%s)" % node['qname'])
node.expand("sys", "test")
for verifies in node["verifies"]:
node.cache[verifies].show()
for t in node["tests"]:
node.cache[t].show()
def show_concept(node, args=None):
"""
Show the 'concept' view of the 'sys' category.
"""
INFO("sys.show_concept(%s)" % node['qname'])
node.expand("sys", "concept")
for expansion in [
"requirements", "states", "properties", "constraints", "tests",
"designs"
]:
for qname in node[expansion]:
node.cache[qname].show("sys")
def RunEpisode(self, max_steps = None):
if max_steps == None:
max_steps = RLGame.DEFAULT_NUM_STEPS_PER_EP
self._history.clear()
# create flag to keep track of whether all agents are terminal
all_agents_terminal = True
step = 0
while step < max_steps:
# Iterate through all agents in the game
for id, agent in self._agents.items():
# if the current agent is terminal, move to next agent
if self._IsTerminal(agent):
self._history[id].Push(agent.GetCurrState(), None, None)
continue
# Otherwise, call StepAgent on the agent
self._StepAgent(agent)
# Update all_agents_terminal flag
all_agents_terminal = (all_agents_terminal and self._IsTerminal(agent))
step += 1
# if all agents are terminal, break early
if all_agents_terminal:
INFO(f"Ending episode early at step {step}, all agents are at terminal state")
self._episodes.append(self._history)
return (step, self.GetLatestEpisodeHistory())
INFO(f"Ran for specified number of max steps, at least some agents are still not at terminal state")
self._episodes.append(self._history)
return (step, self.GetLatestEpisodeHistory())
def show_fb(node, args=None):
"""
Show the 'fb' (function block) view of the 'soft' category.
"""
INFO("soft.show_fb(%s)" % node['qname'])
show_type(node)
node.expand("soft", "fb")
if node["implementation"] is not None:
node.cache[node["implementation"]].show("soft")
for kind in ["var_in", "var_out", "var_inout", "var_local", "methods"]:
for qname in node[kind]:
node.cache[qname].show("soft")
def show_namespace(node, args=None):
"""
Show the 'namespace' view of the 'soft' category.
"""
INFO("soft.show_namespace(%s)" % node['qname'])
node.expand("soft", "namespace")
for ns in node["namespaces"]:
node.cache[ns].show("soft")
for fb in node["FBs"]:
node.cache[fb].show("soft")
for struct in node["STRUCTs"]:
node.cache[struct].show("soft")
for enum in node["ENUMs"]:
node.cache[enum].show("soft")
def show_requirement(node, args=None):
"""
Show the 'requirement' view of the 'sys' category.
"""
INFO("sys.show_requirement(%s)" % node['qname'])
node.expand("sys", "requirement")
for derived in node["derives"]:
node.cache[derived].show()
for derivedFrom in node["derived_from"]:
node.cache[derivedFrom].show()
for d in node["declared_by"]:
node.cache[d].show("sys")
for s in node["satisfied_by"]:
node.cache[s].show("sys")
def getCommonVariablesOfContext(variableUri, contextUri):
"""
Get the common variables of a given variable and a given context.
"""
INFO("soft.getCommonVariablesOfContext(%s,%s)" % (variableUri, contextUri))
ret = []
results = QUERY("""
SELECT DISTINCT ?member
WHERE {
<%s> soft:hasVariable ?member .
?member (soft:hasVariable)* <%s> .
}
""" % (contextUri, variableUri))
for (member, ) in results:
ret.append(member.toPython())
return ret
def getExpressionString(ex):
"""
Get a string serialization of the given expression.
"""
INFO("soft.getExpressionString(%s)" % (ex))
if IS_LITERAL(ex):
v = ex.toPython()
if isinstance(v, str) or isinstance(v, unicode):
return "'%s'" % v
elif str(v).lower() == "false":
return "FALSE"
elif str(v).lower() == "true":
return "TRUE"
else:
return str(v)
else:
return str(ex)
def show_operator(node, args=None):
"""
Show the 'operator' view of the 'soft' category.
"""
INFO("soft.show_operator(%s)" % node['qname'])
results = QUERY("""
SELECT DISTINCT ?symbol
WHERE {
OPTIONAL { %s iec61131:hasSymbol ?symbol . } .
}
""" % ((node["qname"], ) * 1))
node["plc_symbol"] = None
for (symbol, ) in results:
if symbol is not None:
node["plc_symbol"] = symbol.toPython()
def show_primitive(node, args=None):
"""
Show the 'primitive' view of the 'soft' category.
"""
INFO("soft.show_InterfaceInstance(%s)" % node['qname'])
# extra query to get the string value or numeric value
results = QUERY("""
SELECT DISTINCT ?value
WHERE {
%s expr:hasValue|expr:hasNumericValue ?value .
}
""" % node["qname"])
node["value"] = None
for (value, ) in results:
if value is not None:
node["value"] = getExpressionString(value)
def getImplementationExpressions(cache, qname):
"""
Expand the implementation expressions of a node.
"""
INFO("soft.getImplementationExpressions(%s)" % (qname))
# unsorted list:
itemQNames = generic.getRelated(cache,
subject=qname,
property="cont:contains",
sortedByNumber=True)
for itemQName in itemQNames:
itemNode = cache[itemQName]
itemNode.show(
"soft"
) # the view priority (defined by allviews.py) will determine which show_ function will be called!
# return the sorted list of expressions:
return itemQNames
def show_call(node, args=None):
"""
Show the 'call' view of the 'soft' category.
"""
INFO("soft.show_call(%s)" % node['qname'])
node.expand("soft", "call")
calls = generic.getRelated(node.cache, node["qname"], "soft:calls")
if len(calls) != 1:
raise Exception(
"Error in show_call: expected one result for '%s soft:calls ...' but we received: %s"
% (node["qname"], calls))
node["calls"] = calls[0]
node.cache[calls[0]].show("soft")
for ass in node["assignments"]:
node.cache[ass].show("soft")
def show_type(node, args=None):
"""
Show the 'type' view of the 'soft' category.
"""
INFO("soft.show_type(%s)" % node['qname'])
results = QUERY("""
SELECT DISTINCT ?implementation ?symbol ?extends ?returnType
WHERE {
OPTIONAL { %s soft:hasImplementation ?implementation . } .
OPTIONAL { ?implementation soft:isImplementationOf %s . } .
OPTIONAL { %s (^owl:sameAs)/iec61131:hasSymbol | iec61131:hasSymbol ?symbol . } .
OPTIONAL { %s soft:extends ?extends } .
OPTIONAL { %s soft:hasReturnType ?returnType . } .
}
""" % ((node["qname"], ) * 5))
node["implementation"] = None
node["plc_symbol"] = None
node["extends"] = None
node["returnType"] = None
for implementation, symbol, extends, returnType in results:
if implementation is not None:
implementationQName = URI_TO_QNAME(implementation.toPython())
node["implementation"] = implementationQName
generic.getDefaultNode(node.cache, implementationQName)
# don't show it here
if symbol is not None:
node["plc_symbol"] = symbol.toPython()
if extends is not None:
extendsQName = URI_TO_QNAME(extends.toPython())
node["extends"] = extendsQName
generic.getDefaultNode(node.cache, extendsQName).show("soft")
if returnType is not None:
returnTypeQName = URI_TO_QNAME(returnType.toPython())
node["returnType"] = returnTypeQName
generic.getDefaultNode(node.cache, returnTypeQName).show("soft")
def getEnumPath(variableUri):
"""
Get the path to an enumeration.
"""
INFO("soft.getEnumPath(%s)" % (variableUri))
results = QUERY("""
SELECT DISTINCT ?enum ?label
WHERE {
?enum soft:hasEnumerationItem <%s> .
?enum rdfs:label ?label .
}
""" % (variableUri))
if len(results) == 0:
return None
elif len(results) > 1:
raise Exception("Multiple enum definitions were found for item %s!" %
variableUri)
else:
for (enumUri, enumLabel) in results:
return {"path": [enumLabel.toPython()]}
def show_plc_method(node, args=None):
"""
Show the 'plc_method' view of the 'soft' category.
"""
INFO("soft.show_plc_method(%s)" % node['qname'])
show_type(node)
node.expand("soft", "plc_method")
node["this"] = generic.getRelated(
node.cache, node["qname"],
"(^iec61131:hasMethod) | (^iec61131:hasMethodInstance)")[0]
node.cache[node["this"]].show("soft")
if node["implementation"] is not None:
node.cache[node["implementation"]].show("soft")
for memberOf in node["member_of"]:
node.cache[memberOf].show("soft")
for kind in ["var_in", "var_out", "var_inout", "var_local"]:
for qname in node[kind]:
node.cache[qname].show("soft")
def show_if_then(node, args=None):
"""
Show the 'if_then' view of the 'soft' category.
"""
INFO("soft.show_if_then(%s)" % node['qname'])
results = QUERY("""
SELECT DISTINCT ?if ?then ?else
WHERE {
%s soft:if ?if .
%s soft:then ?then .
OPTIONAL { %s soft:else ?else . }
}
""" % ((node["qname"], ) * 3))
if len(results) != 1:
raise Exception("Invalid specification of IfThen operation %s" %
node["qname"])
node["if"] = None
node["then"] = None
node["else"] = None
for (i, t, e) in results:
iNode = generic.getDefaultNode(node.cache, URI_TO_QNAME(i))
tNode = generic.getDefaultNode(node.cache, URI_TO_QNAME(t))
node["if"] = iNode["qname"]
node["then"] = tNode["qname"]
iNode.show("soft")
tNode.show("soft")
if e is not None:
eNode = generic.getDefaultNode(node.cache, URI_TO_QNAME(e))
node["else"] = eNode["qname"]
eNode.show("soft")
def show_qualifier(node, args=None):
"""
Show the 'qualifier' view of the 'soft' category.
"""
INFO("soft.show_qualifier(%s)" % node['qname'])
results = QUERY("""
SELECT DISTINCT ?symbol ?value
WHERE {
OPTIONAL { %s iec61131:hasSymbol ?symbol . } .
OPTIONAL { %s expr:hasValue ?value . } .
}
""" % ((node["qname"], ) * 2))
node["plc_symbol"] = None
node["value"] = None
for symbol, value in results:
if symbol is not None:
node["plc_symbol"] = symbol.toPython()
if value is not None:
node["value"] = value.toPython()
def getInstances(cache, className, filterNotExists=None):
"""
Get the instances of a class.
"""
INFO(" Get instances of %s" %className)
if filterNotExists is None:
filterNotExistsLine = ""
else:
filterNotExistsLine = "FILTER NOT EXISTS { %s }" %filterNotExists
results = QUERY("""
SELECT DISTINCT ?instance ?label ?comment ?counter ?rdfClass
WHERE {
?instance rdf:type/rdfs:subClassOf* %s .
OPTIONAL { ?instance rdfs:label ?label } .
OPTIONAL { ?instance rdfs:comment ?comment } .
OPTIONAL { ?instance ontoscript:counter ?counter } .
OPTIONAL { ?instance a/(rdfs:subClassOf*) ?rdfClass . FILTER (!isBlank(?rdfClass)) } .
%s
}
""" %(className, filterNotExistsLine))
d = {}
for uri, label, comment, counter, rdfClass in results:
qname = URI_TO_QNAME(uri)
if not d.has_key(qname):
d[qname] = Node(
qname = qname,
uri = uri.toPython(),
cache = cache)
if label is not None:
d[qname]["label"] = label.toPython()
if comment is not None:
d[qname]["comment"] = comment.toPython()
if counter is not None:
d[qname]["counter"] = int(counter.toPython())
if rdfClass is not None:
d[qname].registerClass(URI_TO_QNAME(rdfClass.toPython()))
keysStr = ""
for key in d.keys():
keysStr += (key + " ")
INFO(" --> " + keysStr)
for qname, node in d.items():
node.registerKnownViews()
if not cache.has_key(qname):
DEBUG("Caching %s" %qname)
cache[qname] = node
# return a list of QNames
ret = [] # list of qnames
resultNodes = sorted(d.values(), key=lambda x: x["counter"])
for resultNode in resultNodes:
ret.append(resultNode['qname'])
return ret
def create_folder(path, erase=False, quiet=False):
"""Create folder at @path.
- @erase - erase existing folder
- @quiet - don't ask user about particular actions
- if @quiet is False, new folder with name @path[i]
will be created
- @erase has more priority than @quiet
"""
# >:( a lot of returns - not good style
DEBUG("Creating '%s' folder", path)
try:
os.makedirs(path)
except OSError as ex:
# we can't support other errors, except 'Folder already exists'
if ex.errno != 17:
CRITICAL("Can't create folder %s", path)
EXCEPTION("")
emergency_exit()
else:
DEBUG("Folder '%s' created", path)
return path
# Looks like folder already exists
# lets try to erase it or create new
# at different path
ERROR("Can't create '%s' folder", path)
if erase:
try:
erase_dir(path)
except Exception:
CRITICAL("Folder '%s' can't be erased")
else:
INFO("Folder erased: '{}'".format(path))
return path
# Well, erase == False or folder can't be erased
if not quiet:
answ = ''
while not answ:
answ = raw_input(("Type (E) to erase existing folder, "
"type (Q) to exit the script "
"or enter new folder name: ")).lower()
if answ == "e":
return create_folder(path, erase=True, quiet=quiet)
elif answ == "q":
script_exit()
elif answ:
return create_folder(answ, erase=False, quiet=quiet)
else:
return create_folder(find_unic_path(path), erase=erase, quiet=quiet)
def fillFields(node, mandatories={}, optionals={}):
"""
Fill some mandatory and/or optional fields of a node.
"""
subject = node['qname']
INFO(" Fill these fields of %s: mandatories=%s, optionals=%s" %(subject, mandatories.keys(), optionals.keys()))
selectLine = ""
wherePart = ""
for key,value in mandatories.items():
selectLine += " ?%s" %key
wherePart += "%s %s ?%s .\n" %(subject, value, key)
for key,value in optionals.items():
selectLine += " ?%s" %key
wherePart += "OPTIONAL { %s %s ?%s } .\n" %(subject, value, key)
query = """
SELECT DISTINCT %s
WHERE {
%s
}
""" %(selectLine, wherePart)
results = QUERY(query)
if len(results) == 0:
raise Exception("No results for query:\n%s" %query)
infoStr = " --> mandatories ["
for result in results:
# the mandatories
for i in xrange(len(mandatories)):
key = mandatories.keys()[i]
try:
if IS_URI(result[i]):
node.cache[subject][key] = URI_TO_QNAME(result[i].toPython())
else:
node.cache[subject][key] = result[i].toPython()
except:
node.cache[subject][key] = None
if i > 0: infoStr += ","
infoStr += str(node.cache[subject][key])
infoStr += "], optionals ["
for i in xrange(len(optionals)):
key = optionals.keys()[i]
try:
j = len(mandatories) + i
if IS_URI(result[j]):
node.cache[subject][key] = URI_TO_QNAME(result[j].toPython())
else:
node.cache[subject][key] = result[j].toPython()
except:
node.cache[subject][key] = None
if i > 0: infoStr += ","
infoStr += str(node.cache[subject][key])
infoStr += "]"
INFO(infoStr)
def getRelated(cache, subject, property, restriction=None, remove=None, sortedByNumber=False, filterNotExists=None):
"""
Get the related individuals of an individual.
"""
INFO(" Get related %s of %s" %(property, subject))
extraVariables = ""
if restriction is None:
restrictionLine = ""
else:
restrictionLine = "\n ?result rdf:type/rdfs:subClassOf* %s ." %restriction
if remove is None:
removeLine = ""
else:
removeLine = "\n FILTER NOT EXISTS { %s %s ?result }" %(subject, remove)
if sortedByNumber:
numberLine = "\n OPTIONAL { ?result (cont:isItemOf|(^sys:hasItem))/cont:hasNumber ?number }"
extraVariables += "?number"
else:
numberLine = ""
if filterNotExists is None:
filterNotExistsLine = ""
else:
filterNotExistsLine = "FILTER NOT EXISTS { %s }" %filterNotExists
results = QUERY("""
SELECT DISTINCT ?result ?label ?comment ?counter ?rdfClass %s
WHERE {
%s %s ?result . %s%s%s
OPTIONAL { ?result rdfs:label ?label } .
OPTIONAL { ?result rdfs:comment ?comment } .
OPTIONAL { ?result ontoscript:counter ?counter } .
OPTIONAL { ?result a/(rdfs:subClassOf*) ?rdfClass . FILTER (!isBlank(?rdfClass)) } .
%s
}
""" %(extraVariables, subject, property, restrictionLine, removeLine, numberLine, filterNotExistsLine))
d = {}
for result in results:
resultQName = URI_TO_QNAME(result[0])
if resultQName not in d.keys():
d[resultQName] = Node(uri = result[0].toPython(),
qname = resultQName,
cache = cache)
if result[1] is not None:
d[resultQName]["label"] = result[1].toPython()
if result[2] is not None:
d[resultQName]["comment"] = result[2].toPython()
if result[3] is not None:
d[resultQName]["counter"] = int(result[3].toPython())
if result[4] is not None:
d[resultQName].registerClass(URI_TO_QNAME(result[4].toPython()))
if sortedByNumber:
if result[5] is not None:
d[resultQName]["number"] = int(result[5].toPython())
else:
d[resultQName]["number"] = None
keysStr = ""
for key in d.keys():
keysStr += (key + " ")
INFO(" --> " + keysStr)
for resultQName, resultNode in d.items():
resultNode.registerKnownViews()
if not cache.has_key(resultQName):
cache[resultQName] = d[resultQName]
# return a list of QNames
ret = [] # list of qnames
# first sort by 'counter' key:
resultNodes = sorted(d.values(), key=lambda x: x['counter']) # entries with None will be put first in the sorted list
# then, if necessary, sort by number:
if sortedByNumber:
resultNodes = sorted(resultNodes, key=lambda x: x['number'])
for resultNode in resultNodes:
ret.append(resultNode['qname'])
return ret
sanitizer = LogSanitizer()
sanitizer.sanitize_files(args)
sanitizer.save_secrets(args.out, __version__)
if args.time:
sanitizer.timings()
#===============================================================================
# LINE = "Failed opening persistent JSON file C:\Program Files\SentinelOne\Sentinel Agent 4.3.2.86\config\LocationEngineState.json with error 2"
LINE = "Starting loop sentinel::ResourceMonitor::{ctor}::<lambda_3>::operator ()"
def test():
sanitizer = LogSanitizer()
output = sanitizer.sanitize_line(LINE)
print(output)
#===============================================================================
if __name__ == "__main__":
try:
# test()
main()
except Exception as e:
print_exc()
INFO("Done.")