def createLinko(self, commands, ontology, abstraction, linkograph,
unique_id):
"""createLinko:
This function takes the names of the commands, ontology, and abstraction files to be used
in creating a linkograph, as well as a potential name for the resulting linkograph, and
returns the name of the saved file.
"""
#Come up with a name for the saved linko file
fileName = commands + ontology + abstraction
if (len(linkograph) != 0):
fileName = linkograph
commandsName = commands
#Loading in all of the necessary files
commands = fs.loadFile(commands, 'commands', unique_id)['content']
ontology = fs.loadFile(ontology, 'ontology', unique_id)['content']
abstraction = fs.loadFile(abstraction, 'abstraction',
unique_id)['content']
commands = json.loads(commands)
ontology = json.loads(ontology)
abstraction = json.loads(abstraction)
#Labeling the commands
lr = labels.Labeler(abstraction)
inverseLabeling = lr.labelCommands(commands, defaultLabel="NoLabel")
inverseLabeling = labels.writesLabelsToJsonFile(inverseLabeling)
#Creating and saving the actual linkograph
linkograph = linkoCreate.createLinko(inverseLabeling, ontology)
fileContent = linkoCreate.writesLinkoJson(linkograph)
fs.saveFile(fileName, fileContent, 'linkograph', commandsName,
unique_id)
return fileName
def performTestForParams(self):
""""Performs the tests for each set of parameters."""
for (testNum, params) in enumerate(self.testParams):
actualLinkograph = linkoCreate.createLinko(
params['inverseLabeling'], params['ontology'])
self.assertEqual(actualLinkograph, params['ExpectedLinkograph'],
("testNum = {}"
" inversLabling = {}"
" ontology= {}"
" actualLinkograph = {}"
" ExpectedLinkograph = {}").format(
testNum, params['inverseLabeling'],
params['ontology'], actualLinkograph,
params['ExpectedLinkograph']))
def visualize(lg, o, base_name, iteration):
# derive linkograph using labels from existing linkograph
# @attention is there a better way to extract labeling from a linkograph?
labeling = dict()
for node_index in range(len(lg)):
label_set = lg[node_index][0].copy()
label = label_set.pop()
if not label in labeling:
labeling[label] = [node_index]
else:
labeling[label].append(node_index)
derived_lg = llinkoCreate.createLinko(labeling, o)
# persist image of derived linkograph
svg = llinkoDrawSVG.linkoDrawSVG(derived_lg)
with open(base_name + "_" + str(iteration) + ".svg", "w") as svg_file:
svg_file.write(svg)
def genLinkograph(self, n, ontology=None):
"""Generate a linkograph on n-nodes."""
# If no ontology is passed, use one stored in the model.
if ontology is None:
ontology = self.ontology
# If the model did not have an ontology either, then raise an
# error.
if ontology is None:
raise OntologyError('Model does not have ontology.')
invLabel = self.inverseLabeling(n)
linko = lc.createLinko(invLabel, ontology)
# Set the linkographs labels to ensure same order as the
# abstraction classes used in the model
linko.labels = self.absClasses
return linko
def bulk_transform(method):
for session_filename in sys.argv[1:]:
#########################
# create input linkograph
#########################
label_rules = open("abstraction.json", "r")
labeler = llabels.Labeler(json.load(label_rules))
label_rules.close()
commands = open(session_filename, "r")
json_commands = json.load(commands)
commands.close()
labeled = labeler.labelCommands(json_commands, "NoLabel")
llabels.writeLabelsToJsonFile(labeled, "labeled.json")
ontology_file = open("ontology.json", "r")
inv_labeling_file = open("labeled.json", "r")
lg = llinkoCreate.createLinko(json.load(inv_labeling_file), json.load(ontology_file))
inv_labeling_file.close()
ontology_file.close()
##################################
# transform linkograph to ontology
##################################
if 0 == method:
extracted_ontology = oe.simple_lg_to_ontology(lg)
elif 1 == method:
extracted_ontology = oe.threshold_lg_to_ontology(lg)
else:
print("unknown method:", method)
#########
# cleanup
#########
os.remove("labeled.json")
#########################
# create input linkograph
#########################
label_rules = open("abstraction.json", "r")
labeler = llabels.Labeler(json.load(label_rules))
label_rules.close()
commands = open(session_filename, "r")
json_commands = json.load(commands)
commands.close()
labeled = labeler.labelCommands(json_commands, "NoLabel")
llabels.writeLabelsToJsonFile(labeled, "labeled.json")
ontology_file = open("ontology.json", "r")
inv_labeling_file = open("labeled.json", "r")
labeling = json.load(inv_labeling_file)
lg = llinkoCreate.createLinko(labeling, json.load(ontology_file))
inv_labeling_file.close()
ontology_file.close()
# a linkograph is a list of tuples (nodes)
# a node is a 3-tuple
# component 0 is a set of labels
# component 1 is a set of backlinks
# component 2 is a set of forelinks
##################################
# transform linkograph to ontology
##################################
extracted_ontology = simple_lg_to_ontology(lg)
def processSession(session_filename, writer):
#####################
# generate linkograph
#####################
# label commands
label_rules = open("abstraction.json", "r")
labeler = llabels.Labeler(json.load(label_rules))
label_rules.close()
commands = open(session_filename, "r")
json_commands = json.load(commands)
commands.close()
if 2 > len(json_commands):
print("can't process", session_filename,
"because session files must have at least two commands")
return
last_command = json_commands.pop()
# access_next, look_next, transfer_next, move_next, execute_next, cleanup_next
last_command_labels = labeler.labelCommands([last_command], "NoLabel")
access_next = 0
look_next = 0
transfer_next = 0
move_next = 0
execute_next = 0
cleanup_next = 0
if "Access" in last_command_labels:
access_next = 1
if "Look" in last_command_labels:
look_next = 1
if "Transfer" in last_command_labels:
transfer_next = 1
if "Move" in last_command_labels:
move_next = 1
if "Execute" in last_command_labels:
execute_next = 1
if "Cleanup" in last_command_labels:
cleanup_next = 1
labeled = labeler.labelCommands(json_commands, "NoLabel")
# @todo cleanup labeled.json when its safe
llabels.writeLabelsToJsonFile(labeled, "labeled.json")
# link commands
ontology = open("ontology.json", "r")
inv_labeling = open("labeled.json", "r")
lg = llinkoCreate.createLinko(json.load(inv_labeling), json.load(ontology))
inv_labeling.close()
os.remove("labeled.json")
ontology.close()
##################
# extract features
##################
# node_count
node_count = len(lg)
# critical_node_count
# @todo: pick something real for critical_threshold
critical_threshold = node_count / 2
critical_node_count = lstats.countCriticalNodes(lg, critical_threshold)
# x_bar, Sigma_x, range_x, y_bar, Sigma_y, range_y
x_bar, Sigma_x, range_x, y_bar, Sigma_y, range_y = lstats.calculateCartesianStatistics(
lg)
# percentage_of_links
percentage_of_links = lstats.percentageOfLinks(lg)
# entropy
entropy = lstats.graphEntropy(lg)
# T-Complexity
encoded_lg = lstats.linkographToString(lg)
t_complexity = lstats.tComplexity(encoded_lg)
# link_index
link_index = lstats.links(lg) / len(lg)
# graph differences
graph_differences = lstats.summaryDifference(lg)
# entropy deviation
entropy_deviation = lstats.entropyDeviation(lg)
# mean link coverage
mean_link_coverage = lstats.meanLinkCoverage(lg)
# top cover
top_cover = lstats.topCover(lg)
first_command = json_commands[0]
first_datetime = utils.stringToDatetime(first_command['ts'])
session_start_time = first_datetime.hour * 3600 + first_datetime.minute * 60 + first_datetime.second
# session_length_seconds, mean_delay_seconds
last_command = json_commands[-1]
last_datetime = utils.stringToDatetime(last_command['ts'])
session_length_timedelta = last_datetime - first_datetime
session_length_seconds = session_length_timedelta.total_seconds()
if 1 < len(lg):
mean_delay_seconds = session_length_seconds / (len(lg) - 1)
else:
mean_delay_seconds = None
# access_ratio, look_ratio, transfer_ratio, move_ratio, execute_ratio, cleanup_ratio
access_ratio = look_ratio = transfer_ratio = move_ratio = execute_ratio = cleanup_ratio = 0
if "Access" in labeled.keys():
access_ratio = len(labeled['Access']) / len(lg)
if "Look" in labeled.keys():
look_ratio = len(labeled['Look']) / len(lg)
if "Transfer" in labeled.keys():
transfer_ratio = len(labeled['Transfer']) / len(lg)
if "Move" in labeled.keys():
move_ratio = len(labeled['Move']) / len(lg)
if "Execute" in labeled.keys():
execute_ratio = len(labeled['Execute']) / len(lg)
if "Cleanup" in labeled.keys():
cleanup_ratio = len(labeled['Cleanup']) / len(lg)
#################
# persist in .csv
#################
writer.writerow([
node_count, critical_node_count, x_bar, Sigma_x, range_x, y_bar,
Sigma_y, range_y, percentage_of_links, entropy, t_complexity,
link_index, graph_differences, entropy_deviation, mean_link_coverage,
top_cover, session_start_time, session_length_seconds,
mean_delay_seconds, access_ratio, look_ratio, transfer_ratio,
move_ratio, execute_ratio, cleanup_ratio, access_next, look_next,
transfer_next, move_next, execute_next, cleanup_next
])
def histogram(length,
ontology,
function=None,
absClasses=None,
samples=None,
random=False,
seed=None):
"""Finds the linkographs produced by the ontology.
Finds every derived linkograph on length nodes according to the
given ontology and groups them according to their value under the
function. The return type is a dictionary so the range of the
function needs to be a type that can be used as a key for the
dictionary.
Arguments:
length -- the number of nodes for the linkograph.
ontology -- the ontology to consider.
function -- the function to apply to the linkographs. The range
must be a type that can be used as a key to a dictionary.
absClasses -- optional set of classes if the ontology does not
contain all the abstraction classes of interest.
samples -- The number of labelings to generate. Assigning None set
samples to (sizeOfAbastrctionClasses)**length, which is the number
of different possible labelings.
random -- If False, then the labelings are considered according to
an internal counter. If True, the labelings are randomly selected.
seed -- seed for the interal random number generator.
Returns:
{functionValue: linkographEnumList} -- a dictionary with keys the
values of the function and values the list of linkograph enums
that map to the function value.
"""
if function is None:
function = linkoToEnum
# Create the count dictionary.
hist = {}
# Get the abstraction classes.
if absClasses is None:
absClasses = [key for key in ontology.keys()]
# Labelings are provided by the help of a modularCounter. This is
# and n-element counter that counts in modular arithmetic where
# the base can be specified.
counter = modularCounter(length, len(absClasses))
# Set the number of labelings to consider if a limit is not
# provided.
if samples is None:
samples = len(absClasses)**length
# Loop through the possible labelings.
for n in range(samples):
# Convert the current count on the modularCounter to a
# labeling for the given abstraction class.
invLabeling = counter.toInverseLabeling(absClasses)
# Create the linkograph based on the labeling and the
# ontology.
linko = linkoCreate.createLinko(invLabeling, ontology)
# Convert the linkograph into an integer.
enum = linkoToEnum(linko)
# Get the value of the function.
value = function(linko)
# Check if the value has been seen before.
cachedSet = hist.get(value)
if not cachedSet:
hist[value] = {tuple(counter.toLabeling(absClasses))}
else:
hist[value].add(tuple(counter.toLabeling(absClasses)))
if random:
# Randomize the count
counter.randomize()
else:
# Increment the modularCounter
counter.inc()
return hist
def frequency(length,
ontology,
function=None,
absClasses=None,
samples=None,
random=False,
seed=None):
"""Finds the linkographs produced by the ontology.
Finds the number of derived linkographs that map to the same value
under the given function.
Arguments:
length -- the number of nodes for the linkograph.
ontology -- the ontology to condier.
function -- the function to compute on the linkographs. The range
must be a type that can be used as a key to a dictionary.
absClasses -- optional set of classes if the ontology does not
contain all the abstraction classes of interest.
samples -- The number of labelings to generate. Assigning None set
samples to (sizeOfAbastrctionClasses)**length, which is the number
of different possible labelings.
random -- If False, then the labelings are considered according to
an internal counter. If True, the labelings are randomly selected.
seed -- seed for the interal random number generator.
Returns:
{functionValue: count} -- a dictionary with keys the function's
value and values the number of times a linkograph is produced with
that value.
"""
# Set the default function.
if function is None:
function = lambda x: linkoToEnum(x)
# Create the count dictionary.
freq = {}
# If the length is zero, then the empty linkograph is the only
# possible linkograph and there is only one possible labeling (the
# empty labeling).
if length == 0:
# Create an empty linkograph.
linko0_0 = enumToLinko((0, 0))
freq[function(linko0_0)] = 1
return freq
# Get the abstraction classes.
if absClasses is None:
absClasses = [key for key in ontology.keys()]
# Labelings are provided by the help of a modularCounter. This is
# and n-element counter that counts in modular arithmetic where
# the base can be specified.
counter = modularCounter(length, len(absClasses), seed=seed)
# Set the number of labelings to consider if a limit is not
# provided.
if samples is None:
samples = len(absClasses)**length
# Loop through the possible labelings.
for n in range(samples):
# Convert the current count on the modularCounter to a
# labeling for the given abstraction class.
invLabeling = counter.toInverseLabeling(absClasses)
# Create the linkograph based on the labeling and the
# ontology.
linko = linkoCreate.createLinko(invLabeling, ontology)
# Find the value of the function.
value = function(linko)
# Record the count.
currentCount = freq.get(value)
if not currentCount:
freq[value] = 1
else:
freq[value] = currentCount + 1
if random:
# Randomize the count
counter.randomize()
else:
# Increment the modularCounter
counter.inc()
return freq