def createLinko(self, commands, ontology, abstraction, linkograph, unique_id): """createLinko: This function takes the names of the commands, ontology, and abstraction files to be used in creating a linkograph, as well as a potential name for the resulting linkograph, and returns the name of the saved file. """ #Come up with a name for the saved linko file fileName = commands + ontology + abstraction if (len(linkograph) != 0): fileName = linkograph commandsName = commands #Loading in all of the necessary files commands = fs.loadFile(commands, 'commands', unique_id)['content'] ontology = fs.loadFile(ontology, 'ontology', unique_id)['content'] abstraction = fs.loadFile(abstraction, 'abstraction', unique_id)['content'] commands = json.loads(commands) ontology = json.loads(ontology) abstraction = json.loads(abstraction) #Labeling the commands lr = labels.Labeler(abstraction) inverseLabeling = lr.labelCommands(commands, defaultLabel="NoLabel") inverseLabeling = labels.writesLabelsToJsonFile(inverseLabeling) #Creating and saving the actual linkograph linkograph = linkoCreate.createLinko(inverseLabeling, ontology) fileContent = linkoCreate.writesLinkoJson(linkograph) fs.saveFile(fileName, fileContent, 'linkograph', commandsName, unique_id) return fileName
def performTestForParams(self): """"Performs the tests for each set of parameters.""" for (testNum, params) in enumerate(self.testParams): actualLinkograph = linkoCreate.createLinko( params['inverseLabeling'], params['ontology']) self.assertEqual(actualLinkograph, params['ExpectedLinkograph'], ("testNum = {}" " inversLabling = {}" " ontology= {}" " actualLinkograph = {}" " ExpectedLinkograph = {}").format( testNum, params['inverseLabeling'], params['ontology'], actualLinkograph, params['ExpectedLinkograph']))
def visualize(lg, o, base_name, iteration): # derive linkograph using labels from existing linkograph # @attention is there a better way to extract labeling from a linkograph? labeling = dict() for node_index in range(len(lg)): label_set = lg[node_index][0].copy() label = label_set.pop() if not label in labeling: labeling[label] = [node_index] else: labeling[label].append(node_index) derived_lg = llinkoCreate.createLinko(labeling, o) # persist image of derived linkograph svg = llinkoDrawSVG.linkoDrawSVG(derived_lg) with open(base_name + "_" + str(iteration) + ".svg", "w") as svg_file: svg_file.write(svg)
def genLinkograph(self, n, ontology=None): """Generate a linkograph on n-nodes.""" # If no ontology is passed, use one stored in the model. if ontology is None: ontology = self.ontology # If the model did not have an ontology either, then raise an # error. if ontology is None: raise OntologyError('Model does not have ontology.') invLabel = self.inverseLabeling(n) linko = lc.createLinko(invLabel, ontology) # Set the linkographs labels to ensure same order as the # abstraction classes used in the model linko.labels = self.absClasses return linko
def bulk_transform(method): for session_filename in sys.argv[1:]: ######################### # create input linkograph ######################### label_rules = open("abstraction.json", "r") labeler = llabels.Labeler(json.load(label_rules)) label_rules.close() commands = open(session_filename, "r") json_commands = json.load(commands) commands.close() labeled = labeler.labelCommands(json_commands, "NoLabel") llabels.writeLabelsToJsonFile(labeled, "labeled.json") ontology_file = open("ontology.json", "r") inv_labeling_file = open("labeled.json", "r") lg = llinkoCreate.createLinko(json.load(inv_labeling_file), json.load(ontology_file)) inv_labeling_file.close() ontology_file.close() ################################## # transform linkograph to ontology ################################## if 0 == method: extracted_ontology = oe.simple_lg_to_ontology(lg) elif 1 == method: extracted_ontology = oe.threshold_lg_to_ontology(lg) else: print("unknown method:", method) ######### # cleanup ######### os.remove("labeled.json")
######################### # create input linkograph ######################### label_rules = open("abstraction.json", "r") labeler = llabels.Labeler(json.load(label_rules)) label_rules.close() commands = open(session_filename, "r") json_commands = json.load(commands) commands.close() labeled = labeler.labelCommands(json_commands, "NoLabel") llabels.writeLabelsToJsonFile(labeled, "labeled.json") ontology_file = open("ontology.json", "r") inv_labeling_file = open("labeled.json", "r") labeling = json.load(inv_labeling_file) lg = llinkoCreate.createLinko(labeling, json.load(ontology_file)) inv_labeling_file.close() ontology_file.close() # a linkograph is a list of tuples (nodes) # a node is a 3-tuple # component 0 is a set of labels # component 1 is a set of backlinks # component 2 is a set of forelinks ################################## # transform linkograph to ontology ################################## extracted_ontology = simple_lg_to_ontology(lg)
def processSession(session_filename, writer): ##################### # generate linkograph ##################### # label commands label_rules = open("abstraction.json", "r") labeler = llabels.Labeler(json.load(label_rules)) label_rules.close() commands = open(session_filename, "r") json_commands = json.load(commands) commands.close() if 2 > len(json_commands): print("can't process", session_filename, "because session files must have at least two commands") return last_command = json_commands.pop() # access_next, look_next, transfer_next, move_next, execute_next, cleanup_next last_command_labels = labeler.labelCommands([last_command], "NoLabel") access_next = 0 look_next = 0 transfer_next = 0 move_next = 0 execute_next = 0 cleanup_next = 0 if "Access" in last_command_labels: access_next = 1 if "Look" in last_command_labels: look_next = 1 if "Transfer" in last_command_labels: transfer_next = 1 if "Move" in last_command_labels: move_next = 1 if "Execute" in last_command_labels: execute_next = 1 if "Cleanup" in last_command_labels: cleanup_next = 1 labeled = labeler.labelCommands(json_commands, "NoLabel") # @todo cleanup labeled.json when its safe llabels.writeLabelsToJsonFile(labeled, "labeled.json") # link commands ontology = open("ontology.json", "r") inv_labeling = open("labeled.json", "r") lg = llinkoCreate.createLinko(json.load(inv_labeling), json.load(ontology)) inv_labeling.close() os.remove("labeled.json") ontology.close() ################## # extract features ################## # node_count node_count = len(lg) # critical_node_count # @todo: pick something real for critical_threshold critical_threshold = node_count / 2 critical_node_count = lstats.countCriticalNodes(lg, critical_threshold) # x_bar, Sigma_x, range_x, y_bar, Sigma_y, range_y x_bar, Sigma_x, range_x, y_bar, Sigma_y, range_y = lstats.calculateCartesianStatistics( lg) # percentage_of_links percentage_of_links = lstats.percentageOfLinks(lg) # entropy entropy = lstats.graphEntropy(lg) # T-Complexity encoded_lg = lstats.linkographToString(lg) t_complexity = lstats.tComplexity(encoded_lg) # link_index link_index = lstats.links(lg) / len(lg) # graph differences graph_differences = lstats.summaryDifference(lg) # entropy deviation entropy_deviation = lstats.entropyDeviation(lg) # mean link coverage mean_link_coverage = lstats.meanLinkCoverage(lg) # top cover top_cover = lstats.topCover(lg) first_command = json_commands[0] first_datetime = utils.stringToDatetime(first_command['ts']) session_start_time = first_datetime.hour * 3600 + first_datetime.minute * 60 + first_datetime.second # session_length_seconds, mean_delay_seconds last_command = json_commands[-1] last_datetime = utils.stringToDatetime(last_command['ts']) session_length_timedelta = last_datetime - first_datetime session_length_seconds = session_length_timedelta.total_seconds() if 1 < len(lg): mean_delay_seconds = session_length_seconds / (len(lg) - 1) else: mean_delay_seconds = None # access_ratio, look_ratio, transfer_ratio, move_ratio, execute_ratio, cleanup_ratio access_ratio = look_ratio = transfer_ratio = move_ratio = execute_ratio = cleanup_ratio = 0 if "Access" in labeled.keys(): access_ratio = len(labeled['Access']) / len(lg) if "Look" in labeled.keys(): look_ratio = len(labeled['Look']) / len(lg) if "Transfer" in labeled.keys(): transfer_ratio = len(labeled['Transfer']) / len(lg) if "Move" in labeled.keys(): move_ratio = len(labeled['Move']) / len(lg) if "Execute" in labeled.keys(): execute_ratio = len(labeled['Execute']) / len(lg) if "Cleanup" in labeled.keys(): cleanup_ratio = len(labeled['Cleanup']) / len(lg) ################# # persist in .csv ################# writer.writerow([ node_count, critical_node_count, x_bar, Sigma_x, range_x, y_bar, Sigma_y, range_y, percentage_of_links, entropy, t_complexity, link_index, graph_differences, entropy_deviation, mean_link_coverage, top_cover, session_start_time, session_length_seconds, mean_delay_seconds, access_ratio, look_ratio, transfer_ratio, move_ratio, execute_ratio, cleanup_ratio, access_next, look_next, transfer_next, move_next, execute_next, cleanup_next ])
def histogram(length, ontology, function=None, absClasses=None, samples=None, random=False, seed=None): """Finds the linkographs produced by the ontology. Finds every derived linkograph on length nodes according to the given ontology and groups them according to their value under the function. The return type is a dictionary so the range of the function needs to be a type that can be used as a key for the dictionary. Arguments: length -- the number of nodes for the linkograph. ontology -- the ontology to consider. function -- the function to apply to the linkographs. The range must be a type that can be used as a key to a dictionary. absClasses -- optional set of classes if the ontology does not contain all the abstraction classes of interest. samples -- The number of labelings to generate. Assigning None set samples to (sizeOfAbastrctionClasses)**length, which is the number of different possible labelings. random -- If False, then the labelings are considered according to an internal counter. If True, the labelings are randomly selected. seed -- seed for the interal random number generator. Returns: {functionValue: linkographEnumList} -- a dictionary with keys the values of the function and values the list of linkograph enums that map to the function value. """ if function is None: function = linkoToEnum # Create the count dictionary. hist = {} # Get the abstraction classes. if absClasses is None: absClasses = [key for key in ontology.keys()] # Labelings are provided by the help of a modularCounter. This is # and n-element counter that counts in modular arithmetic where # the base can be specified. counter = modularCounter(length, len(absClasses)) # Set the number of labelings to consider if a limit is not # provided. if samples is None: samples = len(absClasses)**length # Loop through the possible labelings. for n in range(samples): # Convert the current count on the modularCounter to a # labeling for the given abstraction class. invLabeling = counter.toInverseLabeling(absClasses) # Create the linkograph based on the labeling and the # ontology. linko = linkoCreate.createLinko(invLabeling, ontology) # Convert the linkograph into an integer. enum = linkoToEnum(linko) # Get the value of the function. value = function(linko) # Check if the value has been seen before. cachedSet = hist.get(value) if not cachedSet: hist[value] = {tuple(counter.toLabeling(absClasses))} else: hist[value].add(tuple(counter.toLabeling(absClasses))) if random: # Randomize the count counter.randomize() else: # Increment the modularCounter counter.inc() return hist
def frequency(length, ontology, function=None, absClasses=None, samples=None, random=False, seed=None): """Finds the linkographs produced by the ontology. Finds the number of derived linkographs that map to the same value under the given function. Arguments: length -- the number of nodes for the linkograph. ontology -- the ontology to condier. function -- the function to compute on the linkographs. The range must be a type that can be used as a key to a dictionary. absClasses -- optional set of classes if the ontology does not contain all the abstraction classes of interest. samples -- The number of labelings to generate. Assigning None set samples to (sizeOfAbastrctionClasses)**length, which is the number of different possible labelings. random -- If False, then the labelings are considered according to an internal counter. If True, the labelings are randomly selected. seed -- seed for the interal random number generator. Returns: {functionValue: count} -- a dictionary with keys the function's value and values the number of times a linkograph is produced with that value. """ # Set the default function. if function is None: function = lambda x: linkoToEnum(x) # Create the count dictionary. freq = {} # If the length is zero, then the empty linkograph is the only # possible linkograph and there is only one possible labeling (the # empty labeling). if length == 0: # Create an empty linkograph. linko0_0 = enumToLinko((0, 0)) freq[function(linko0_0)] = 1 return freq # Get the abstraction classes. if absClasses is None: absClasses = [key for key in ontology.keys()] # Labelings are provided by the help of a modularCounter. This is # and n-element counter that counts in modular arithmetic where # the base can be specified. counter = modularCounter(length, len(absClasses), seed=seed) # Set the number of labelings to consider if a limit is not # provided. if samples is None: samples = len(absClasses)**length # Loop through the possible labelings. for n in range(samples): # Convert the current count on the modularCounter to a # labeling for the given abstraction class. invLabeling = counter.toInverseLabeling(absClasses) # Create the linkograph based on the labeling and the # ontology. linko = linkoCreate.createLinko(invLabeling, ontology) # Find the value of the function. value = function(linko) # Record the count. currentCount = freq.get(value) if not currentCount: freq[value] = 1 else: freq[value] = currentCount + 1 if random: # Randomize the count counter.randomize() else: # Increment the modularCounter counter.inc() return freq