def _find_needed_params(model_file_name, path, response): from murphy.model import Model from murphy.run_planner import get_params_needed model = Model(model_file_name) path = get_params_needed(path, model.new_worker()) response.put(path)
def _build_view(model_file_name, view_name, view_type, output_dir): ''' Builds the given view ''' import sys, os from murphy.model import Model from murphy import utils, graphviz #FIXME: this is needed in order to import the module, but may be better to #do it in the model object before importing? base_path = os.path.dirname(os.path.dirname(model_file_name)) base_path = os.path.abspath(os.path.dirname(base_path)) sys.path.append(base_path) model = Model(model_file_name) worker = model.new_worker() start_node = model.get_starting_node(view_name) dot = worker.graphs.generate_from_spider(start_node, {}) temp_file = '%s/temp.dot' % output_dir target_file = '%s/flow.xml' % output_dir #FIXME: do silent_remove with proper exception handling... if os.path.isfile(temp_file): os.remove(temp_file) if os.path.isfile(target_file): os.remove(target_file) utils.save_file(dot, temp_file) graphviz.generate_svg(temp_file) os.rename('%s.svg' % temp_file, target_file) #os.remove(temp_file) #svg_content = utils.load_text_file(target_file) import zipfile zip = zipfile.ZipFile('%s.zip' % target_file, 'w') zip.write(target_file, 'flow.xml') zip.close() _build_dot_with_screenshots(dot, worker, output_dir, model.images_dir) svg_content = utils.load_text_file('%s/simple-flow-images.xml' % output_dir) svg_content = svg_content.replace('xlink:href="%s/' % output_dir, 'xlink:href="') utils.save_file(svg_content, '%s/local-simple-flow-images.xml' % output_dir) downloadable_name = os.path.basename(model_file_name) if downloadable_name.find(".") != -1: downloadable_name = downloadable_name.split(".")[0] downloadable_name = '%s-%s-simple.zip' % (downloadable_name, view_name) zip = zipfile.ZipFile('%s/%s' % (output_dir, downloadable_name), 'w') zip.write('%s/local-simple-flow-images.xml' % output_dir, 'flow.xml') for file_name in os.listdir(output_dir): if file_name.endswith(".png") or file_name.endswith(".gif"): zip.write('%s/%s' % (output_dir, file_name), file_name) zip.close()
def run_remote_test(test_function, model_file): remoting_obj = None try: model = Model(model_file) remoting_obj = model.rules.get_remoting_vnc_object() worker = model.new_worker(remoting_obj.automation) test_function(worker) print "PASS" except Exception, e: print "FAILED TEST, error: %s" % str(e)
def run_local_test(test_function, model_file): try: logging.basicConfig(level=logging.INFO, format='%(asctime)s - %(name)s - %(levelname)s - %(message)s') model = Model(model_file) worker = model.new_worker() test_function(worker) print "PASS" except Exception, e: print "FAILED TEST, error: %s" % str(e)
def _run_plan(model_file_name, plan, out_file_name): import os, sys, time, json try: os.makedirs(os.path.dirname(out_file_name)) except: pass print "should log to " + out_file_name import logging from murphy.model import Model fileHandler = logging.FileHandler(out_file_name + '.bsy', mode='a', encoding=None, delay=False) root_logger = logging.getLogger() root_logger.addHandler(fileHandler) formatter = logging.Formatter( '%(asctime)s - %(name)s - %(levelname)s - %(message)s') fileHandler.setFormatter(formatter) logging.getLogger().removeHandler(logging.getLogger().handlers[0]) LOGGER = logging.getLogger('root.' + __name__) LOGGER.info("Run request runs on pid " + str(os.getpid())) remoting = None try: steps = [] run_params = {} for step in plan: steps.append((step['node'], step['arc'])) params = step['params'] for param in params: values = run_params.get(param['name'], []) values.append(param['value']) run_params[param['name']] = values model = Model(model_file_name) remoting = model.rules.get_remoting_vnc_object() worker = model.new_worker(remoting.automation) worker.parameters.update(run_params) LOGGER.info("Requesting run:\n%s" % json.dumps(steps, sort_keys=True, indent=4)) LOGGER.info("Using values:\n%s" % json.dumps(worker.parameters, sort_keys=True, indent=4)) LOGGER.info("Running at ip %s" % remoting.ip_address) LOGGER.info("Remoting at vnc://%s:%s" % (remoting.vnc_host, remoting.vnc_port)) worker.Walk_ext(steps) except Exception, ex: # traceback.print_exc(file=sys.stdout) LOGGER.exception(ex)
def _get_edge_logs(model_file_name, node_name, edge_name, response): try: from murphy.model import Model model = Model(model_file_name) worker = model.new_worker() ret = worker.get_verb_logs(node_name, edge_name) if ret != "": with open(model.working_dir + "/" + ret, "r") as the_log: ret = the_log.read() response.put(ret) except Exception, ex: traceback.print_exc(file=sys.stdout) print "Problem: %s" % str(ex) response.put("Error while getting edge logs: %s" % str(ex))
def _run_plan(model_file_name, plan, out_file_name): import os, sys, time, json try: os.makedirs(os.path.dirname(out_file_name)) except: pass print "should log to " + out_file_name import logging from murphy.model import Model fileHandler = logging.FileHandler(out_file_name + '.bsy', mode='a', encoding=None, delay=False) root_logger = logging.getLogger() root_logger.addHandler(fileHandler) formatter = logging.Formatter('%(asctime)s - %(name)s - %(levelname)s - %(message)s') fileHandler.setFormatter(formatter) logging.getLogger().removeHandler(logging.getLogger().handlers[0]) LOGGER = logging.getLogger('root.' + __name__) LOGGER.info("Run request runs on pid " + str(os.getpid())) remoting = None try: steps = [] run_params = {} for step in plan: steps.append((step['node'], step['arc'])) params = step['params'] for param in params: values = run_params.get(param['name'], []) values.append(param['value']) run_params[param['name']] = values model = Model(model_file_name) remoting = model.rules.get_remoting_vnc_object() worker = model.new_worker(remoting.automation) worker.parameters.update(run_params) LOGGER.info("Requesting run:\n%s" % json.dumps(steps, sort_keys=True, indent=4)) LOGGER.info("Using values:\n%s" % json.dumps(worker.parameters, sort_keys=True, indent=4)) LOGGER.info("Running at ip %s" % remoting.ip_address) LOGGER.info("Remoting at vnc://%s:%s" % (remoting.vnc_host, remoting.vnc_port)) worker.Walk_ext(steps) except Exception, ex: # traceback.print_exc(file=sys.stdout) LOGGER.exception(ex)
def _solve_route(steps, model_file_name, tags, response): ''' Multiprocessing friendly implementation ''' from murphy.model import Model from murphy.run_planner import solve_plan from murphy.errors import NoRouteFound model = Model(model_file_name) plan = [] for step in steps: a_step = {'node': step, 'heuristics': ['shortest']} plan.append(a_step) path = [] try: path = solve_plan(plan, model.new_worker(), model, tags) except NoRouteFound: pass #return an empty path when there's no path response.put(path)
def _get_graph_logs(model_file_name, response): try: from murphy.model import Model model = Model(model_file_name) worker = model.new_worker() views = worker.get_views() ret = {} for node in views.keys(): for edge in views[node]['verbs'].keys(): edge_def = views[node]['verbs'][edge] log = worker.get_verb_logs(node, edge) if log != "": with open(model.working_dir + "/" + log, "r") as the_log: log = the_log.read() ret["%s.%s" % (node, edge)] = json.loads(log) response.put(json.dumps(ret)) except Exception, ex: traceback.print_exc(file=sys.stdout) print "Problem: %s" % str(ex) response.put("Error while getting graph logs: %s" % str(ex))
class ModelComparison(object): def __init__(self, model_file_name, reference_model_file_name): self.model = Model(model_file_name) self.reference_model = Model(reference_model_file_name) def _get_ordered_views(self, model): namespace = model.model['namespace'] ordered_views = [] for name in model.model['modules']: module = sys.modules['%s.%s' % (namespace, name)] ordered_views.append(module.HERE['desc']) return ordered_views def compare_view(self, view, reference_view): return '' def compare_edges(self, view, reference_view, reference_translation): result = '' for name, verb in view['verbs'].items(): if not name in reference_view['verbs']: result += "Edge '%s' does not exists in reference model (new?)<br>\n" % name else: reference_verb = reference_view['verbs'][name] dest = verb.get('goes to', '') reference_dest = reference_verb.get('goes to', '') #print "dest %s, reference dest %s translation %s" % (dest, reference_dest, str(reference_translation)) #fixme translate something when drawing??? #fixme search edge by screenshot if available, by name if it is not if not dest in reference_translation: head = self.model.new_worker().get_views()[verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % { 'title': 'New edge?', 'node_image': get_node_image_b64( self.model, view, True), 'node_name': view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.model, verb), 'edge_name': name, 'head_image': get_node_image_b64( self.model, head, True), 'head_name': head['self'].HERE['desc'] } elif reference_translation[dest] != reference_dest: #result += "Edge '%s' goes to '%s' in model but goes to '%s' in reference model<br>\n" % (name, dest, reference_dest) id1 = str(uuid.uuid1()) id2 = str(uuid.uuid1()) result += "<input type='checkbox' onchange='javascript:change_visibility(\"" + id1 + "\", \"" + id2 + "\");'>" result += "Show parametrized images<br>" for i in range(2): if i == 0: result += "<div id='%s' style='display: none'>" % id1 else: result += "<div id='%s'>" % id2 reference = (i != 0) head = self.model.new_worker().get_views()[ verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % { 'title': 'Now:', 'node_image': get_node_image_b64(self.model, view, reference), 'node_name': view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.model, verb), 'edge_name': name, 'head_image': get_node_image_b64(self.model, head, reference), 'head_name': head['self'].HERE['desc'] } head = self.reference_model.new_worker().get_views()[ reference_verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % { 'title': 'Before:', 'node_image': get_node_image_b64(self.reference_model, reference_view, reference), 'node_name': reference_view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.reference_model, reference_verb), 'edge_name': name, 'head_image': get_node_image_b64(self.reference_model, head, reference), 'head_name': head['self'].HERE['desc'] } result += '</div>' for name, verb in reference_view['verbs'].items(): #FIXME: translation missing if not name in view['verbs']: #FIXME: add image result += "Edge '%s' does not exists in model (removed?)<br>\n" % name if result != '': result = "Edges in node '%s' differs from the reference model.<br>\n%s" % ( view['self'].HERE['desc'], result) return result def find_node_by_image(self, model, view, reference_model, reference_views): ordered_views = self._get_ordered_views(reference_model) for view_name in ordered_views: result = self.compare_view_images(model, view, reference_model, reference_views[view_name]) if result == '': return reference_views[view_name] return None def compare_view_images(self, model, view, reference_model, reference_view): #we compare parametrized images, otherwise dates and times will not match view_images = view['self'].HERE.get('snapshots', []) reference_view_images = reference_view['self'].HERE.get( 'snapshots', []) if len(view_images) != len(reference_view_images): return 'Node "%s" has %s snapshots but the reference node has %s' % ( view['self'].HERE['desc'], len(view_images), len(reference_view_images)) result = '' for index in range(len(view_images)): image1 = Image2(file_name="%s/%s" % (model.images_dir, view_images[index])) image2 = Image2( file_name="%s/%s" % (reference_model.images_dir, reference_view_images[index]), tolerance=0.9999) if image1 != image2: result += 'Node "%s", image "%s" differs from reference node "%s"<br>\n' % ( view['self'].HERE['desc'], view_images[index], reference_view_images[index]) image1.image.save("tmp.png") encoded1 = base64.b64encode(read_binary_file("tmp.png")) image2.image.save("tmp.png") encoded2 = base64.b64encode(read_binary_file("tmp.png")) id1 = str(uuid.uuid1()) id2 = str(uuid.uuid1()) result += "<table>\n\t<tr>\n\t\t<td><img id='%s' title='model' src='data:image/png;base64,%s'></td>\n" % ( id1, encoded1) result += "\t\t<td><input style='font-family:\"Courier New\", Courier, monospace;' type=button id='button-%s' value='<- New \n Reference ->' onclick='swap(\"%s\", \"%s\")'></td>\n" % ( id1, id1, id2) result += "\t\t<td><img id='%s' title='reference' src='data:image/png;base64,%s'></td>\n\t</tr>\n</table><br>\n" % ( id2, encoded2) return result def compare(self): views = self.model.new_worker().get_views() reference_views = self.reference_model.new_worker().get_views() matching_views = [] candidate_for_moved = [] moved_views = [] candidate_for_new = [] changed_views = [] new_views = [] reference_views_used = [] #dictionary of moved nodes, key is model view nam, value is reference view name reference_translation = {} reference_translation[''] = '' result = '' ordered_views = self._get_ordered_views(self.model) for view_name in ordered_views: view = views[view_name] if view_name in reference_views: reference_view = reference_views[view_name] comparison = self.compare_view_images(self.model, view, self.reference_model, reference_view) if comparison == '': matching_views.append(view) reference_views_used.append(reference_view) reference_translation[view_name] = view_name else: candidate_for_moved.append(view) else: candidate_for_moved.append(view) for view in candidate_for_moved: candidate = self.find_node_by_image(self.model, view, self.reference_model, reference_views) if candidate and not candidate in reference_views_used: moved_views.append({'view': view, 'reference view': candidate}) reference_views_used.append(candidate) reference_translation[ view['self'].HERE['desc']] = candidate['self'].HERE['desc'] else: candidate_for_new.append(view) for view in candidate_for_new: view_name = view['self'].HERE['desc'] if view_name in reference_views and not reference_views[ view_name] in reference_views_used: #can this still be wrong? changed_views.append(view) reference_views_used.append(reference_views[view_name]) reference_translation[view_name] = view_name else: new_views.append(view) #print "Translation table: %s" % str(reference_translation) for view in matching_views: reference_view = reference_views[view['self'].HERE['desc']] result += self.compare_edges(view, reference_view, reference_translation) for movement in moved_views: result += "Node '%s' is in reference model as %s<br>\n" % ( movement['view']['self'].HERE['desc'], movement['reference view']['self'].HERE['desc']) result += self.compare_edges(movement['view'], movement['reference view'], reference_translation) for view in changed_views: result += self.compare_view_images( self.model, view, self.reference_model, reference_views[view['self'].HERE['desc']]) for view in new_views: result += "Node '%s' is new, does not seems to exist in the reference model<br>\n" % ( view['self'].HERE['desc']) view_images = view['self'].HERE.get('snapshots', []) if len(view_images) > 0: image = Image2(file_name="%s/%s" % (self.model.images_dir, view_images[0])) image.image.save("tmp.png") encoded = base64.b64encode(read_binary_file("tmp.png")) result += "<img src='data:image/png;base64,%s' title='model'><br>\n" % encoded for view in reference_views.values(): if not view in reference_views_used: result += "Reference node '%s' does not seem to exists in the model (was removed?)<br>\n" % ( view['self'].HERE['desc']) view_images = view['self'].HERE.get('snapshots', []) if len(view_images) > 0: image = Image2( file_name="%s/%s" % (self.reference_model.images_dir, view_images[0])) image.image.save("tmp.png") encoded = base64.b64encode(read_binary_file("tmp.png")) result += "<img src='data:image/png;base64,%s' title='model'><br>\n" % encoded return result
class ModelComparison(object): def __init__(self, model_file_name, reference_model_file_name): self.model = Model(model_file_name) self.reference_model = Model(reference_model_file_name) def _get_ordered_views(self, model): namespace = model.model['namespace'] ordered_views = [] for name in model.model['modules']: module = sys.modules['%s.%s' % (namespace, name)] ordered_views.append(module.HERE['desc']) return ordered_views def compare_view(self, view, reference_view): return '' def compare_edges(self, view, reference_view, reference_translation): result = '' for name, verb in view['verbs'].items(): if not name in reference_view['verbs']: result += "Edge '%s' does not exists in reference model (new?)<br>\n" % name else: reference_verb = reference_view['verbs'][name] dest = verb.get('goes to', '') reference_dest = reference_verb.get('goes to', '') #print "dest %s, reference dest %s translation %s" % (dest, reference_dest, str(reference_translation)) #fixme translate something when drawing??? #fixme search edge by screenshot if available, by name if it is not if not dest in reference_translation: head = self.model.new_worker().get_views()[verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % {'title': 'New edge?', 'node_image': get_node_image_b64(self.model, view, True), 'node_name': view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.model, verb), 'edge_name': name, 'head_image': get_node_image_b64(self.model, head, True), 'head_name': head['self'].HERE['desc']} elif reference_translation[dest] != reference_dest: #result += "Edge '%s' goes to '%s' in model but goes to '%s' in reference model<br>\n" % (name, dest, reference_dest) id1 = str(uuid.uuid1()) id2 = str(uuid.uuid1()) result += "<input type='checkbox' onchange='javascript:change_visibility(\"" + id1 + "\", \"" + id2 + "\");'>" result += "Show parametrized images<br>" for i in range(2): if i == 0: result += "<div id='%s' style='display: none'>" % id1 else: result += "<div id='%s'>" % id2 reference = (i != 0) head = self.model.new_worker().get_views()[verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % {'title': 'Now:', 'node_image': get_node_image_b64(self.model, view, reference), 'node_name': view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.model, verb), 'edge_name': name, 'head_image': get_node_image_b64(self.model, head, reference), 'head_name': head['self'].HERE['desc']} head = self.reference_model.new_worker().get_views()[reference_verb['goes to']] result += EDGE_DIFFERENCE_TEMPLATE % {'title': 'Before:', 'node_image': get_node_image_b64(self.reference_model, reference_view, reference), 'node_name': reference_view['self'].HERE['desc'], 'edge_image': get_verb_image_b64(self.reference_model, reference_verb), 'edge_name': name, 'head_image': get_node_image_b64(self.reference_model, head, reference), 'head_name': head['self'].HERE['desc']} result += '</div>' for name, verb in reference_view['verbs'].items(): #FIXME: translation missing if not name in view['verbs']: #FIXME: add image result += "Edge '%s' does not exists in model (removed?)<br>\n" % name if result != '': result = "Edges in node '%s' differs from the reference model.<br>\n%s" % (view['self'].HERE['desc'], result) return result def find_node_by_image(self, model, view, reference_model, reference_views): ordered_views = self._get_ordered_views(reference_model) for view_name in ordered_views: result = self.compare_view_images(model, view, reference_model, reference_views[view_name]) if result == '': return reference_views[view_name] return None def compare_view_images(self, model, view, reference_model, reference_view): #we compare parametrized images, otherwise dates and times will not match view_images = view['self'].HERE.get('snapshots', []) reference_view_images = reference_view['self'].HERE.get('snapshots', []) if len(view_images) != len(reference_view_images): return 'Node "%s" has %s snapshots but the reference node has %s' % (view['self'].HERE['desc'], len(view_images), len(reference_view_images)) result = '' for index in range(len(view_images)): image1 = Image2(file_name="%s/%s" % (model.images_dir, view_images[index])) image2 = Image2(file_name="%s/%s" % (reference_model.images_dir, reference_view_images[index]), tolerance=0.9999) if image1 != image2: result += 'Node "%s", image "%s" differs from reference node "%s"<br>\n' % (view['self'].HERE['desc'], view_images[index], reference_view_images[index]) image1.image.save("tmp.png") encoded1 = base64.b64encode(read_binary_file("tmp.png")) image2.image.save("tmp.png") encoded2 = base64.b64encode(read_binary_file("tmp.png")) id1 = str(uuid.uuid1()) id2 = str(uuid.uuid1()) result += "<table>\n\t<tr>\n\t\t<td><img id='%s' title='model' src='data:image/png;base64,%s'></td>\n" % (id1, encoded1) result += "\t\t<td><input style='font-family:\"Courier New\", Courier, monospace;' type=button id='button-%s' value='<- New \n Reference ->' onclick='swap(\"%s\", \"%s\")'></td>\n" % (id1, id1, id2) result += "\t\t<td><img id='%s' title='reference' src='data:image/png;base64,%s'></td>\n\t</tr>\n</table><br>\n" % (id2, encoded2) return result def compare(self): views = self.model.new_worker().get_views() reference_views = self.reference_model.new_worker().get_views() matching_views = [] candidate_for_moved = [] moved_views = [] candidate_for_new = [] changed_views = [] new_views = [] reference_views_used = [] #dictionary of moved nodes, key is model view nam, value is reference view name reference_translation = {} reference_translation[''] = '' result = '' ordered_views = self._get_ordered_views(self.model) for view_name in ordered_views: view = views[view_name] if view_name in reference_views: reference_view = reference_views[view_name] comparison = self.compare_view_images(self.model, view, self.reference_model, reference_view) if comparison == '': matching_views.append(view) reference_views_used.append(reference_view) reference_translation[view_name] = view_name else: candidate_for_moved.append(view) else: candidate_for_moved.append(view) for view in candidate_for_moved: candidate = self.find_node_by_image(self.model, view, self.reference_model, reference_views) if candidate and not candidate in reference_views_used: moved_views.append({'view': view, 'reference view': candidate}) reference_views_used.append(candidate) reference_translation[view['self'].HERE['desc']] = candidate['self'].HERE['desc'] else: candidate_for_new.append(view) for view in candidate_for_new: view_name = view['self'].HERE['desc'] if view_name in reference_views and not reference_views[view_name] in reference_views_used: #can this still be wrong? changed_views.append(view) reference_views_used.append(reference_views[view_name]) reference_translation[view_name] = view_name else: new_views.append(view) #print "Translation table: %s" % str(reference_translation) for view in matching_views: reference_view = reference_views[view['self'].HERE['desc']] result += self.compare_edges(view, reference_view, reference_translation) for movement in moved_views: result += "Node '%s' is in reference model as %s<br>\n" % (movement['view']['self'].HERE['desc'], movement['reference view']['self'].HERE['desc']) result += self.compare_edges(movement['view'], movement['reference view'], reference_translation) for view in changed_views: result += self.compare_view_images(self.model, view, self.reference_model, reference_views[view['self'].HERE['desc']]) for view in new_views: result += "Node '%s' is new, does not seems to exist in the reference model<br>\n" % (view['self'].HERE['desc']) view_images = view['self'].HERE.get('snapshots', []) if len(view_images) > 0: image = Image2(file_name="%s/%s" % (self.model.images_dir, view_images[0])) image.image.save("tmp.png") encoded = base64.b64encode(read_binary_file("tmp.png")) result += "<img src='data:image/png;base64,%s' title='model'><br>\n" % encoded for view in reference_views.values(): if not view in reference_views_used: result += "Reference node '%s' does not seem to exists in the model (was removed?)<br>\n" % (view['self'].HERE['desc']) view_images = view['self'].HERE.get('snapshots', []) if len(view_images) > 0: image = Image2(file_name="%s/%s" % (self.reference_model.images_dir, view_images[0])) image.image.save("tmp.png") encoded = base64.b64encode(read_binary_file("tmp.png")) result += "<img src='data:image/png;base64,%s' title='model'><br>\n" % encoded return result