def get_colors(obs, factors, as_df=True): v_colors = dict() for f in factors: v = obs.loc[:, f] d = OrderedDict(zip(sorted(v.unique()), colors[f])) v_colors[f] = pd.Series(v.map(d).values, index=v.index) return v_colors if not as_df else pd.DataFrame(v_colors)
def expand_range(value): """ receive a value in form a-d,x-y,z and return list of expanded values. Note, all remap functions expect strings so the expanded range will also be a list of strings. if unable to expand range then return list with original value """ try: result = [] for v in list(set(value.split(","))): r1 = re.search(RANGE_REGEX, v) if r1 is None: result.append(v) continue start = int(r1.group("s")) end = int(r1.group("e")) if start > end: for i in range(end, start + 1): result.append("%d" % i) else: for i in range(start, end + 1): result.append("%d" % i) return sorted(set(result)) except Exception as e: logger.warn("failed to expand range (%s): %s" % (value, e)) return ["%s" % value]
def __init__(self, game, network, train_directory): self.game = game env_name = '%sNoFrameskip-v4' % game env = gym.make(env_name) #env = gym.wrappers.Monitor(env, '/tmp/temp_%s' % game, mode='evaluation', force=True) vb_file = os.path.join(train_directory, "vb.npy") vb = np.load(vb_file) parameters_file = sorted(os.listdir(train_directory))[-3] self.policy = Policy(env, network, "elu") parameters_path = os.path.join(train_directory, parameters_file) print('Using parameters file %s \n' % parameters_path) with open(parameters_path, 'rb') as f: parameters = pickle.load(f)['parameters'] self.policy.set_parameters(parameters) self.policy.set_vb(vb)
def gen_format_all(self, usage): """ generates string for format all """ format_string = " " core_strings = [] for core, usage in usage.items(): if core == 'usage_cpu' and self.exclude_average: continue elif core == 'usage': continue core = core.replace('usage_', '') string = self.formatter.vformat(self.format_all, (), {'core': core, 'usage': usage}) core_strings.append(string) core_strings = sorted(core_strings) return format_string.join(core_strings)
def gen_format_all(self, usage): """ generates string for format all """ format_string = " " core_strings = [] for core, usage in usage.items(): if core == 'usage_cpu' and self.exclude_average: continue elif core == 'usage': continue core = core.replace('usage_', '') string = self.formatter.format(format_string=self.format_all, core=core, usage=usage) core_strings.append(string) core_strings = sorted(core_strings) return format_string.join(core_strings)
# subsample ds = ds.sample(n=n_cells if n_cells < ds.shape[0] else ds.shape[0], replace=False).loc[:, genes] # sort by timepoint, patient d = data.obs.loc[ds.index].sort_values(['patient_id', 'timepoint']) ds = ds.loc[d.index, :] colors = list() for f, cmap in [ ('patient_id', 'tab20'), ('timepoint', 'tab20'), # ('cell_type', 'tab20') ]: q = data.obs.loc[ds.index, f] qq = sorted(q.unique()) m = dict(zip(qq, range(len(qq)))) cmap = plt.get_cmap(cmap) colors.append(cmap([m[x] for x in q])) g = sns.clustermap(ds.T.dropna().T, row_cluster=False, row_colors=colors, metric="euclidean", xticklabels=False, yticklabels=False, cbar_kws={"label": "Expression\n(Z-score)"}, cmap="RdBu_r", center=0, robust=True) g.ax_heatmap.set_xlabel("Genes (n = {})".format(ds.shape[1]))
def search_entries(entries, regex, ignore_case=False, sort_reg=None, sort_neg=False, retro_option=False, retro_space=False, retro_month=False, retro_full=False, retro_deletion=False): """ loop through entries list and print each full entry that matches regex """ results = [] logging.debug("regex: %s, ignore_case: %r, negative: %r" % (regex, ignore_case, sort_neg)) start = time.time() matches = 0 if ignore_case and regex is not None: for e in entries: matched = False if re.search(regex, e, re.IGNORECASE): matches += 1 matched = True if (matched and not sort_neg) or (not matched and sort_neg): results.append(e) elif regex is not None: for e in entries: matched = False if re.search(regex, e): matches += 1 matched = True if (matched and not sort_neg) or (not matched and sort_neg): results.append(e) else: # get all entries results = entries logging.debug("search time: %s" % td(start, time.time())) logging.debug("search matches: %d" % matches) if len(results) == 0: logging.debug("no results to print") return # sort the 'filtered' results into final result list final_results = [] if sort_reg is not None: logging.debug("sort_reg: %s" % sort_reg) sort_match_count = 0 start = time.time() ts_results_default = [] # catch all if we don't match any results ts_results = {} for r in results: r1 = re.search(sort_reg, r) if r1 is not None: sort_match_count += 1 m = r1.group("m") if m not in ts_results: ts_results[m] = [] ts_results[m].append(r) else: ts_results_default.append(r) # final results is combined ts_results_default and sorted results final_results = ts_results_default for k in sorted(ts_results): final_results += ts_results[k] logging.debug("sort attribute time: %s" % td(start, time.time())) logging.debug("sort attribute matches: %d" % sort_match_count) else: final_results = results # print all results (peek first for suppot for retro_peek) if retro_option: retro_peek = results[0].splitlines()[0] if "# aaaModLR" in retro_peek: new_results = aaa_entry_parse(final_results, retro_full) elif "# eventRecord" in retro_peek: new_results = event_entry_parse(final_results, retro_full) elif "# faultRecord" in retro_peek: new_results = fault_entry_parse(final_results, retro_full, retro_deletion) else: logging.error("Unable to Determine Record Type") new_results = final_results if retro_space: new_results = space_option(new_results) for e in new_results: if retro_month: print(letter_month(e)) else: print(e) # normal non-retro printing else: for r in final_results: print("".join(r))
def execute_snapshot(snapshot_id): """ perform snapshot operation for provided fabric name and definition name. The snapshot will be stored in config["DATA_DIR"]. The progress of the collection and final result are saved to the database. format of snapshot: snapshot.<fabric-name>.<date>.tgz /md5checksum checksum of bundle.tgz /bundle.tgz /snapshot.json snapshot object attributes /definition.json /node-X note, node-0 is for global objects /<classname.json> per-object class collection """ from ..utils import (get_app_config, format_timestamp, pretty_print) logger.debug("execute snapshot for: %s" % snapshot_id) config = get_app_config() src = config.get("TMP_DIR", "/tmp/") dst = config.get("DATA_DIR", "/tmp/") reg = "topology/pod-[0-9]+/node-(?P<node>[0-9]+)(/|$)" reg = re.compile(reg) # verify that snapshot object exists, if not abort since no way to alert # user of final result ts = time.time() apic = None s = Snapshots.load(_id=snapshot_id) if not s.exists(): logger.warn("snapshot %s not found" % snapshot_id) return filename = "snapshot.%s.%s.tgz" % (s.fabric, format_timestamp(ts, msec=True)) filename = get_unique_filename(dst, filename) src = "%s/%s" % (src, re.sub("\.tgz$", "", filename)) # update snapshot state with error message and error status def fail(msg="", cleanup=True): if len(msg) > 0: s.error = msg s.status = "error" s.total_time = abs(time.time() - ts) logger.debug("snapshot %s failed: %s" % (s._id, s.error)) if not s.save(): logger.warn("failed to save snapshot %s state" % s._id) if cleanup: tmp_cleanup() return # clean up working directory def tmp_cleanup(): if os.path.isdir(src): logger.debug("cleanup directory %s" % src) try: shutil.rmtree(src) except Exception as e: logger.warn("failed to cleanup directory(%s): %s" % (src, e)) else: logger.debug("cleanup directory %s not found" % src) if apic is not None: apic.close() # update snapshot progress percentage def progress(i, total): abort_snapshot() # check for abort signal before progress update if total > 0: p = round(i / (1.0 * total), 2) else: p = 0 s.progress = p s.total_time = abs(time.time() - ts) if not s.save(): logger.warn("failed to save snapshot %s state" % s._id) return # user can force snapshot execution to stop by deleting the object # abort_snapshot is checked at a regular interval. If snapshot was deleted # then perform cleanup and return True, else return False def abort_snapshot(): _s = Snapshots.load(_id=s._id) if not _s.exists(): logger.debug("snapshot %s has been deleted, aborting" % s._id) tmp_cleanup() raise Exception("snapshot operation %s aborted" % s._id) # write dict data to specified path def json_write(path, data): if not re.search("\.json$", path): path = "%s.json" % path if not os.path.exists(os.path.dirname(path)): os.makedirs(os.path.dirname(path)) logger.debug("saving to %s" % (path)) with open("%s" % path, "w") as f: json.dump(data, f) # init progress, error, status s.nodes = [] s.fabric_domain = "" s.progess = 0.0 s.filename = "" s.filesize = 0 s.start_time = ts s.total_time = 0 s.wait_time = 0 s.error = "" s.status = "running" s.save() try: # setup working directory to store collection outputs if os.path.exists(src): return fail("tmp directory already exists: %s" % src, cleanup=False) os.makedirs(src) # get definition managed_objects to collect d = Definitions.load(definition=s.definition) if not d.exists(): return fail("unable to read definition: %s" % s.definition) managed_objects = d.get_managed_objects() # ensure all required objects are within definition. If not add them for r in Remap.REQUIRED_MANAGED_OBJECTS: if r not in managed_objects: logger.debug("adding required mo to definition: %s" % r) mo = ManagedObjects(classname=r) mo.analyzer = "exclude" managed_objects[mo.classname] = mo.to_json() # create apic session apic = aci_utils.get_apic_session(s.fabric) if apic is None: return fail("unable to connect to %s apic" % s.fabric) ret = aci_utils.get_class(apic, "infraCont") if ret is not None and len(ret)>0 and "infraCont" in ret[0] and \ "attributes" in ret[0]["infraCont"] and \ "fbDmNm" in ret[0]["infraCont"]["attributes"]: s.fabric_domain = ret[0]["infraCont"]["attributes"]["fbDmNm"] complete_count = 0 for classname in managed_objects: complete_count += 1 o = managed_objects[classname] if len(o["classname"]) == 0: logger.warn("%s ignoring empty classname %s" % (s.definition, o["classname"])) continue _t = time.time() response_classes = [] logger.debug("getting data for class %s" % o["classname"]) if o["pseudo"]: # for pseudo mo, collect each of the classnames defined in the # corresponding customer analyzer. If no customer analyzer then # no objects will be collected... if o["analyzer"] in ManagedObjects.ANALYZERS and \ "classnames" in ManagedObjects.ANALYZERS[o["analyzer"]]: ret = [] for ac in ManagedObjects.ANALYZERS[ o["analyzer"]]["classnames"]: order = "%s.%s" % (ac, "dn") # static to dn for now sret = aci_utils.get_class(apic, ac, orderBy=order) if sret is not None: ret += sret response_classes.append(ac) elif len(o["uri"]) > 0: ret = aci_utils.get(apic, o["uri"]) response_classes.append(o["classname"]) else: order = "%s.%s" % (o["classname"], o["key"]) ret = aci_utils.get_class(apic, o["classname"], orderBy=order) response_classes.append(o["classname"]) s.wait_time += abs(time.time() - _t) # if failed to get object, might not exists on current version of # code. just continue if ret is None: progress(complete_count, len(managed_objects) + 1) continue # need to parse each received object and group based on node-id cc_match = None # cache expected response_classes value (usually the # same for all objects expect for pseudo case) nodes = {} for c in ret: node = "0" if type(c) is dict and len(c) >= 1: cc = None # check for each possible response_class in returned object if cc_match is not None and cc_match in c: cc = c[cc_match] else: for rclass in response_classes: if rclass in c: cc_match = rclass cc = c[cc_match] break else: # check if 'children' is present and first level # child has corresponding object. before child # check, try to extract node-id (i.e., stats # object node-id from original parent object) tc = c[c.keys()[0]] if "attributes" in tc and "dn" in tc[ "attributes"]: r = reg.search(tc["attributes"]["dn"]) if r is not None: node = r.group("node") if "children" in tc: for child in tc["children"]: if rclass in child: cc = child[rclass] c = child break if cc is None: logger.debug("failed to extract data for %s from %s" % (o["classname"], c)) continue if type(cc) is dict and len( cc) >= 1 and "attributes" in cc: if "dn" in cc["attributes"]: r = reg.search(cc["attributes"]["dn"]) if r is not None: node = r.group("node") if node not in nodes: nodes[node] = [] if node not in s.nodes: s.nodes.append(node) nodes[node].append(c) continue logger.debug("skipping unsupported object: %s" % c) # for each node, write results to file for n in nodes: json_write("%s/node-%s/%s" % (src, n, o["classname"]), nodes[n]) # check for abort and then update progress progress(complete_count, len(managed_objects) + 1) # sort nodes before saving s.nodes = sorted(s.nodes) json_write("%s/snapshot" % src, s.to_json()) json_write("%s/definition" % src, { "definition": s.definition, "managed_objects": managed_objects }) # bundle all files ret = aci_utils.run_command("cd %s ; tar -zcvf bundle.tgz ./*" % src) if ret is None: return fail("failed to create snapshot bundle") # get md5 for bundle md5 = aci_utils.get_file_md5("%s/bundle.tgz" % src) if md5 is None: return fail("failed to calculate bundle md5") json_write("%s/md5checksum" % src, {"md5checksum": md5}) # create final bundle with bundle.tgz and md5checksum.json cmd = "cd %s ; tar --force-local -zcvf %s " % (src, filename) cmd += "./bundle.tgz ./md5checksum.json" if aci_utils.run_command(cmd) is None: return fail("failed to create snapshot bundle") # to prevent race condition of 'delete' operation during compression, # perform one last abort check before moving complete file to dst abort_snapshot() # create the directory if not present if not os.path.isdir(dst): os.makedirs(dst) # move final bundle to dst directory and cleanup tmp directory if aci_utils.run_command("mv %s/%s %s/" % (src, filename, dst)): return fail("failed to save snapshot bundle") tmp_cleanup() # after everything is complete, update progress to 100% and complete s.progress = 1.0 s.status = "complete" s.filename = "%s/%s" % (dst, filename) s.filesize = os.path.getsize("%s/%s" % (dst, filename)) s.total_time = abs(time.time() - ts) if not s.save(): logger.warn("failed to save snapshot status") logger.debug("snapshot %s complete: %s " % (snapshot_id, s)) except Exception as e: logger.debug(traceback.format_exc()) fail("unexpected error occurred: %s" % e)
def gen_gif(path_in, pattern='.*', fn_out='movie.gif', duration=0.5, file_range=None, repeat={}, path_out=None, user_confirm=True): """Generate a gif from a collection of images in a given directory. path_in: path of input images pattern regular expression matching file to include in gif fn_out filename for output gif duration duration between frames in seconds file_range replaces "{number}" in pattern with re matching a range of numbers repeat dict of frame numbers and the number of times those frames should be repeated path_out directory to write gif to """ assert os.path.isdir(path_in) if path_out is None: path_out = path_in assert os.path.isdir(path_out) if (file_range is not None) and ('{range}' in fn_out): fn_out.format(range='{}-{}'.format(file_range[0], file_range[1])) if file_range is not None: assert '{number}' in pattern, 'Include "{number}" in pattern when using file range' pattern = pattern.format(number=regexp_range(*file_range)) filenames = fn_filter(path_in, pattern) filenames = sorted(filenames) nframes = len(filenames) assert nframes > 0, 'No frames to create gif from' if -1 in repeat.keys( ): # If repeating final frame, replace '-1' with index repeat[nframes - 1] = repeat[-1] repeat.pop(-1) if user_confirm: print('{} frames will be combined into gif in: {}'.format( nframes, os.path.join(path_in, fn_out))) if nframes < 60: pprint(filenames) choice = input('Proceed? [y/n]: ') if not choice == 'y': print('gif was not produced') return ## return from function without renaming with imageio.get_writer(os.path.join(path_out, fn_out), mode='I', format='GIF-FI', duration=0.12) as writer: # duration = 0.4 for i, filename in enumerate(filenames): image = imageio.imread(os.path.join(path_in, filename)) writer.append_data(image) if repeat is not None and i in repeat.keys(): for j in np.arange(repeat[i]): writer.append_data(image) print('Wrote gif containing {} frames to: {}'.format( nframes, os.path.join(path_in, fn_out)))
def execute_compare(compare_id): """ perform comparison between two snapshots """ logger.debug("execute compare for: %s" % compare_id) ts = time.time() c = Compare.load(_id=compare_id) if not c.exists(): logger.warn("compare object %s not found" % compare_id) return # init progress, error, status to block second process from running c.progess = 0.0 c.start_time = ts c.total_time = 0 c.error = "" c.status = "running" c.save() # delete any previous CompareResults CompareResults.delete(_filters={"compare_id":c._id}) # create a working directory to extract objects config = get_app_config() tmp = config.get("TMP_DIR", "/tmp/") tmp = "%s/compare.%s.%s" % (tmp, c._id, int(ts)) # update state with error message and error status def fail(msg="", cleanup=True): if len(msg)>0: c.error = msg c.status = "error" c.total_time = abs(time.time() - ts) logger.debug("compare %s failed: %s" % (c._id, c.error)) if not c.save(): logger.warn("failed to save compare %s state"%c._id) if cleanup: tmp_cleanup() return # clean up working directory def tmp_cleanup(): if os.path.isdir(tmp): logger.debug("cleanup directory %s" % tmp) try: shutil.rmtree(tmp) except Exception as e: logger.warn("failed to cleanup directory(%s): %s"%(tmp,e)) else: logger.debug("cleanup directory %s not found" % tmp) # update snapshot progress percentage def progress(i, total): abort_compare() # check for abort signal before progress update if total>0: p = round(i/(1.0*total), 2) else: p = 0 if p > 1: p = 1 c.progress = p c.total_time = abs(time.time() - ts) if not c.save(): logger.warn("failed to save compare %s state"%c._id) logger.debug("progress %s/%s = %s" % (i, total, p)) return # user can force compare execution to stop by deleting the object # abort_compare is checked at a regular interval. If compare was deleted # then perform cleanup and return True, else return False def abort_compare(): _s = Compare.load(_id=c._id) if not _s.exists() or _s.status == "abort": logger.debug("compare object %s has been aborted"%c._id) tmp_cleanup() raise Exception("compare operation %s aborted" % c._id) # return dict representation of json in provided filename def get_json_data(fname): with open(fname, "r") as f: return json.load(f) # load snapshot1 (s1) and snapshot2 (s2) along with definition in compare object s1 = Snapshots.load(_id=c.snapshot1) s2 = Snapshots.load(_id=c.snapshot2) d = Definitions.load(definition=c.definition) classnames = [] if not s1.exists(): return fail("snapshot %s not found" % c.snapshot1) if not s2.exists(): return fail("snapshot %s not found" % c.snapshot2) if not s1.status=="complete": return fail("snapshot %s not complete"%c.snapshot1) if not s2.status=="complete": return fail("snapshot %s not complete"%c.snapshot2) if not d.exists(): logger.debug("unknown definition provided in compare request: '%s'", c.definition) else: classnames = d.get_managed_objects() try: # create tmp directory for extracting files if os.path.exists(tmp): return fail("tmp directory already exists: %s"%tmp, cleanup=False) logger.debug("make directory: %s" % tmp) os.makedirs(tmp) # extract snaphots to tmp directories for index, s in enumerate([s1, s2]): td = "%s/%s/%s" % (tmp, index, s._id) logger.debug("make directory: %s" % td) os.makedirs(td) if not os.path.exists(s.filename): return fail("snapshot file %s not found" % s.filename) shutil.copy(s.filename, td) fname = s.filename.split("/")[-1] # extract snapshot contents cmd = "cd %s ; tar --force-local -zxf %s" % (td, fname) if aci_utils.run_command(cmd) is None: return fail("failed to extract snapshot %s" % s.id) for f in ["bundle.tgz", "md5checksum.json"]: if not os.path.exists("%s/%s" % (td, f)): return fail("snapshot %s missing required file %s"%(s.id,f)) # check bundle against md5 embedded = get_json_data("%s/md5checksum.json" % td)["md5checksum"] md5 = aci_utils.get_file_md5("%s/bundle.tgz" % td) if md5 is None or md5!=embedded: return fail("snapshot %s invalid md5(%s) expected(%s)" % ( s.id, md5, embedded)) # extract bundle cmd = "cd %s ; tar -zxf %s" % (td, "bundle.tgz") if aci_utils.run_command(cmd) is None: return fail("failed to extract bundle %s" % s.id) # get the union of the list of classes from both s1 and s2 definitions, # use the managed_object definition from s2 (should always be the same) # use the definition in s2 for list of objects, attributes, and flags # if the classname is not present in definition from s1 then skip it js1 = get_json_data("%s/0/%s/definition.json" % (tmp, s1._id)) js2 = get_json_data("%s/1/%s/definition.json" % (tmp, s2._id)) if len(js1)==0: return fail("invalid definition in snapshot %s"%s1._id) if len(js2)==0: return fail("invalid definition in snapshot %s"%s2._id) s1_classes = [cn for cn in js1["managed_objects"].keys()] managed_objects = {} for classname in js2["managed_objects"]: if classname not in s1_classes: logger.debug("skipping %s not in snapshot1" % classname) continue managed_objects[classname] = js2["managed_objects"][classname] # build list of all nodes (join of s1 and s2 nodes) nodes =[ int(x) for x in sorted(list(set().union(s1.nodes, s2.nodes)))] # create pool to perform work pool = Pool(processes=config.get("MAX_POOL_SIZE", 4)) filtered_nodes = [] for n in nodes: if len(c.nodes)==0 or n in c.nodes: filtered_nodes.append(n) # build out work that needs to be completed class_work = [] # list of classes to perform 'class' work analyzer_work = [] # list of customer analyzers to work for classname in managed_objects: mo = managed_objects[classname] if len(classnames)>0 and classname not in classnames: continue if mo["analyzer"]=="" or mo["analyzer"]=="default": class_work.append(classname) for analyzer in ManagedObjects.ANALYZERS: # skip empty analyzer and default analyzer as they are 'class-work' if len(analyzer)==0 or analyzer=="default": continue # if a filter is set on the comparison for specific classname, then # ensure should be the name of the analyzer if len(classnames)==0 or analyzer in classnames: analyzer_work.append(analyzer) # think of progress as the sum of all tasks that need to be completed. # one task for each class in 'class_work' per node plus # one task for each analyzer in 'analyzer_work' per node plus # two task per node to for remapping (one for each snapshot) total_progress =len(class_work)*len(filtered_nodes) + \ len(analyzer_work)*len(filtered_nodes) + \ len(filtered_nodes)*2 current_progress = 0 progress(current_progress, total_progress) # perform comparision for each node for i,n in enumerate(filtered_nodes): if n!="0" and c.remap: # build remap for s1 and s2 r1 = Remap(n, "%s/0/%s/node-%s" % (tmp, s1._id, n)) current_progress+=1 r2 = Remap(n, "%s/1/%s/node-%s" % (tmp, s2._id, n)) current_progress+=1 else: # create disabled remap objects r1 = Remap(n) r2 = Remap(n) current_progress+=2 progress(current_progress, total_progress) work = [] # build out classwork for classname in class_work: o = managed_objects[classname] cn = "node-%s/%s.json" % (n, classname) f1 = "%s/0/%s/%s" % (tmp, s1._id, cn) f2 = "%s/1/%s/%s" % (tmp, s2._id, cn) work.append(("class", c, o, f1, f2, r1, r2)) # build out analyzer work f1 = "%s/0/%s/node-%s/" % (tmp, s1._id, n) f2 = "%s/1/%s/node-%s/" % (tmp, s2._id, n) for analyzer in analyzer_work: work.append(("custom", analyzer, c, f1, f2, r1, r2 )) # start the work using multiprocessing or serialize if c.serialize: for w in work: generic_compare(w) current_progress+=1 progress(current_progress, total_progress) else: pool.map(generic_compare, work) current_progress+= len(work) progress(current_progress, total_progress) # cleanup tmp extracted files tmp_cleanup() # create totals by summing over current compareResults counters = ["created", "modified", "deleted", "equal"] base = {"created":0, "modified":0, "deleted":0, "equal":0} total = copy.deepcopy(base) per_class = {} # indexed per classname per_node = {} # indexed per node_id results = CompareResults.read(compare_id=c._id) for o in results["objects"]: if o["classname"] not in per_class: per_class[o["classname"]] = copy.deepcopy(base) per_class[o["classname"]]["classname"] = o["classname"] per_class[o["classname"]]["name"] = o["name"] if o["node_id"] not in per_node: per_node[o["node_id"]] = copy.deepcopy(base) per_node[o["node_id"]]["node_id"] = o["node_id"] for counter in counters: per_class[o["classname"]][counter]+= o["total"][counter] per_node[o["node_id"]][counter]+= o["total"][counter] total[counter]+= o["total"][counter] # save totals c.total = total c.total_per_class = [per_class[k] for k in sorted(per_class.keys())] c.total_per_node = [per_node[k] for k in sorted(per_node.keys())] # after everything is complete, update progress to 100% and complete c.progress = 1.0 c.status = "complete" c.total_time = abs(time.time() - ts) if not c.save(): logger.warn("failed to save compare status") logger.debug("compare complete: %s " % c._id) except Exception as e: logger.debug(traceback.format_exc()) fail("unexpected error occurred: %s" % e)