def comparisons(self, cover, compound_length) : CP = predicates.CompoundPredicate block_index = {} for predicate, blocks in viewitems(cover): block_index[predicate] = {} for block_id, blocks in viewitems(blocks) : for id in self._blocks(blocks) : block_index[predicate].setdefault(id, set()).add(block_id) compounder = self.Compounder(cover, block_index) comparison_count = {} simple_predicates = sorted(cover, key=str) for i in range(2, compound_length+1) : for combo in itertools.combinations(simple_predicates, i) : comparison_count[CP(combo)] = sum(self.pairs(ids) for ids in viewvalues(compounder(combo))) for pred in simple_predicates : comparison_count[pred] = sum(self.pairs(ids) for ids in viewvalues(cover[pred])) return comparison_count
def update_category(self, category, samples_and_values): """Update an existing column Parameters ---------- category : str The category to update samples_and_values : dict A mapping of {sample_id: value} Raises ------ QiitaDBUnknownIDError If a sample_id is included in values that is not in the template QiitaDBColumnError If the column does not exist in the table. This is implicit, and can be thrown by the contained Samples. ValueError If one of the new values cannot be inserted in the DB due to different types """ with TRN: if not set(self.keys()).issuperset(samples_and_values): missing = set(self.keys()) - set(samples_and_values) table_name = self._table_name(self._id) raise QiitaDBUnknownIDError(missing, table_name) for k, v in viewitems(samples_and_values): sample = self[k] sample.setitem(category, v) try: TRN.execute() except ValueError as e: # catching error so we can check if the error is due to # different column type or something else value_types = set(type_lookup(type(value)) for value in viewvalues(samples_and_values)) sql = """SELECT udt_name FROM information_schema.columns WHERE column_name = %s AND table_schema = 'qiita' AND (table_name = %s OR table_name = %s)""" TRN.add(sql, [category, self._table, self._table_name(self._id)]) column_type = TRN.execute_fetchlast() if any([column_type != vt for vt in value_types]): value_str = ", ".join([str(value) for value in viewvalues(samples_and_values)]) value_types_str = ", ".join(value_types) raise ValueError( 'The new values being added to column: "%s" are "%s" ' '(types: "%s"). However, this column in the DB is of ' 'type "%s". Please change the values in your updated ' "template or reprocess your template." % (category, value_str, value_types_str, column_type) ) raise e
def _checkData(self, data_1, data_2): if len(data_1) == 0: raise ValueError("Dictionary of records from first dataset is empty.") elif len(data_2) == 0: raise ValueError("Dictionary of records from second dataset is empty.") self.data_model.check(next(iter(viewvalues(data_1)))) self.data_model.check(next(iter(viewvalues(data_2))))
def factor_one_bit(tree): if isinstance(tree, set): return tree new_keys = defaultdict(lambda: defaultdict(dict)) if len(tree) == 1: return tree for k, v in viewitems(tree): if k == "mn": new_keys[k] = v continue l, fmask, fbits, fname, flen = k if flen is not None or l <= 1: new_keys[k] = v continue cfmask = fmask >> (l - 1) nfmask = fmask & ((1 << (l - 1)) - 1) cfbits = fbits >> (l - 1) nfbits = fbits & ((1 << (l - 1)) - 1) ck = 1, cfmask, cfbits, None, flen nk = l - 1, nfmask, nfbits, fname, flen if nk in new_keys[ck]: raise NotImplementedError('not fully functional') new_keys[ck][nk] = v for k, v in list(viewitems(new_keys)): new_keys[k] = factor_one_bit(v) # try factor sons if len(new_keys) != 1: return new_keys subtree = next(iter(viewvalues(new_keys))) if len(subtree) != 1: return new_keys if next(iter(subtree)) == 'mn': return new_keys return new_keys
def log(self, input_count, batch_count, additional_values): logdict = OrderedDict() delta_t = time.time() - self.last_time delta_count = input_count - self.last_input_count self.last_time = time.time() self.last_input_count = input_count logdict['time_spent'] = delta_t logdict['cumulative_time_spent'] = time.time() - self.start_time logdict['input_count'] = delta_count logdict['cumulative_input_count'] = input_count logdict['cumulative_batch_count'] = batch_count if delta_t > 0: logdict['inputs_per_sec'] = delta_count / delta_t else: logdict['inputs_per_sec'] = 0.0 for k in sorted(viewkeys(additional_values)): logdict[k] = additional_values[k] # Write the headers if they are not written yet if self.headers is None: self.headers = list(viewkeys(logdict)) self.logstr(",".join(self.headers)) self.logstr(",".join(str(v) for v in viewvalues(logdict))) for logger in self.external_loggers: try: logger.log(logdict) except Exception as e: logging.warn( "Failed to call ExternalLogger: {}".format(e), e)
def cb_arm_fix_call(mn, cur_bloc, loc_db, offsets_to_dis, *args, **kwargs): """ for arm: MOV LR, PC LDR PC, [R5, 0x14] * is a subcall * """ if len(cur_bloc.lines) < 2: return l1 = cur_bloc.lines[-1] l2 = cur_bloc.lines[-2] if l1.name != "LDR": return if l2.name != "MOV": return values = viewvalues(mn.pc) if not l1.args[0] in values: return if not l2.args[1] in values: return loc_key_cst = loc_db.get_or_create_offset_location(l1.offset + 4) cur_bloc.add_cst(loc_key_cst, AsmConstraint.c_next) offsets_to_dis.add(l1.offset + 4)
def aggressive_coalesce_block(self): """Try to coalesce phi var with their pre/post variables""" ircfg = self.ssa.graph # Run coalesce on the post phi parallel copy for irblock in viewvalues(ircfg.blocks): if not irblock_has_phi(irblock): continue parallel_copies = {} for dst in self.phi_destinations[irblock.loc_key]: parallel_copies[dst] = self.phi_new_var[dst] self.aggressive_coalesce_parallel_copy(parallel_copies, None) # Run coalesce on the pre phi parallel copy # Stand for the virtual parallel copies at the end of Phi's block # parents parent_to_parallel_copies = {} for dst in irblock[0]: new_var = self.phi_new_var[dst] for parent, src in self.phi_parent_sources[dst]: parent_to_parallel_copies.setdefault(parent, {})[new_var] = src for parent, parallel_copies in viewitems(parent_to_parallel_copies): self.aggressive_coalesce_parallel_copy(parallel_copies, parent)
def lib_get_add_func(self, libad, imp_ord_or_name, dst_ad=None): if not libad in viewvalues(self.name2off): raise ValueError('unknown lib base!', hex(libad)) # test if not ordinatl # if imp_ord_or_name >0x10000: # imp_ord_or_name = vm_get_str(imp_ord_or_name, 0x100) # imp_ord_or_name = imp_ord_or_name[:imp_ord_or_name.find('\x00')] #/!\ can have multiple dst ad if not imp_ord_or_name in self.lib_imp2dstad[libad]: self.lib_imp2dstad[libad][imp_ord_or_name] = set() self.lib_imp2dstad[libad][imp_ord_or_name].add(dst_ad) if imp_ord_or_name in self.lib_imp2ad[libad]: return self.lib_imp2ad[libad][imp_ord_or_name] # log.debug('new imp %s %s' % (imp_ord_or_name, dst_ad)) ad = self.libbase2lastad[libad] self.libbase2lastad[libad] += 0x10 # arbitrary self.lib_imp2ad[libad][imp_ord_or_name] = ad name_inv = dict( (value, key) for key, value in viewitems(self.name2off) ) c_name = canon_libname_libfunc(name_inv[libad], imp_ord_or_name) self.fad2cname[ad] = c_name self.cname2addr[c_name] = ad self.fad2info[ad] = libad, imp_ord_or_name return ad
def ShiftActivationDevices(model, activations, shifts): ''' Function to enable simple model-parallellism for data_parallel_model models. 'shifts' is a dictionary from_gpu -> to_gpu, and activations is a list of activation blobs (wout gpu_x/ prefix -- use GetActivationBlobs()). Operators handling these activations are shifted to the gpu declared in 'shifts'. Also related operators such as gradient operators will be moved. Appropriate copy-ops are inserted. This allows shifting memory usage from one gpu to another, enabling bigger models to be trained. ''' assert set(viewvalues(shifts)).intersection(set(viewkeys(shifts))) == set() for from_device, to_device in viewitems(shifts): log.info( "Shifting {} activations from {} --> {}". format(len(activations), from_device, to_device) ) _ShiftActivationDevices(model, activations, from_device, to_device) param_init_net, blob_to_device = core.InjectCrossDeviceCopies(model.param_init_net) net, _blob_to_device = core.InjectCrossDeviceCopies(model.net, blob_to_device) model.param_init_net = param_init_net model.net = net
def generateFeatureGroups(fgiContainer, linkageGroups, matchArr, timeKey, massKey, logMassKey, massScalingFactor): """ #TODO: docstring :param fgiContainer: :param linkageGroups: :returns: a list of ids of the newly generated :class:`Fgi` """ #Generate feature groups from the linked features newFgiIds = list() for linkageGroup in viewvalues(linkageGroups): fgiId = fgiContainer._getNextFgiId() fgi = fiGroupFromLinkageGroup(matchArr, linkageGroup, fgiId, timeKey, massKey ) fgiContainer.container[fgiId] = fgi fgi.metrics = clusterMetrics(matchArr[timeKey][linkageGroup], matchArr[logMassKey][linkageGroup], massScalingFactor=massScalingFactor ) fgi.rt = fgi.metrics['meanTime'] fgi.mz = fgi.metrics['meanMass'] newFgiIds.append(fgiId) return newFgiIds
def isolate_phi_nodes_block(self): """ Init structures and virtually insert parallel copy before/after each phi node """ ircfg = self.ssa.graph for irblock in viewvalues(ircfg.blocks): if not irblock_has_phi(irblock): continue for dst, sources in viewitems(irblock[0]): assert sources.is_op('Phi') new_var = self.create_copy_var(dst) self.phi_new_var[dst] = new_var var_to_parents = get_phi_sources_parent_block( self.ssa.graph, irblock.loc_key, sources.args ) for src in sources.args: parents = var_to_parents[src] self.new_var_to_srcs_parents.setdefault(new_var, set()).update(parents) for parent in parents: self.phi_parent_sources.setdefault(dst, set()).add((parent, src)) self.phi_destinations[irblock.loc_key] = set(irblock[0])
def treat_element(): "Display an element" global graphs, comments, sol_nb, settings, addr, ir_arch, ircfg try: graph = next(graphs) except StopIteration: comments = {} print("Done: %d solutions" % (sol_nb)) return sol_nb += 1 print("Get graph number %02d" % sol_nb) filename = os.path.join(tempfile.gettempdir(), "solution_0x%08x_%02d.dot" % (addr, sol_nb)) print("Dump the graph to %s" % filename) open(filename, "w").write(graph.graph.dot()) for node in graph.relevant_nodes: try: offset = ircfg.blocks[node.loc_key][node.line_nb].instr.offset except IndexError: print("Unable to highlight %s" % node) continue comments[offset] = comments.get(offset, []) + [node.element] idc.SetColor(offset, idc.CIC_ITEM, settings.color) if graph.has_loop: print('Graph has dependency loop: symbolic execution is inexact') else: print("Possible value: %s" % next(iter(viewvalues(graph.emul(ir_arch))))) for offset, elements in viewitems(comments): idc.MakeComm(offset, ", ".join(map(str, elements)))
def index(data, offset=0): if isIndexed(data, offset): return data else: data = dict(zip(itertools.count(offset), viewvalues(data))) return data
def test_collect_modules_provenance(self): metascript = self.prepare(name=MODULES) metascript.deployment._collect_modules_provenance() modules = {m.name for m in viewvalues(metascript.modules_store.store)} self.assertIn("ast", modules) self.assertIn("script", modules)
def modSymbolsFromLabelInfo(labelDescriptor): """Returns a set of all modiciation symbols which were used in the labelDescriptor :param labelDescriptor: :class:`LabelDescriptor` describes the label setup of an experiment :returns: #TODO: docstring """ modSymbols = set() for labelStateEntry in viewvalues(labelDescriptor.labels): for labelPositionEntry in viewvalues(labelStateEntry['aminoAcidLabels']): for modSymbol in aux.toList(labelPositionEntry): if modSymbol != '': modSymbols.add(modSymbol) return modSymbols
def _blockData(self, data_d): blocks = core.TempShelve('blocks') if not self.loaded_indices: self.blocker.indexAll(data_d) block_groups = itertools.groupby(self.blocker(viewitems(data_d)), lambda x: x[1]) for record_id, block in block_groups: record = data_d[record_id] block_ids = sorted(block_key for block_key, _ in block) while block_ids: id = block_ids.pop() if id in blocks: blocks[id] += [(record_id, record, set(block_ids))] else: blocks[id] = [(record_id, record, set(block_ids))] if not self.loaded_indices: self.blocker.resetIndices() for block in viewvalues(blocks): if len(block) > 1: yield block blocks.close()
def indexAll(self, data_d) : for field in self.index_fields : unique_fields = {record[field] for record in viewvalues(data_d) if record[field]} self.index(unique_fields, field)
def _blockData(self, data_d): blocks = defaultdict(dict) for field in self.blocker.index_fields : unique_fields = {record[field] for record in viewvalues(data_d) if record[field]} self.blocker.index(unique_fields, field) for block_key, record_id in self.blocker(viewitems(data_d)) : blocks[block_key][record_id] = data_d[record_id] self.blocker.resetIndices() blocks = (records for records in blocks.values() if len(records) > 1) blocks = {frozenset(d.keys()) : d for d in blocks} blocks = blocks.values() for block in self._redundantFree(blocks) : yield block
def can_be_updated(self, columns): """Gets if the template can be updated Parameters ---------- columns : set A set of the names of the columns to be updated Returns ------- bool If the template can be updated Notes ----- The prep template can be updated when (1) it has no preprocessed data or the prep template data-type is not part of TARGET_GENE_DATA_TYPES, (2) if is part of TARGET_GENE_DATA_TYPES then we will only update if the columns being updated are not part of PREP_TEMPLATE_COLUMNS_TARGET_GENE """ if (not self.preprocessed_data or self.data_type() not in TARGET_GENE_DATA_TYPES): return True tg_columns = set(chain.from_iterable( [v.columns for v in viewvalues(PREP_TEMPLATE_COLUMNS_TARGET_GENE)])) if not columns & tg_columns: return True return False
def all_args(self): """List arguments of function call""" return list(itertools.chain( itertools.chain.from_iterable(self.args), self.starargs, self.kwargs, itertools.chain.from_iterable(viewvalues(self.keywords)) ))
def test_collect_environment_provenance(self): metascript = self.prepare() metascript.deployment._collect_environment_provenance() env = {e.name: e.value for e in viewvalues(metascript.environment_attrs_store.store)} self.assertIn("PWD", env) self.assertIn("PYTHON_VERSION", env) self.assertIn("PYTHON_IMPLEMENTATION", env)
def check_dst_ad(self): for ad in self.lib_imp2dstad: all_ads = sorted(viewvalues(self.lib_imp2dstad[ad])) for i, x in enumerate(all_ads[:-1]): if x is None or all_ads[i + 1] is None: return False if x + 4 != all_ads[i + 1]: return False return True
def find_call(ircfg): """Returns (irb, index) which call""" for irb in viewvalues(ircfg.blocks): out = set() if len(irb) < 2: continue assignblk = irb[-2] for src in viewvalues(assignblk): if not isinstance(src, ExprOp): continue if not src.op.startswith('call_func'): continue out.add((irb.loc_key, len(irb) - 2)) if len(out) != 1: continue loc_key, index = out.pop() yield loc_key, index
def find_variables_rec(self, expr): """Recursive method called by find_variable to expand @expr. Set @var_names and @var_values. This implementation is faster than an expression visitor because we do not rebuild each expression. """ if (expr in self.var_asked): # Expr has already been asked if expr not in viewvalues(self._vars): # Create var identifier = m2_expr.ExprId( "%s%s" % ( self.var_prefix, next(self.var_indice) ), size = expr.size ) self._vars[identifier] = expr # Recursion stop case return else: # First time for @expr self.var_asked.add(expr) if isinstance(expr, m2_expr.ExprOp): for a in expr.args: self.find_variables_rec(a) elif isinstance(expr, m2_expr.ExprInt): pass elif isinstance(expr, m2_expr.ExprId): pass elif isinstance(expr, m2_expr.ExprLoc): pass elif isinstance(expr, m2_expr.ExprMem): self.find_variables_rec(expr.ptr) elif isinstance(expr, m2_expr.ExprCompose): for arg in expr.args: self.find_variables_rec(arg) elif isinstance(expr, m2_expr.ExprSlice): self.find_variables_rec(expr.arg) elif isinstance(expr, m2_expr.ExprCond): self.find_variables_rec(expr.cond) self.find_variables_rec(expr.src1) self.find_variables_rec(expr.src2) else: raise NotImplementedError("Type not handled: %s" % expr)
def blockTraining(pairs, predicate_set, eta=.1, epsilon=0, matching = "Dedupe"): ''' Takes in a set of training pairs and predicates and tries to find a good set of blocking rules. ''' blocker = blocking.Blocker(predicate_set) prepare_index(blocker, pairs, matching) if len(pairs['match']) < 50 : compound_length = 2 else : compound_length = 3 dupe_cover = cover(blocker, pairs['match'], compound_length) distinct_cover = cover(blocker, pairs['distinct'], compound_length) distinct_count = defaultdict(int, {pred : len(pairs) for pred, pairs in viewitems(distinct_cover)}) # Throw away the predicates that cover too many distinct pairs coverage_threshold = eta * len(pairs['distinct']) logger.info("coverage threshold: %s", coverage_threshold) dupe_cover = {pred : pairs for pred, pairs in viewitems(dupe_cover) if distinct_count[pred] < coverage_threshold} if not dupe_cover : raise ValueError(NO_PREDICATES_ERROR) uncoverable_dupes = set(pairs['match']) - set.union(*viewvalues(dupe_cover)) if len(uncoverable_dupes) > epsilon : logger.warning(OUT_OF_PREDICATES_WARNING) logger.debug(uncoverable_dupes) epsilon = 0 else : epsilon -= len(uncoverable_dupes) chvatal_set = greedy(dupe_cover.copy(), distinct_count, epsilon) dupe_cover = {pred : dupe_cover[pred] for pred in chvatal_set} final_predicates = tuple(dominating(dupe_cover)) logger.info('Final predicate set:') for predicate in final_predicates : logger.info(predicate) return final_predicates
def test_rnn(self): from caffe2.python import rnn_cell T = 5 model = model_helper.ModelHelper() seq_lengths, labels = \ model.net.AddExternalInputs( 'seq_lengths', 'labels', ) init_blobs = [] for i in range(2): hidden_init, cell_init = model.net.AddExternalInputs( "hidden_init_{}".format(i), "cell_init_{}".format(i) ) init_blobs.extend([hidden_init, cell_init]) model.param_init_net.ConstantFill([], ["input"], shape=[T, 4, 10]) output, last_hidden, _, last_state = rnn_cell.LSTM( model=model, input_blob="input", seq_lengths=seq_lengths, initial_states=init_blobs, dim_in=10, dim_out=[10, 10], scope="lstm1", forward_only=False, drop_states=True, return_last_layer_only=True, ) softmax, loss = model.net.SoftmaxWithLoss( [model.Flatten(output), "labels"], ['softmax', 'loss'], ) model.AddGradientOperators([loss]) blobs_before = count_blobs(model.net.Proto()) optim_proto = memonger.share_grad_blobs( model.net, ["loss"], set(viewvalues(model.param_to_grad)), "", share_activations=True, dont_share_blobs=set(), ) blobs_after = count_blobs(optim_proto) self.assertLess(blobs_after, blobs_before) # Run once to see all blobs are set up correctly for init_blob in init_blobs: workspace.FeedBlob(init_blob, np.zeros( [1, 4, 10], dtype=np.float32 )) workspace.FeedBlob("seq_lengths", np.array([T] * 4, dtype=np.int32)) workspace.FeedBlob("labels", np.random.rand(T).astype(np.int32)) workspace.RunNetOnce(model.param_init_net) workspace.RunNetOnce(model.net)
def unindex(self, data): # pragma: no cover for field in self.blocker.index_fields: self.blocker.unindex((record[field] for record in viewvalues(data)), field) for block_key, record_id in self.blocker(viewitems(data)): try: del self.blocked_records[block_key][record_id] except KeyError: pass
def find_defaults(d, op): if isinstance(d, list): for i in d: find_defaults(i, op) elif isinstance(d, dict): if "default" in d: op(d) else: for i in viewvalues(d): find_defaults(i, op)
def _cleanup_record(cls, elt): for k, v in viewitems(elt): if isinstance(v, list) and len(v) == 1 and \ isinstance(v[0], dict) and \ all(x is None for x in viewvalues(v[0])): elt[k] = [] cls.from_dbdict(cls._get_props(elt["elt"])) new_meta = {} if isinstance(elt["meta"], list): for rec in elt["meta"]: if rec["info"] is None and rec["link"] is None: continue info = rec["info"] or {} info_props = cls._get_props(info) link = rec["link"] or {} link_tag = link.get("type", link.get("labels", [""])[0]).lower() link_props = cls._get_props(link) key = "%s%s" % ( "_".join(label for label in cls._get_labels(info, info_props) if label != "Intel"), "_%s" % link_tag if link_tag else "" ) new_data = dict(("%s_%s" % (link_tag, k), v) for k, v in viewitems(link_props)) new_data.update(info_props) new_meta.setdefault(key, []).append(new_data) if new_meta: elt["meta"] = new_meta for reclist in viewvalues(new_meta): for rec in reclist: cls.from_dbdict(rec) if ("times" in elt["meta"] and elt["meta"]["times"] and isinstance(elt["meta"]["times"], list) and isinstance(elt["meta"]["times"][0], float)): elt["meta"]["times"] = [datetime.fromtimestamp(val) for val in elt["meta"]["times"]] if not elt["meta"]: del elt["meta"]
def _display_honeyd_conf(host, honeyd_routes, honeyd_entries, out=sys.stdout): addr = utils.int2ip(host['addr']) hname = "host_%s" % addr.replace('.', '_') out.write("create %s\n" % hname) defaction = HONEYD_DEFAULT_ACTION if 'extraports' in host: extra = host['extraports'] defaction = max( max(viewvalues(extra), key=lambda state: viewitems(state['total'])['reasons']), key=lambda reason: reason[1], )[0] defaction = HONEYD_ACTION_FROM_NMAP_STATE.get(defaction) out.write('set %s default tcp action %s\n' % (hname, defaction)) for p in host.get('ports', []): try: out.write('add %s %s port %d %s\n' % ( hname, p['protocol'], p['port'], _nmap_port2honeyd_action(p), )) except KeyError: # let's skip pseudo-port records that are only containers for host # scripts. pass if 'traces' in host and len(host['traces']) > 0: trace = max(host['traces'], key=lambda x: len(x['hops']))['hops'] if trace: trace.sort(key=lambda x: x['ttl']) curhop = trace[0] honeyd_entries.add(curhop['ipaddr']) for t in trace[1:]: key = (curhop['ipaddr'], t['ipaddr']) latency = max(t['rtt'] - curhop['rtt'], 0) route = honeyd_routes.get(key) if route is None: honeyd_routes[key] = { 'count': 1, 'high': latency, 'low': latency, 'mean': latency, 'targets': set([host['addr']]) } else: route['targets'].add(host['addr']) honeyd_routes[key] = { 'count': route['count'] + 1, 'high': max(route['high'], latency), 'low': min(route['low'], latency), 'mean': (route['mean'] * route['count'] + latency) / (route['count'] + 1), 'targets': route['targets'], } curhop = t out.write('bind %s %s\n\n' % (addr, hname)) return honeyd_routes, honeyd_entries
def makeRequest_httplib(self, uri, data, verb, headers): """ Make a request to the remote database. for a give URI. The type of request will determine the action take by the server (be careful with DELETE!). Data should be a dictionary of {dataname: datavalue}. Returns a tuple of the data from the server, decoded using the appropriate method the response status and the response reason, to be used in error handling. You can override the method to encode/decode your data by passing in an encoding/decoding function to this method. Your encoded data must end up as a string. """ if verb == 'GET' and data: uri = "%s?%s" % (uri, data) assert isinstance(data, (str, bytes)), \ "Data in makeRequest is %s and not encoded to a string" % type(data) # And now overwrite any headers that have been passed into the call: # WARNING: doesn't work with deplate so only accept gzip headers["Accept-Encoding"] = "gzip,identity" # httplib2 will allow sockets to close on remote end without retrying # try to send request - if this fails try again - should then succeed try: conn = self._getURLOpener() response, result = conn.request(uri, method=verb, body=data, headers=headers) if response.status == 408: # timeout can indicate a socket error raise socket.error except ServerNotFoundError as ex: # DNS cannot resolve this domain name, let's call it 'Service Unavailable' e = HTTPException() setattr(e, 'url', uri) setattr(e, 'status', 503) setattr(e, 'reason', 'Service Unavailable') setattr(e, 'result', str(ex)) raise e except (socket.error, AttributeError): self['logger'].warn("Http request failed, retrying once again..") # AttributeError implies initial connection error - need to close # & retry. httplib2 doesn't clear httplib state before next request # if this is threaded this may spoil things # only have one endpoint so don't need to determine which to shut for con in viewvalues(conn.connections): con.close() conn = self._getURLOpener() # ... try again... if this fails propagate error to client try: response, result = conn.request(uri, method=verb, body=data, headers=headers) except AttributeError: msg = traceback.format_exc() # socket/httplib really screwed up - nuclear option conn.connections = {} raise socket.error('Error contacting: %s: %s' % (self.getDomainName(), msg)) if response.status >= 400: e = HTTPException() setattr(e, 'req_data', data) setattr(e, 'req_headers', headers) setattr(e, 'url', uri) setattr(e, 'result', result) setattr(e, 'status', response.status) setattr(e, 'reason', response.reason) setattr(e, 'headers', response) raise e return result, response
def itervalues(self): for src in viewvalues(self._assigns): yield src
def visit_group(self, group): """Visit Group Node (Visitor Pattern)""" result = [] for element in viewvalues(group.nodes): result += element.visit(self) return result
def values(self): return viewvalues(self._d)
def __init__(self, custom_methods, *args, **kwargs): from miasm.jitter.loader.pe import vm_load_pe, vm_load_pe_libs,\ preload_pe, libimp_pe, vm_load_pe_and_dependencies from miasm.os_dep import win_api_x86_32, win_api_x86_32_seh methods = dict((name.encode(),func) for name, func in viewitems(win_api_x86_32.__dict__)) methods.update(custom_methods) super(OS_Win, self).__init__(methods, *args, **kwargs) # Import manager libs = libimp_pe() self.libs = libs win_api_x86_32.winobjs.runtime_dll = libs self.name2module = {} fname_basename = os.path.basename(self.fname).lower() # Load main pe with open(self.fname, "rb") as fstream: self.pe = vm_load_pe( self.jitter.vm, fstream.read(), load_hdr=self.options.load_hdr, name=self.fname, **kwargs ) self.name2module[fname_basename] = self.pe # Load library if self.options.loadbasedll: # Load libs in memory self.name2module.update( vm_load_pe_libs( self.jitter.vm, self.ALL_IMP_DLL, libs, self.modules_path, **kwargs ) ) # Patch libs imports for pe in viewvalues(self.name2module): preload_pe(self.jitter.vm, pe, libs) if self.options.dependencies: vm_load_pe_and_dependencies( self.jitter.vm, fname_basename, self.name2module, libs, self.modules_path, **kwargs ) win_api_x86_32.winobjs.current_pe = self.pe # Fix pe imports preload_pe(self.jitter.vm, self.pe, libs) # Library calls handler self.jitter.add_lib_handler(libs, methods) # Manage SEH if self.options.use_windows_structs: win_api_x86_32_seh.main_pe_name = fname_basename win_api_x86_32_seh.main_pe = self.pe win_api_x86_32.winobjs.hcurmodule = self.pe.NThdr.ImageBase win_api_x86_32_seh.name2module = self.name2module win_api_x86_32_seh.set_win_fs_0(self.jitter) win_api_x86_32_seh.init_seh(self.jitter) self.entry_point = self.pe.rva2virt( self.pe.Opthdr.AddressOfEntryPoint)
def _col_iterator(): for r_set in ALL_RESTRICTIONS: for restriction in viewvalues(r_set): for cols in viewkeys(restriction.columns): yield cols
def test_gradient_optim(self, input_dim, output_dim, batch_size): m = model_helper.ModelHelper() with core.NameScope("name_x"): fc1 = brew.fc(m, "data", "fc1", dim_in=input_dim, dim_out=output_dim) fc2 = brew.fc(m, fc1, "fc2", dim_in=output_dim, dim_out=output_dim) fc3 = brew.fc(m, fc2, "fc3", dim_in=output_dim, dim_out=output_dim) fc4 = brew.fc(m, fc3, "fc4", dim_in=output_dim, dim_out=output_dim) fc5 = brew.fc(m, fc4, "fc5", dim_in=output_dim, dim_out=output_dim) fc5.Relu([], fc5)\ .Softmax([], "pred") \ .LabelCrossEntropy(["label"], ["xent"]) \ .AveragedLoss([], "loss") input_to_grad = m.AddGradientOperators(["name_x/loss"]) blobs_before = count_blobs(m.net.Proto()) optim_proto = memonger.share_grad_blobs( m.net, ["name_x/loss"], set(viewvalues(m.param_to_grad)), "name_x/", share_activations=False, ) blobs_after = count_blobs(optim_proto) self.assertLess(blobs_after, blobs_before) optim_proto_wacts = memonger.share_grad_blobs( m.net, ["name_x/loss"], set(viewvalues(m.param_to_grad)), "name_x/", share_activations=True, dont_share_blobs=set([str(input_to_grad["name_x/fc1_w"])]), ) blobs_wact_optim = count_blobs(optim_proto_wacts) self.assertLessEqual(blobs_wact_optim, blobs_after) # Check that the last activations are not shared self.assertTrue(has_blob(optim_proto, "name_x/fc5")) self.assertTrue( has_blob(optim_proto_wacts, "name_x/fc5"), "Dont remap final activation", ) # Test networks produce exactly same gradients data = np.random.randn(batch_size, input_dim).astype(np.float32) label = np.random.randint(low=0, high=output_dim, size=(batch_size, )).astype(np.int32) workspace.RunNetOnce(m.param_init_net) workspace.FeedBlob("name_x/data", data) workspace.FeedBlob("name_x/label", label) workspace.RunNetOnce(m.net) loss = workspace.FetchBlob("name_x/loss") grad = workspace.FetchBlob(str(input_to_grad["name_x/fc1_w"])) workspace.RunNetOnce(optim_proto) optimized_loss = workspace.FetchBlob("name_x/loss") optimized_grad = workspace.FetchBlob(str( input_to_grad["name_x/fc1_w"])) np.testing.assert_almost_equal(loss, optimized_loss) np.testing.assert_almost_equal(grad, optimized_grad) workspace.FeedBlob(str(input_to_grad["name_x/fc1_w"]), np.array([0.0])) # Run with the forward optimization workspace.RunNetOnce(optim_proto_wacts) optimized_loss = workspace.FetchBlob("name_x/loss") optimized_grad = workspace.FetchBlob(str( input_to_grad["name_x/fc1_w"])) np.testing.assert_almost_equal(loss, optimized_loss) np.testing.assert_almost_equal(grad, optimized_grad)
def iter_ranges(self): for start, length in sorted(viewvalues(self.ranges)): yield utils.int2ip(start), utils.int2ip(start + length - 1)
def iter_nets(self): for start, length in sorted(viewvalues(self.ranges)): for net in utils.range2nets( (utils.int2ip(start), utils.int2ip(start + length - 1))): yield net
def iter_int_ranges(self): for start, length in sorted(viewvalues(self.ranges)): yield start, start + length - 1
def _add_global_constants(self, init_net): for initializer_op in viewvalues(self.global_constant_initializers): init_net._net.op.extend([initializer_op])
def get_graph(self): simplify = self.simplify dontmodstack = self.dontmodstack loadmemint = self.loadmemint type_graph = self.type_graph bin_str = "" for s in self.data.segments: bin_str += self.data.read(s.start, len(s)) # add padding between each segment if s.end != self.data.end: bin_str += '\x00' * (((s.end | 0xfff) + 1) - s.end) bs = bin_stream_str(input_str=bin_str, base_address=self.data.start) machine = Machine(archs[self.data.arch.name]) mdis = machine.dis_engine(bs) asmcfg = mdis.dis_multiblock(self.function.start) entry_points = set( [mdis.loc_db.get_offset_location(self.function.start)]) class IRADelModCallStack(machine.ira): def call_effects(self, addr, instr): assignblks, extra = super(IRADelModCallStack, self).call_effects(addr, instr) if not dontmodstack: return assignblks, extra out = [] for assignblk in assignblks: dct = dict(assignblk) dct = { dst: src for (dst, src) in viewitems(dct) if dst != self.sp } out.append(AssignBlock(dct, assignblk.instr)) return out, extra ir_arch = IRADelModCallStack(mdis.loc_db) ircfg = ir_arch.new_ircfg_from_asmcfg(asmcfg) for irb in list(viewvalues(ircfg.blocks)): irs = [] for assignblk in irb: new_assignblk = { expr_simp(dst): expr_simp(src) for dst, src in viewitems(assignblk) } irs.append(AssignBlock(new_assignblk, instr=assignblk.instr)) ircfg.blocks[irb.loc_key] = IRBlock(irb.loc_key, irs) head = list(entry_points)[0] if simplify: ircfg_simplifier = IRCFGSimplifierCommon(ir_arch) ircfg_simplifier.simplify(ircfg, head) if type_graph == TYPE_GRAPH_IR: return MiasmIRGraph(self.add_names(ircfg)) class IRAOutRegs(machine.ira): def get_out_regs(self, block): regs_todo = super(IRAOutRegs, self).get_out_regs(block) out = {} for assignblk in block: for dst in assignblk: reg = self.ssa_var.get(dst, None) if reg is None: continue if reg in regs_todo: out[reg] = dst return set(viewvalues(out)) # Add dummy dependency to uncover out regs affectation for loc in ircfg.leaves(): irblock = ircfg.blocks.get(loc) if irblock is None: continue regs = {} for reg in ir_arch.get_out_regs(irblock): regs[reg] = reg assignblks = list(irblock) new_assiblk = AssignBlock(regs, assignblks[-1].instr) assignblks.append(new_assiblk) new_irblock = IRBlock(irblock.loc_key, assignblks) ircfg.blocks[loc] = new_irblock class CustomIRCFGSimplifierSSA(IRCFGSimplifierSSA): def do_simplify(self, ssa, head): modified = super(CustomIRCFGSimplifierSSA, self).do_simplify(ssa, head) if loadmemint: modified |= load_from_int(ssa.graph, bs, is_addr_ro_variable) return modified def simplify(self, ircfg, head): ssa = self.ircfg_to_ssa(ircfg, head) ssa = self.do_simplify_loop(ssa, head) if type_graph == TYPE_GRAPH_IRSSA: ret = ssa.graph elif type_graph == TYPE_GRAPH_IRSSAUNSSA: ircfg = self.ssa_to_unssa(ssa, head) ircfg_simplifier = IRCFGSimplifierCommon(self.ir_arch) ircfg_simplifier.simplify(ircfg, head) ret = ircfg else: raise ValueError("Unknown option") return ret # dirty patch to synchronize nodes and blocks lists in ircfg nodes_to_del = [ node for node in ircfg.nodes() if not node in ircfg.blocks ] for node in nodes_to_del: ircfg.del_node(node) head = list(entry_points)[0] simplifier = CustomIRCFGSimplifierSSA(ir_arch) ircfg = simplifier.simplify(ircfg, head) return MiasmIRGraph(self.add_names(ircfg))
def create_qiime_mapping_file(self): """This creates the QIIME mapping file and links it in the db. Returns ------- filepath : str The filepath of the created QIIME mapping file Raises ------ ValueError If the prep template is not a subset of the sample template QiitaDBWarning If the QIIME-required columns are not present in the template Notes ----- We cannot ensure that the QIIME-required columns are present in the metadata map. However, we have to generate a QIIME-compliant mapping file. Since the user may need a QIIME mapping file, but not these QIIME-required columns, we are going to create them and populate them with the value XXQIITAXX. """ with qdb.sql_connection.TRN: rename_cols = { 'barcode': 'BarcodeSequence', 'primer': 'LinkerPrimerSequence', 'description': 'Description', } if 'reverselinkerprimer' in self.categories(): rename_cols['reverselinkerprimer'] = 'ReverseLinkerPrimer' new_cols = [ 'BarcodeSequence', 'LinkerPrimerSequence', 'ReverseLinkerPrimer' ] else: new_cols = ['BarcodeSequence', 'LinkerPrimerSequence'] # Retrieve the latest sample template # Since we sorted the filepath retrieval, the first result contains # the filepath that we want. `retrieve_filepaths` returns a # 3-tuple, in which the fp is the second element sample_template_fp = qdb.util.retrieve_filepaths( "sample_template_filepath", "study_id", self.study_id, sort='descending')[0][1] # reading files via pandas st = qdb.metadata_template.util.load_template_to_dataframe( sample_template_fp) pt = self.to_dataframe() st_sample_names = set(st.index) pt_sample_names = set(pt.index) if not pt_sample_names.issubset(st_sample_names): raise ValueError( "Prep template is not a sub set of the sample template, " "file: %s - samples: %s" % (sample_template_fp, ', '.join(pt_sample_names - st_sample_names))) mapping = pt.join(st, lsuffix="_prep") mapping.rename(columns=rename_cols, inplace=True) # Pre-populate the QIIME-required columns with the value XXQIITAXX index = mapping.index placeholder = ['XXQIITAXX'] * len(index) missing = [] for val in viewvalues(rename_cols): if val not in mapping: missing.append(val) mapping[val] = pd.Series(placeholder, index=index) if missing: warnings.warn( "Some columns required to generate a QIIME-compliant " "mapping file are not present in the template. A " "placeholder value (XXQIITAXX) has been used to populate " "these columns. Missing columns: %s" % ', '.join(sorted(missing)), qdb.exceptions.QiitaDBWarning) # Gets the orginal mapping columns and readjust the order to comply # with QIIME requirements cols = mapping.columns.values.tolist() cols.remove('BarcodeSequence') cols.remove('LinkerPrimerSequence') cols.remove('Description') new_cols.extend(cols) new_cols.append('Description') mapping = mapping[new_cols] # figuring out the filepath for the QIIME map file _id, fp = qdb.util.get_mountpoint('templates')[0] filepath = join( fp, '%d_prep_%d_qiime_%s.txt' % (self.study_id, self.id, strftime("%Y%m%d-%H%M%S"))) # Save the mapping file mapping.to_csv(filepath, index_label='#SampleID', na_rep='', sep='\t', encoding='utf-8') # adding the fp to the object self.add_filepath(filepath, fp_id=qdb.util.convert_to_id( "qiime_map", "filepath_type")) return filepath
def iter_addrs(self): for start, length in sorted(viewvalues(self.ranges)): for val in range(start, start + length): yield utils.int2ip(val)
loc_db = LocationDB() print('disasm...') cont = Container.from_stream(open(args.filename, 'rb'), loc_db) machine = Machine("x86_32") mdis = machine.dis_engine(cont.bin_stream, loc_db=loc_db) mdis.follow_call = True asmcfg = mdis.dis_multiblock(ad) print('ok') print('generating dataflow graph for:') ir_arch_analysis = machine.ira(loc_db) ircfg = ir_arch_analysis.new_ircfg_from_asmcfg(asmcfg) deadrm = DeadRemoval(ir_arch_analysis) for irblock in viewvalues(ircfg.blocks): print(irblock) if args.symb: block_flow_cb = intra_block_flow_symb else: block_flow_cb = intra_block_flow_raw gen_block_data_flow_graph(ir_arch_analysis, ircfg, ad, block_flow_cb) print('*' * 40) print(""" View with: dotty dataflow.dot or Generate ps with pdf:
def views(self): for t in viewvalues(self.tables): if t.isview: yield t
def test_gradient_optim_tree(self, input_dim, output_dim, batch_size): m = model_helper.ModelHelper() with core.NameScope("name_x"): fc1 = brew.fc(m, "data", "fc1", dim_in=input_dim, dim_out=output_dim) fc2 = brew.fc(m, fc1, "fc2", dim_in=output_dim, dim_out=output_dim) fc3 = brew.fc(m, fc2, "fc3", dim_in=output_dim, dim_out=output_dim) fc4 = brew.fc(m, fc3, "fc4", dim_in=output_dim, dim_out=output_dim) fc5 = brew.fc(m, fc4, "fc5", dim_in=output_dim, dim_out=output_dim) fc5.Relu([], fc5) \ .Softmax([], "pred1") \ .LabelCrossEntropy(["label"], ["xent1"]) \ .AveragedLoss([], "loss1") fc6 = brew.fc(m, fc5, "fc6", dim_in=output_dim, dim_out=output_dim) fc6.Relu([], fc6) \ .Softmax([], "pred2") \ .LabelCrossEntropy(["label"], ["xent2"]) \ .AveragedLoss([], "loss2") input_to_grad = m.AddGradientOperators( ["name_x/loss1", "name_x/loss2"]) blobs_before = count_blobs(m.net.Proto()) optim_proto = memonger.share_grad_blobs( m.net, ["name_x/loss1", "name_x/loss2"], set(viewvalues(m.param_to_grad)), "name_x", # "name_x//shared_gradinp_0_shared" if using "name_x/" share_activations=True, dont_share_blobs=set([ 'name_x/fc6', 'name_x/fc5', str(input_to_grad["name_x/fc1_w"]) ]), ) blobs_after = count_blobs(optim_proto) self.assertLess(blobs_after, blobs_before) self.assertTrue(has_blob(optim_proto, "name_x/fc6")) # Test networks produce exactly same gradients data = np.random.randn(batch_size, input_dim).astype(np.float32) label = np.random.randint(low=0, high=output_dim, size=(batch_size, )).astype(np.int32) workspace.RunNetOnce(m.param_init_net) workspace.FeedBlob("name_x/data", data) workspace.FeedBlob("name_x/label", label) workspace.RunNetOnce(m.net) loss1 = workspace.FetchBlob("name_x/loss1") loss2 = workspace.FetchBlob("name_x/loss2") grad = workspace.FetchBlob(str(input_to_grad["name_x/fc1_w"])) workspace.FeedBlob(str(input_to_grad["name_x/fc1_w"]), np.array([0.0])) workspace.RunNetOnce(optim_proto) optimized_loss1 = workspace.FetchBlob("name_x/loss1") optimized_loss2 = workspace.FetchBlob("name_x/loss2") optimized_grad = workspace.FetchBlob(str( input_to_grad["name_x/fc1_w"])) np.testing.assert_almost_equal(loss1, optimized_loss1) np.testing.assert_almost_equal(loss2, optimized_loss2) np.testing.assert_almost_equal(grad, optimized_grad)
def columns(self): for t in viewvalues(self.tables): for c in t.columns: yield c
def launch_depgraph(): global graphs, comments, sol_nb, settings, addr, ir_arch, ircfg # Get the current function addr = idc.get_screen_ea() func = ida_funcs.get_func(addr) # Init machine = guess_machine(addr=func.start_ea) mn, dis_engine, ira = machine.mn, machine.dis_engine, machine.ira bs = bin_stream_ida() mdis = dis_engine(bs, dont_dis_nulstart_bloc=True) ir_arch = ira(mdis.loc_db) # Populate symbols with ida names for ad, name in idautils.Names(): if name is None: continue mdis.loc_db.add_location(name, ad) asmcfg = mdis.dis_multiblock(func.start_ea) # Generate IR ircfg = ir_arch.new_ircfg_from_asmcfg(asmcfg) # Get settings settings = depGraphSettingsForm(ir_arch, ircfg, mn) settings.Execute() loc_key, elements, line_nb = settings.loc_key, settings.elements, settings.line_nb # Simplify assignments for irb in list(viewvalues(ircfg.blocks)): irs = [] offset = ir_arch.loc_db.get_location_offset(irb.loc_key) fix_stack = offset is not None and settings.unalias_stack for assignblk in irb: if fix_stack: stk_high = m2_expr.ExprInt(idc.get_spd(assignblk.instr.offset), ir_arch.sp.size) fix_dct = { ir_arch.sp: mn.regs.regs_init[ir_arch.sp] + stk_high } new_assignblk = {} for dst, src in viewitems(assignblk): if fix_stack: src = src.replace_expr(fix_dct) if dst != ir_arch.sp: dst = dst.replace_expr(fix_dct) dst, src = expr_simp(dst), expr_simp(src) new_assignblk[dst] = src irs.append(AssignBlock(new_assignblk, instr=assignblk.instr)) ircfg.blocks[irb.loc_key] = IRBlock(irb.loc_db, irb.loc_key, irs) # Get dependency graphs dg = settings.depgraph graphs = dg.get(loc_key, elements, line_nb, set([ir_arch.loc_db.get_offset_location(func.start_ea)])) # Display the result comments = {} sol_nb = 0 # Register and launch ida_kernwin.add_hotkey("Shift-N", next_element) treat_element()
offset = mdis.loc_db.get_location_offset(dest.loc_key) todo.append((mdis, instr, offset)) if args.funcswatchdog is not None and args.funcswatchdog <= 0: finish = True if args.try_disasm_all: for a, b in done_interval.intervals: if b in done: continue log.debug('add func %s' % hex(b)) todo.append((mdis, None, b)) # Generate dotty graph all_asmcfg = AsmCFG(mdis.loc_db) for blocks in viewvalues(all_funcs_blocks): all_asmcfg += blocks log.info('generate graph file') open('graph_execflow.dot', 'w').write(all_asmcfg.dot(offset=True)) log.info('generate intervals') all_lines = [] total_l = 0 print(done_interval) if args.image: log.info('build img') done_interval.show()
_PSetHolder_ Dummy PSet object used to construct the Tweak object to mimic the config structure """ def __init__(self, psetName): self.psetName_ = psetName self.parameters_ = [] # // # // Assistant lambda functions #// childPSets = lambda x: [ value for value in viewvalues(x.__dict__) if value.__class__.__name__ == "PSetHolder" ] childParameters = lambda p, x: ["%s.%s" % (p, i) for i in x.parameters_] recursiveGetattr = lambda obj, attr: reduce(getattr, attr.split("."), obj) def parameterIterator(obj): """ _parameterIterator_ Util to iterate through the parameters in a PSetHolder """ x = None
def getTotalTopLevelJobsInWMBS(self): inWMBS = 0 if "AgentJobInfo" in self.data: for agentRequestInfo in viewvalues(self.data["AgentJobInfo"]): inWMBS += agentRequestInfo['status'].get('inWMBS', 0) return inWMBS
def values(self): return list(viewvalues(self._assigns))
def __init__(self, reg_expr): self.dct_str_inv = dict((v.name, k) for k, v in viewitems(reg_expr)) self.dct_expr = reg_expr self.dct_expr_inv = dict((v, k) for k, v in viewitems(reg_expr)) reg_str = [v.name for v in viewvalues(reg_expr)] self.parser = literal_list(reg_str).setParseAction(self.cb_parse)
def testRecordInCouch(self): """ _testRecordInCouch_ Verify that jobs, state transitions and fwjrs are recorded correctly. """ change = ChangeState(self.config, "changestate_t") locationAction = self.daoFactory(classname="Locations.New") locationAction.execute("site1", pnn="T2_CH_CERN") testWorkflow = Workflow(spec=self.specUrl, owner="Steve", name="wf001", task=self.taskName) testWorkflow.create() testFileset = Fileset(name="TestFileset") testFileset.create() testSubscription = Subscription(fileset=testFileset, workflow=testWorkflow, split_algo="FileBased") testSubscription.create() testFileA = File(lfn="SomeLFNA", events=1024, size=2048, locations=set(["T2_CH_CERN"])) testFileB = File(lfn="SomeLFNB", events=1025, size=2049, locations=set(["T2_CH_CERN"])) testFileA.create() testFileB.create() testFileset.addFile(testFileA) testFileset.addFile(testFileB) testFileset.commit() splitter = SplitterFactory() jobFactory = splitter(package="WMCore.WMBS", subscription=testSubscription) jobGroup = jobFactory(files_per_job=1)[0] assert len(jobGroup.jobs) == 2, \ "Error: Splitting should have created two jobs." testJobA = jobGroup.jobs[0] testJobA["user"] = "******" testJobA["group"] = "DMWM" testJobA["taskType"] = "Merge" testJobB = jobGroup.jobs[1] testJobB["user"] = "******" testJobB["group"] = "DMWM" testJobB["taskType"] = "Processing" change.propagate([testJobA, testJobB], "new", "none") change.propagate([testJobA, testJobB], "created", "new") change.propagate([testJobA, testJobB], "executing", "created") testJobADoc = change.jobsdatabase.document(testJobA["couch_record"]) for transition in viewvalues(testJobADoc["states"]): self.assertTrue(isinstance(transition["timestamp"], int)) self.assertEqual(testJobADoc["jobid"], testJobA["id"], "Error: ID parameter is incorrect.") assert testJobADoc["name"] == testJobA["name"], \ "Error: Name parameter is incorrect." assert testJobADoc["jobgroup"] == testJobA["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobADoc["workflow"] == testJobA["workflow"], \ "Error: Workflow parameter is incorrect." assert testJobADoc["task"] == testJobA["task"], \ "Error: Task parameter is incorrect." assert testJobADoc["owner"] == testJobA["owner"], \ "Error: Owner parameter is incorrect." assert testJobADoc["mask"]["FirstEvent"] == testJobA["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobADoc["mask"]["FirstLumi"] == testJobA["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["LastLumi"] == testJobA["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobADoc["mask"]["FirstRun"] == testJobA["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobADoc["mask"]["LastEvent"] == testJobA["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobADoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." testJobBDoc = change.jobsdatabase.document(testJobB["couch_record"]) assert testJobBDoc["jobid"] == testJobB["id"], \ "Error: ID parameter is incorrect." assert testJobBDoc["name"] == testJobB["name"], \ "Error: Name parameter is incorrect." assert testJobBDoc["jobgroup"] == testJobB["jobgroup"], \ "Error: Jobgroup parameter is incorrect." assert testJobBDoc["mask"]["FirstEvent"] == testJobB["mask"]["FirstEvent"], \ "Error: First event in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastEvent"], \ "Error: Last event in mask is incorrect." assert testJobBDoc["mask"]["FirstLumi"] == testJobB["mask"]["FirstLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["LastLumi"] == testJobB["mask"]["LastLumi"], \ "Error: First lumi in mask is incorrect." assert testJobBDoc["mask"]["FirstRun"] == testJobB["mask"]["FirstRun"], \ "Error: First run in mask is incorrect." assert testJobBDoc["mask"]["LastEvent"] == testJobB["mask"]["LastRun"], \ "Error: First event in mask is incorrect." assert len(testJobBDoc["inputfiles"]) == 1, \ "Error: Input files parameter is incorrect." changeStateDB = self.couchServer.connectDatabase( dbname="changestate_t/jobs") allDocs = changeStateDB.document("_all_docs") self.assertEqual(len(allDocs["rows"]), 3, "Error: Wrong number of documents.") couchJobDoc = changeStateDB.document("1") assert couchJobDoc["name"] == testJobA["name"], \ "Error: Name is wrong" assert len(couchJobDoc["inputfiles"]) == 1, \ "Error: Wrong number of input files." result = changeStateDB.loadView("JobDump", "jobsByWorkflowName") self.assertEqual(len(result["rows"]), 2, "Error: Wrong number of rows.") for row in result["rows"]: couchJobDoc = changeStateDB.document(row["value"]["id"]) self.assertEqual(couchJobDoc["_rev"], row["value"]["rev"], "Error: Rev is wrong.") return
def has_blobs(self): return all(field.has_blobs() for field in viewvalues(self.fields))
def _checkData(self, data): if len(data) == 0: raise ValueError('Dictionary of records is empty.') self.data_model.check(next(iter(viewvalues(data))))
def gen_c_assignments(self, assignblk): """ Return C information used to generate the C code of the @assignblk @assignblk: an AssignBlock instance """ c_var = [] c_main = [] c_mem = [] c_updt = [] c_prefetch = [] dst_index = {8: 0, 16: 0, 32: 0, 64: 0, 128: 0} dst_var = {} prefetchers = self.get_mem_prefetch(assignblk) for expr, prefetcher in viewitems(prefetchers): str_src = self.id_to_c(expr) str_dst = self.id_to_c(prefetcher) c_prefetch.append('%s = %s;' % (str_dst, str_src)) for var in viewvalues(prefetchers): if var.size <= self.translator.NATIVE_INT_MAX_SIZE: c_var.append("uint%d_t %s;" % (var.size, var)) else: c_var.append("bn_t %s; // %d" % (var, var.size)) for dst, src in viewitems(assignblk): src = src.replace_expr(prefetchers) if dst == self.ir_arch.IRDst: pass elif isinstance(dst, ExprId): new_dst = self.add_local_var(dst_var, dst_index, dst) if dst in self.ir_arch.arch.regs.regs_flt_expr: # Don't mask float assignment c_main.append('%s = (%s);' % (self.id_to_c(new_dst), self.id_to_c(src))) elif new_dst.size <= self.translator.NATIVE_INT_MAX_SIZE: c_main.append('%s = (%s)&%s;' % (self.id_to_c(new_dst), self.id_to_c(src), SIZE_TO_MASK[src.size])) else: c_main.append( '%s = bignum_mask(%s, %d);' % (self.id_to_c(new_dst), self.id_to_c(src), src.size)) elif isinstance(dst, ExprMem): ptr = dst.ptr.replace_expr(prefetchers) if ptr.size <= self.translator.NATIVE_INT_MAX_SIZE: new_dst = ExprMem(ptr, dst.size) str_dst = self.id_to_c(new_dst).replace( 'MEM_LOOKUP', 'MEM_WRITE') c_mem.append('%s, %s);' % (str_dst[:-1], self.id_to_c(src))) else: ptr_str = self.id_to_c(ptr) if ptr.size <= self.translator.NATIVE_INT_MAX_SIZE: c_mem.append('%s, %s);' % (str_dst[:-1], self.id_to_c(src))) else: if src.size <= self.translator.NATIVE_INT_MAX_SIZE: c_mem.append( 'MEM_WRITE_BN_INT(jitcpu, %d, %s, %s);' % (src.size, ptr_str, self.id_to_c(src))) else: c_mem.append( 'MEM_WRITE_BN_BN(jitcpu, %d, %s, %s);' % (src.size, ptr_str, self.id_to_c(src))) else: raise ValueError("Unknown dst") for dst, new_dst in viewitems(dst_var): if dst == self.ir_arch.IRDst: continue c_updt.append('%s = %s;' % (self.id_to_c(dst), self.id_to_c(new_dst))) if dst.size <= self.translator.NATIVE_INT_MAX_SIZE: c_var.append("uint%d_t %s;" % (new_dst.size, new_dst)) else: c_var.append("bn_t %s; // %d" % (new_dst, new_dst.size)) return c_prefetch, c_var, c_main, c_mem, c_updt