def __checkSituations(self): """ To check if point with no elevation on line, and one or more elevation from other layers, and if there are different elevations at the same point """ situations = [] differences = [] for p in range(len(self.__points)): pt = self.__points[p] num_lines = len(self.__selectedIds) zz = [] for i in range(num_lines): if pt['z'][i] is not None: zz.append(i) if len(zz) == 0: self.__iface.messageBar().pushMessage( QCoreApplication.translate("VDLTools", "No line z ?!?"), level=QgsMessageBar.WARNING) elif len(zz) == 1: z0 = pt['z'][zz[0]] for i in range(num_lines, len(pt['z'])): if pt['z'][i] is None: continue if abs(pt['z'][i]-z0) > self.ALT_TOLERANCE: situations.append({'point': p, 'layer': (i-num_lines+1), 'vertex': z0}) elif len(zz) == 2: z0 = pt['z'][zz[0]] if abs(pt['z'][zz[1]] - z0) > self.ALT_TOLERANCE: differences.append({'point': p, 'v1': z0, 'v2': pt['z'][zz[1]]}) else: for i in range(num_lines, len(pt['z'])): if pt['z'][i] is None: continue if abs(pt['z'][i]-z0) > self.ALT_TOLERANCE: situations.append({'point': p, 'layer': (i-num_lines+1), 'vertex': z0}) else: self.__iface.messageBar().pushMessage( QCoreApplication.translate("VDLTools", "More than 2 lines z ?!?"), level=QgsMessageBar.WARNING) if (len(situations) > 0) or (len(differences) > 0): self.__setMessageDialog(situations, differences, self.__getNames()) self.__rubberSit.reset() self.__rubberDif.reset() for situation in situations: pt = self.__points[situation['point']] point = QgsPoint(pt['x'], pt['y']) if self.__rubberSit.numberOfVertices() == 0: self.__rubberSit.setToGeometry(QgsGeometry().fromPoint(point), None) else: self.__rubberSit.addPoint(point) for difference in differences: pt = self.__points[difference['point']] point = QgsPoint(pt['x'], pt['y']) if self.__rubberDif.numberOfVertices() == 0: self.__rubberDif.setToGeometry(QgsGeometry().fromPoint(point), None) else: self.__rubberDif.addPoint(point) self.__msgDlg.show() else: self.__checkZeros()
def english_to_key (s): """english_to_key(string):string(2.x)/bytes(2.x) Transform a string into a corresponding key. The string must contain words separated by whitespace; the number of words must be a multiple of 6. """ L=s.upper().split() ; key=b('') for index in range(0, len(L), 6): sublist=L[index:index+6] ; char=9*[0] ; bits=0 for i in sublist: index = wordlist.index(i) shift = (8-(bits+11)%8) %8 y = index << shift cl, cc, cr = (y>>16), (y>>8)&0xff, y & 0xff if (shift>5): char[bits>>3] = char[bits>>3] | cl char[(bits>>3)+1] = char[(bits>>3)+1] | cc char[(bits>>3)+2] = char[(bits>>3)+2] | cr elif shift>-3: char[bits>>3] = char[bits>>3] | cc char[(bits>>3)+1] = char[(bits>>3)+1] | cr else: char[bits>>3] = char[bits>>3] | cr bits=bits+11 subkey=reduce(lambda x,y:x+bchr(y), char, b('')) # Check the parity of the resulting key skbin=_key2bin(subkey) p=0 for i in range(0, 64, 2): p=p+_extract(skbin, i, 2) if (p&3) != _extract(skbin, 64, 2): raise ValueError("Parity error in resulting key") key=key+subkey[0:8] return key
def Run(self): if data_store.RelationalDBEnabled(): replace = {} for i in range(0, 2): with test_lib.FakeTime((1 + i) * 1000): hunt_id = self.CreateHunt(description="hunt_%d" % i) if i % 2: hunt.StopHunt(hunt_id) replace[hunt_id] = "H:00000%d" % i else: replace = {} for i in range(0, 2): with test_lib.FakeTime((1 + i) * 1000): with self.CreateHunt(description="hunt_%d" % i) as hunt_obj: if i % 2: hunt_obj.Stop() replace[hunt_obj.urn.Basename()] = "H:00000%d" % i self.Check( "ListHunts", args=hunt_plugin.ApiListHuntsArgs(), replace=replace) self.Check( "ListHunts", args=hunt_plugin.ApiListHuntsArgs(count=1), replace=replace) self.Check( "ListHunts", args=hunt_plugin.ApiListHuntsArgs(offset=1, count=1), replace=replace)
def testReturnsAllDataByDefault(self): """Checks that stats collection works.""" stats_collector = stats_collector_instance.Get() stats_collector.IncrementCounter("grr_client_received_bytes", 1566) stats_collector.IncrementCounter("grr_client_sent_bytes", 2000) results = self.RunAction( admin.GetClientStats, grr_worker=MockClientWorker(), arg=rdf_client_action.GetClientStatsRequest()) response = results[0] self.assertEqual(response.bytes_received, 1566) self.assertEqual(response.bytes_sent, 2000) self.assertLen(response.cpu_samples, 3) for i in range(3): self.assertEqual(response.cpu_samples[i].timestamp, rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100 + i * 10)) self.assertAlmostEqual(response.cpu_samples[i].user_cpu_time, 0.1) self.assertAlmostEqual(response.cpu_samples[i].system_cpu_time, 0.1 * (i + 1)) self.assertAlmostEqual(response.cpu_samples[i].cpu_percent, 10.0 + 5 * i) self.assertLen(response.io_samples, 3) for i in range(3): self.assertEqual(response.io_samples[i].timestamp, rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100 + i * 10)) self.assertEqual(response.io_samples[i].read_bytes, 100 * (i + 1)) self.assertEqual(response.io_samples[i].write_bytes, 100 * (i + 1)) self.assertEqual(response.boot_time, 100 * 1e6)
def testNotificationsAreDeletedFromAllShards(self): manager = queue_manager.QueueManager(token=self.token) manager.QueueNotification( session_id=rdfvalue.SessionID( base="aff4:/hunts", queue=queues.HUNTS, flow_name="42")) manager.Flush() manager.QueueNotification( session_id=rdfvalue.SessionID( base="aff4:/hunts", queue=queues.HUNTS, flow_name="43")) manager.Flush() # There should be two notifications in two different shards. shards_with_data = 0 for _ in range(manager.num_notification_shards): shard_sessions = manager.GetNotifications(queues.HUNTS) if shard_sessions: shards_with_data += 1 self.assertLen(shard_sessions, 1) self.assertEqual(shards_with_data, 2) # This should still work, as we delete notifications from all shards. manager.DeleteNotification( rdfvalue.SessionID( base="aff4:/hunts", queue=queues.HUNTS, flow_name="43")) manager.DeleteNotification( rdfvalue.SessionID( base="aff4:/hunts", queue=queues.HUNTS, flow_name="42")) for _ in range(manager.num_notification_shards): shard_sessions = manager.GetNotifications(queues.HUNTS) self.assertFalse(shard_sessions)
def inner_product_mat(num_states, num_rows, num_cols, max_vecs_per_node, verbosity=1): """ Computes inner products from known vecs. Remember that rows correspond to adjoint modes and cols to direct modes """ col_vec_handles = [mr.VecHandlePickle(join(data_dir, col_vec_name%col_num)) for col_num in range(num_cols)] row_vec_handles = [mr.VecHandlePickle(join(data_dir, row_vec_name%row_num)) for row_num in range(num_rows)] generate_vecs(data_dir, num_states, row_vec_handles+col_vec_handles) my_VS = mr.VectorSpaceHandles(np.vdot, max_vecs_per_node=max_vecs_per_node, verbosity=verbosity) prof = cProfile.Profile() start_time = T.time() prof.runcall(my_VS.compute_inner_product_mat, *(col_vec_handles, row_vec_handles)) total_time = T.time() - start_time prof.dump_stats('IP_mat_r%d.prof'%_parallel.get_rank()) return total_time
def lin_combine(num_states, num_bases, num_products, max_vecs_per_node, verbosity=1): """ Computes linear combination of vecs from saved vecs and random coeffs num_bases is number of vecs to be linearly combined num_products is the resulting number of vecs """ basis_handles = [mr.VecHandlePickle(join(data_dir, basis_name%basis_num)) for basis_num in range(num_bases)] product_handles = [mr.VecHandlePickle(join(data_dir, product_name%product_num)) for product_num in range(num_products)] generate_vecs(data_dir, num_states, basis_handles) my_VS = mr.VectorSpaceHandles(np.vdot, max_vecs_per_node=max_vecs_per_node, verbosity=verbosity) coeff_mat = np.random.random((num_bases, num_products)) _parallel.barrier() prof = cProfile.Profile() start_time = T.time() prof.runcall(my_VS.lin_combine, *(product_handles, basis_handles, coeff_mat)) total_time = T.time() - start_time prof.dump_stats('lincomb_r%d.prof'%_parallel.get_rank()) return total_time
def _ratios_matrix(self): """ Calculate all relative frequencies of *k*-mers. If a division by 0 occurs, the frequency will be set to -1.0. :return: A matrix with relative frequencies. :rtype: float[][] """ # Perhaps we can use something like this for a future normalization # operation. # Todo: Do this directly with NumPy. ratios = [] for i in range(self.number): ratios.append(self.number * [0.0]) # Fill the matrix. for i in range(self.number): for j in range(self.number): if self.counts[j]: ratios[i][j] = (self.counts[i] / self.counts[j]) / self.total else: ratios[i][j] = -1.0 return ratios
def TestOffsetAndCount(self, fetch_all_fn, fetch_range_fn, error_desc = None): """Tests a DB API method with different offset/count combinations. This helper method works by first fetching all available objects with fetch_all_fn and then fetching all possible ranges using fetch_fn. The test passes if subranges returned by fetch_fn match subranges of values in the list returned by fetch_all_fn. Args: fetch_all_fn: Function without arguments that fetches all available objects using the API method that's being tested. fetch_range_fn: Function that calls an API method that's being tested passing 2 positional arguments: offset and count. It should return a list of objects. error_desc: Optional string to be used in error messages. May be useful to identify errors from a particular test. """ all_objects = fetch_all_fn() self.assertNotEmpty(all_objects, "Fetched objects can't be empty (%s)." % error_desc) for i in range(len(all_objects)): for l in range(1, len(all_objects) + 1): results = fetch_range_fn(i, l) expected = all_objects[i:i + l] self.assertListEqual( results, expected, "Results differ from expected (offset %d, count %d%s): %s vs %s" % (i, l, (", " + error_desc) if error_desc else "", results, expected))
def plot_lstsq(rec, ax=None, fname=None, base=np.e): err, g0, b, W, R, info = rec tcoda, tbulk, Ecoda, Ebulk, Gcoda, Gbulk = info fig = None if ax is None: fig = plt.figure() ax = fig.add_subplot(111) tmin = min(tcoda[i][0] for i in range(len(tcoda))) tmax = max(tcoda[i][-1] for i in range(len(tcoda))) for i in range(len(tcoda)): offset = R[i % len(R)] * W[i // len(R)] / W[0] #offset = R[i] if len(W) == 1 else C[i] #Bci = np.log(Ecoda[i]) - np.log(FS * Gcoda[i]) - np.log(offset) # + R0 Bci = np.log(Ecoda[i]) - np.log(Gcoda[i]) - np.log(offset) ax.plot(tcoda[i], Bci / np.log(base), color='0.7') for i in range(len(tbulk)): #offset = R[i] if len(W) == 1 else C[i] offset = R[i % len(R)] * W[i // len(R)] / W[0] #Bbi = np.log(Ebulk[i]) - np.log(FS * Gbulk[i]) - np.log(offset) # + R0 Bbi = np.log(Ebulk[i]) - np.log(Gbulk[i]) - np.log(offset) ax.plot(tbulk[i], Bbi / np.log(base), 'o', color='0.4', mec='0.4', ms=MS) tmin = min(tmin, tbulk[i]) t = np.linspace(tmin, tmax, 100) ax.plot(t, (np.log(W[0]) - b * t) / np.log(base), color='k') ax.set_xlim(right=tmax) if fig and fname: _savefig(fig, fname=fname)
def testFilterConsidersOffsetAndCount(self): client_id = self.client_ids[0] # Create five approval requests without granting them. for i in range(10): with test_lib.FakeTime(42 + i): self.RequestClientApproval( client_id.Basename(), reason="Request reason %d" % i) args = user_plugin.ApiListClientApprovalsArgs( client_id=client_id, offset=0, count=5) result = self.handler.Handle(args, token=self.token) # Approvals are returned newest to oldest, so the first five approvals # have reason 9 to 5. self.assertLen(result.items, 5) for item, i in zip(result.items, reversed(range(6, 10))): self.assertEqual(item.reason, "Request reason %d" % i) # When no count is specified, take all items from offset to the end. args = user_plugin.ApiListClientApprovalsArgs(client_id=client_id, offset=7) result = self.handler.Handle(args, token=self.token) self.assertLen(result.items, 3) for item, i in zip(result.items, reversed(range(0, 3))): self.assertEqual(item.reason, "Request reason %d" % i)
def _create_legend(plot_axes, distribution_markers, distribution_labels, marker_type): """Creates a legend on the supplied axes.""" # We have to use a proxy artist for the legend because box plots currently # don't have a very useful legend in matplotlib, and using the default # legend for bar/scatterplots chokes on empty/null distributions. # # Note: This code is based on the following examples: # http://matplotlib.sourceforge.net/users/legend_guide.html # http://stackoverflow.com/a/11423554 if len(distribution_markers) != len(distribution_labels): raise ValueError("The number of distribution markers does not match " "the number of distribution labels.") if marker_type == 'colors': legend_proxy = [Rectangle((0, 0), 1, 1, fc=marker) for marker in distribution_markers] plot_axes.legend(legend_proxy, distribution_labels, loc='best') elif marker_type == 'symbols': legend_proxy = [Line2D(range(1), range(1), color='white', markerfacecolor='black', marker=marker) for marker in distribution_markers] plot_axes.legend(legend_proxy, distribution_labels, numpoints=3, scatterpoints=3, loc='best') else: raise ValueError("Invalid marker_type: '%s'. marker_type must be " "either 'colors' or 'symbols'." % marker_type)
def thdGetLogLines(conn): # get a set of chunks that completely cover the requested range tbl = self.db.model.logchunks q = sa.select([tbl.c.first_line, tbl.c.last_line, tbl.c.content, tbl.c.compressed]) q = q.where(tbl.c.logid == logid) q = q.where(tbl.c.first_line <= last_line) q = q.where(tbl.c.last_line >= first_line) q = q.order_by(tbl.c.first_line) rv = [] for row in conn.execute(q): # Retrieve associated "reader" and extract the data # Note that row.content is stored as bytes, and our caller expects unicode data = self.COMPRESSION_BYID[ row.compressed]["read"](row.content) content = data.decode('utf-8') if row.first_line < first_line: idx = -1 count = first_line - row.first_line for _ in range(count): idx = content.index('\n', idx + 1) content = content[idx + 1:] if row.last_line > last_line: idx = len(content) + 1 count = row.last_line - last_line for _ in range(count): idx = content.rindex('\n', 0, idx) content = content[:idx] rv.append(content) return u'\n'.join(rv) + u'\n' if rv else u''
def distances(self, distance_fn): """Compute distances between all pairs of sequences Parameters ---------- distance_fn : function Function for computing the distance between a pair of sequences. This must take two sequences as input (as `skbio.Sequence` objects) and return a single integer or float value. Returns ------- skbio.DistanceMatrix Matrix containing the distances between all pairs of sequences. """ sequence_count = self.sequence_count() dm = np.zeros((sequence_count, sequence_count)) ids = [] for i in range(sequence_count): self_i = self[i] ids.append(self_i.metadata['id']) for j in range(i): dm[i, j] = dm[j, i] = self_i.distance(self[j], distance_fn) return DistanceMatrix(dm, ids)
def __lt__(self, o): """less than `o` :param Route o: other rule to compare with """ if self == o: return False variables, strings = self._sort_struct() variables_o, strings_o = o._sort_struct() if len(variables) != len(variables_o): return len(variables) < len(variables_o) if len(''.join(strings)) != len(''.join(strings_o)): return len(''.join(strings)) > len(''.join(strings_o)) for i in range(len(strings)): if strings[i] != strings_o[i]: return strings[i] < strings_o[i] # strings are the same, so check variables for non-default parser for i in range(len(variables)): if variables[i] != 'str' and variables_o[i] == 'str': return True return False
def setUp(self): super(SystemCronTestMixin, self).setUp() one_hour_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("1h") eight_day_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("8d") ancient_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("61d") self.SetupClientsWithIndices( range(0, 10), system="Windows", ping=eight_day_ping) self.SetupClientsWithIndices( range(10, 20), system="Linux", ping=eight_day_ping) self.SetupClientsWithIndices( range(20, 22), system="Darwin", fleetspeak_enabled=True, ping=one_hour_ping) # These clients shouldn't be analyzed by any of the stats cronjobs. self.SetupClientsWithIndices( range(22, 24), system="Linux", ping=ancient_ping) for i in range(0, 10): client_id = u"C.1%015x" % i if data_store.AFF4Enabled(): with aff4.FACTORY.Open( client_id, mode="rw", token=self.token) as client: client.AddLabels([u"Label1", u"Label2"], owner=u"GRR") client.AddLabel(u"UserLabel", owner=u"jim") if data_store.RelationalDBEnabled(): data_store.REL_DB.AddClientLabels(client_id, u"GRR", [u"Label1", u"Label2"]) data_store.REL_DB.AddClientLabels(client_id, u"jim", [u"UserLabel"])
def fd1d(size): """ Produce a 1D finite difference matrix. Parameters: * size : int The number of points Returns: * fd : sparse CSR matrix The finite difference matrix Examples: >>> fd1d(2).todense() matrix([[ 1, -1]]) >>> fd1d(3).todense() matrix([[ 1, -1, 0], [ 0, 1, -1]]) >>> fd1d(4).todense() matrix([[ 1, -1, 0, 0], [ 0, 1, -1, 0], [ 0, 0, 1, -1]]) """ i = list(range(size - 1)) + list(range(size - 1)) j = list(range(size - 1)) + list(range(1, size)) v = [1] * (size - 1) + [-1] * (size - 1) return scipy.sparse.coo_matrix((v, (i, j)), (size - 1, size)).tocsr()
def testOffsetIsRelativeToFilteredResultsWhenFilterIsPresent(self): for i in range(5): self.CreateHunt(description="foo_hunt_%d" % i) for i in range(3): self.CreateHunt(description="bar_hunt_%d" % i) result = self.handler.Handle( hunt_plugin.ApiListHuntsArgs( description_contains="bar", active_within="1d", offset=1), token=self.token) self.assertLen(result.items, 2) for item in result.items: self.assertIn("bar", item.description) result = self.handler.Handle( hunt_plugin.ApiListHuntsArgs( description_contains="bar", active_within="1d", offset=2), token=self.token) self.assertLen(result.items, 1) for item in result.items: self.assertIn("bar", item.description) result = self.handler.Handle( hunt_plugin.ApiListHuntsArgs( description_contains="bar", active_within="1d", offset=3), token=self.token) self.assertEmpty(result.items)
def do_addLogLines_huge_log(self, NUM_CHUNKS=3000, chunk=(u'xy' * 70 + u'\n') * 3): if chunk.endswith("\n"): chunk = chunk[:-1] linesperchunk = chunk.count("\n") + 1 test_data = [ fakedb.LogChunk(logid=201, first_line=i * linesperchunk, last_line=i * linesperchunk + linesperchunk - 1, compressed=0, content=chunk) for i in range(NUM_CHUNKS) ] yield self.insertTestData( self.backgroundData + [ fakedb.Log(id=201, stepid=101, name=u'stdio', slug=u'stdio', complete=0, num_lines=NUM_CHUNKS * 3, type=u's')] + test_data) wholeLog = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3) for i in range(10): yield self.db.logs.compressLog(201) wholeLog2 = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3) self.assertEqual(wholeLog, wholeLog2) self.assertEqual(wholeLog, wholeLog2) def countChunk(conn): tbl = self.db.model.logchunks q = sa.select([sa.func.count(tbl.c.content)]) q = q.where(tbl.c.logid == 201) return conn.execute(q).fetchone()[0] chunks = yield self.db.pool.do(countChunk) # make sure MAX_CHUNK_LINES is taken in account self.assertGreaterEqual( chunks, NUM_CHUNKS * linesperchunk / logs.LogsConnectorComponent.MAX_CHUNK_LINES)
def _fasta_sniffer(fh): # Strategy: # Ignore up to 5 blank/whitespace-only lines at the beginning of the # file. Read up to 10 FASTA records. If at least one record is read (i.e. # the file isn't empty) and no errors are thrown during reading, assume # the file is in FASTA format. Next, try to parse the file as QUAL, which # has stricter requirements. If this succeeds, do *not* identify the file # as FASTA since we don't want to sniff QUAL files as FASTA (technically # they can be read as FASTA since the sequences aren't validated but it # probably isn't what the user wanted). Also, if we add QUAL as its own # file format in the future, we wouldn't want the FASTA and QUAL sniffers # to both identify a QUAL file. if _too_many_blanks(fh, 5): return False, {} num_records = 10 try: not_empty = False for _ in zip(range(num_records), _fasta_to_generator(fh)): not_empty = True if not_empty: fh.seek(0) try: list(zip(range(num_records), _parse_fasta_raw(fh, _parse_quality_scores, 'QUAL'))) except FASTAFormatError: return True, {} else: return False, {} else: return False, {} except FASTAFormatError: return False, {}
def test_filter(self): levPairs = [('Mammalia', 'Primates'), ('Vertebrata', 'Euarchontoglires'), ('Euarchontoglires', 'Rodents'), ('Vertebrata', 'HUMAN'), ('Primates', 'PANTR'), ('Primates', 'HUMAN')] expRes = [ [ [fa.FamIdent('1'), fa.FamIdent('2')], [fa.FamIdent('1')], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')], [fa.FamIdent('1')], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a')] ], [ [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamDupl('3', ['3.1a', '3.1b'])], [fa.FamIdent('1'), fa.FamNovel('2'), fa.FamDupl('3', ['3.1a', '3.1b'])], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')], [fa.FamIdent('1'), fa.FamNovel('2'), fa.FamDupl('3', '3.1a'), fa.FamSingleton('5')], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')], [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamLost('3.1b'), fa.FamSingleton('5')] ] ] filters = ['identical', {'identical', 'lost', 'singleton', 'novel', 'duplicated'}] res = [ [], [] ] for i in range(len(levPairs)): lev1, lev2 = levPairs[i] comp = self.compareLevelsSingletonAware(lev1, lev2) for j in range(len(filters)): fams = comp.filter(filters[j]) res[j].append(fams) self.assertListEqual(res[j][i], expRes[j][i], "failed for {} vs {}".format(lev1, lev2))
def _gen_rrg_block(self, xmlgen, block): """Generate a <block_type> tag for the given ``block``. Args: xmlgen (`XMLGenerator`): the generator to be used block (:obj:`dict`): a `dict` satisfying the JSON schema 'schema/block.schema.json' """ # 1. validate argument validate(instance = block, schema = _block_schema) # 2. generate tag with xmlgen.element("block_type", { "name": block["name"], "id": block["id"], "width": block.get("width", 1), "height": block.get("height", 1), }): ptc_it = count() capacity = block.get("capacity", 1) for z, key in product(range(capacity), ("input", "output", "clock")): for port in block.get(key, []): for bit in range(port["num_pins"]): with xmlgen.element("pin_class", { "type": "OUTPUT" if key == "output" else "INPUT", }): if capacity == 1: xmlgen.element_leaf("pin", { "ptc": next(ptc_it), }, "{}.{}[{}]".format(block["name"], port["name"], bit)) else: xmlgen.element_leaf("pin", { "ptc": next(ptc_it), }, "{}[{}].{}[{}]".format(block["name"], z, port["name"], bit))
def test_which_pools(self): """FortunaAccumulator.which_pools""" # which_pools(0) should fail self.assertRaises(AssertionError, FortunaAccumulator.which_pools, 0) self.assertEqual(FortunaAccumulator.which_pools(1), [0]) self.assertEqual(FortunaAccumulator.which_pools(2), [0, 1]) self.assertEqual(FortunaAccumulator.which_pools(3), [0]) self.assertEqual(FortunaAccumulator.which_pools(4), [0, 1, 2]) self.assertEqual(FortunaAccumulator.which_pools(5), [0]) self.assertEqual(FortunaAccumulator.which_pools(6), [0, 1]) self.assertEqual(FortunaAccumulator.which_pools(7), [0]) self.assertEqual(FortunaAccumulator.which_pools(8), [0, 1, 2, 3]) for i in range(1, 32): self.assertEqual(FortunaAccumulator.which_pools(2**i-1), [0]) self.assertEqual(FortunaAccumulator.which_pools(2**i), list(range(i+1))) self.assertEqual(FortunaAccumulator.which_pools(2**i+1), [0]) self.assertEqual(FortunaAccumulator.which_pools(2**31), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**32), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**33), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**34), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**35), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**36), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**64), list(range(32))) self.assertEqual(FortunaAccumulator.which_pools(2**128), list(range(32)))
def herdif(n, m, b): """Computes differentiation matrices D1, D2, ..., Dm on Hermite points. Args: n: np.mber of points, which is also the order of accuracy. m: np.mber of derivative matrices to return. b: Scaling parameter. Real and positive. Returns: x: Array of nodes, zeros of Hermite polynomial of degree n, scaled by b. Dm: A list s.t. Dm[i] is the (i+1)-th derivative matrix, i=0...m-1. np.te: 0 < m < n-1. """ x = herroots(n) # Compute weights alpha = np.exp(-x**2 / 2.) # Set up beta matrix s.t. beta[i,j] = # ( (i+1)-th derivative of alpha(x) )/alpha(x), evaluated at x = x(j). beta = np.zeros((m+1, x.shape[0])) beta[0] = 1.0 beta[1] = -x for i in range(2, m+1): beta[i] = -x * beta[i-1] - (i-1) * beta[i-2] # Remove initializing row from beta beta = np.delete(beta, 0, 0) # Compute differentiation matrix (b=1). Dm = poldif(x, alpha=alpha, B=beta) # Scale nodes by the factor b. x = x/b # Adjust derivatives for b not equal to 1. for i in range(1, m+1): Dm[i-1] *= b**i return x, Dm
def encrypt(self, plaintext): AES.encrypt(self, plaintext) plaintextBytes = stringToBytes(plaintext) chainBytes = stringToBytes(self.IV) #CBC Mode: For each block... for x in range(len(plaintextBytes)/16): #XOR with the chaining block blockBytes = plaintextBytes[x*16 : (x*16)+16] for y in range(16): blockBytes[y] ^= chainBytes[y] blockString = bytesToString(blockBytes) #Encrypt it encryptedBytes = stringToBytes(self.rijndael.encrypt(blockString)) #Overwrite the input with the output for y in range(16): plaintextBytes[(x*16)+y] = encryptedBytes[y] #Set the next chaining block chainBytes = encryptedBytes self.IV = bytesToString(chainBytes) return bytesToString(plaintextBytes)
def test_codes(self): # Codes that mean uneventful copies (including no copy at all). for i in [0, 1]: yield self._run_simple_test( r'D:\source', r'E:\dest', expected_code=i, expected_res=SUCCESS ) # Codes that mean some mismatched or extra files were found. for i in range(2, 8): yield self._run_simple_test( r'D:\source', r'E:\dest', expected_code=i, expected_res=WARNINGS ) # Codes that mean errors have been encountered. for i in range(8, 32): yield self._run_simple_test( r'D:\source', r'E:\dest', expected_code=i, expected_res=FAILURE ) # bit 32 is meaningless yield self._run_simple_test( r'D:\source', r'E:\dest', expected_code=32, expected_res=EXCEPTION )
def snapCurvedIntersections(mapPoint, mapCanvas, mapTool, featureId=None): """ To snap on curved intersections :param mapPoint: the map position :param mapCanvas: the used QgsMapCanvas :param mapTool: a QgsMapTool instance :param featureId: if we want to snap on a given feature :return: intersection point """ snap_layers = Finder.getLayersSettings(mapCanvas, [QGis.Line, QGis.Polygon, QGis.Point]) features = Finder.findFeaturesLayersAt(mapPoint, snap_layers, mapTool) inter = None if len(features) > 1: if len(features) > 2: for i in range(len(features)): for j in range(i, len(features)): feat1 = features[i] feat2 = features[j] if feat1 != feat2: interP = Finder.intersect(featureId, feat1, feat2, mapPoint) if interP is not None: if inter is None or mapPoint.sqrDist(interP) < mapPoint.sqrDist(inter): inter = interP else: feat1 = features[0] feat2 = features[1] inter = Finder.intersect(featureId, feat1, feat2, mapPoint) return inter
def testMaxSize(self): mocks = [] def gen_mock(): c = mock.MagicMock() mocks.append(c) return c proxies = [] pool = mysql_pool.Pool(gen_mock, max_size=5) for _ in builtins.range(5): c = pool.get(blocking=False) self.assertIsNotNone(c) proxies.append(c) self.assertIsNone(pool.get(blocking=False)) for p in proxies: p.close() for m in mocks: # Should be returned to the pool. m.close.assert_not_called() for _ in builtins.range(5): c = pool.get(blocking=False) self.assertIsNotNone(c) proxies.append(c) for p in proxies: p.close() self.assertLen(mocks, 5, 'Should have created only 5 mocks.')
def test_array_horizontal_shear(self): # tests function array_rotation_strain with synthetic data with pure # horizontal shear strain, no rotation or dilation array_coords = self.array_coords subarray = self.subarray ts1 = self.ts1 ts2 = self.ts2 sigmau = self.sigmau Vp = self.Vp Vs = self.Vs shear_strainh = ( 0.00001 * np.exp(-1 * np.square(np.linspace(-2, 2, 1000))) * np.sin(np.linspace(-10 * np.pi, 10 * np.pi, 1000)) ) ts3 = np.zeros((1000, 7)) for stat in range(7): for t in range(1000): ts1[t, stat] = array_coords[stat, 1] * shear_strainh[t] ts2[t, stat] = array_coords[stat, 0] * shear_strainh[t] out = array_rotation_strain(subarray, ts1, ts2, ts3, Vp, Vs, array_coords, sigmau) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_d"], decimal=12) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_dh"], decimal=12) np.testing.assert_array_almost_equal(abs(shear_strainh), out["ts_s"], decimal=12) np.testing.assert_array_almost_equal(abs(shear_strainh), out["ts_sh"], decimal=12) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w1"], decimal=12) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w2"], decimal=12) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w3"], decimal=12) np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_M"], decimal=12)
def testYamlPluginWithValuesOfSameType(self): responses = [] for i in range(10): responses.append( rdf_client_fs.StatEntry( pathspec=rdf_paths.PathSpec( path="/foo/bar/%d" % i, pathtype="OS"), st_mode=33184, # octal = 100640 => u=rw,g=r,o= => -rw-r----- st_ino=1063090, st_dev=64512, st_nlink=1 + i, st_uid=139592, st_gid=5000, st_size=0, st_atime=1336469177, st_mtime=1336129892, st_ctime=1336129892)) zip_fd, prefix = self.ProcessValuesToZip( {rdf_client_fs.StatEntry: responses}) self.assertEqual( set(zip_fd.namelist()), { "%s/MANIFEST" % prefix, "%s/ExportedFile/from_StatEntry.yaml" % prefix }) parsed_manifest = yaml.load(zip_fd.read("%s/MANIFEST" % prefix)) self.assertEqual(parsed_manifest, {"export_stats": { "StatEntry": { "ExportedFile": 10 } }}) parsed_output = yaml.load( zip_fd.read("%s/ExportedFile/from_StatEntry.yaml" % prefix)) self.assertLen(parsed_output, 10) for i in range(10): # Only the client_urn is filled in by the plugin. Doing lookups for # all the clients metadata is possible but expensive. It doesn't seem to # be worth it. self.assertEqual(parsed_output[i]["metadata"]["client_urn"], str(self.client_id)) self.assertEqual(parsed_output[i]["metadata"]["source_urn"], str(self.results_urn)) self.assertEqual(parsed_output[i]["urn"], self.client_id.Add("/fs/os/foo/bar").Add(str(i))) self.assertEqual(parsed_output[i]["st_mode"], "-rw-r-----") self.assertEqual(parsed_output[i]["st_ino"], "1063090") self.assertEqual(parsed_output[i]["st_dev"], "64512") self.assertEqual(parsed_output[i]["st_nlink"], str(1 + i)) self.assertEqual(parsed_output[i]["st_uid"], "139592") self.assertEqual(parsed_output[i]["st_gid"], "5000") self.assertEqual(parsed_output[i]["st_size"], "0") self.assertEqual(parsed_output[i]["st_atime"], "2012-05-08 09:26:17") self.assertEqual(parsed_output[i]["st_mtime"], "2012-05-04 11:11:32") self.assertEqual(parsed_output[i]["st_ctime"], "2012-05-04 11:11:32") self.assertEqual(parsed_output[i]["st_blksize"], "0") self.assertEqual(parsed_output[i]["st_rdev"], "0") self.assertEqual(parsed_output[i]["symlink"], "")
def test_slice_range(self): r = range(8) self.assertEqual(r[:], range(8)) self.assertEqual(r[:2], range(2)) self.assertEqual(r[:-2], range(6)) self.assertEqual(r[2:], range(2, 8)) self.assertEqual(r[-2:], range(6, 8)) self.assertEqual(r[2:-2], range(2, 6)) self.assertEqual(r[-2:2:-1], range(6, 2, -1)) r = r[::-1] self.assertEqual(r, range(7, -1, -1)) self.assertEqual(r[:], range(7, -1, -1)) self.assertEqual(r[:2], range(7, 5, -1)) self.assertEqual(r[:-2], range(7, 1, -1)) self.assertEqual(r[2:], range(5, -1, -1)) self.assertEqual(r[-2:], range(1, -1, -1)) self.assertEqual(r[2:-2], range(5, 1, -1)) self.assertEqual(r[-2:2:-1], range(1, 5))
def test_slice_empty_range(self): self.assertEqual(range(0)[:], range(0)) self.assertEqual(range(0)[::-1], range(-1, -1, -1))
def test_equality_range(self): self.assertEqual(range(7), range(7)) self.assertEqual(range(0), range(1, 1)) self.assertEqual(range(0, 10, 3), range(0, 11, 3))
def test_bool_range(self): self.assertFalse(range(0)) self.assertTrue(range(1)) self.assertFalse(range(1, 1)) self.assertFalse(range(5, 2))
def test_range(self): self.assertTrue(isinstance(range(0), Sequence)) self.assertTrue(isinstance(reversed(range(0)), Iterator))
def testBlockingTasks(self): # The pool starts off with the minimum number of threads. self.assertLen(self.test_pool, self.NUMBER_OF_THREADS) done_event = threading.Event() self.lock = threading.Lock() res = [] def Block(done): done.wait() def Insert(list_obj, element): with self.lock: list_obj.append(element) # Schedule the maximum number of threads of blocking tasks and the same of # insert tasks. The threads are now all blocked, and the inserts are # waiting in the queue. for _ in range(self.MAXIMUM_THREADS): self.test_pool.AddTask(Block, (done_event, ), "Blocking") # Wait until the threadpool picks up tasks. self.WaitUntil( lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS) # Now there's maximum number of threads active and the queue is empty. self.assertEqual(self.test_pool.pending_tasks, 0) # Now we push these tasks on the queue, but they're not going to be # processed, since all threads are busy. for i in range(self.MAXIMUM_THREADS): self.test_pool.AddTask(Insert, (res, i), "Insert", blocking=True, inline=False) # There should be 20 workers created and they should consume all the # blocking tasks. self.WaitUntil( lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS) # No Insert tasks are running yet. self.assertEqual(res, []) # There are 20 tasks waiting on the queue. self.assertEqual(self.test_pool.pending_tasks, self.MAXIMUM_THREADS) # Inserting more tasks than the queue can hold should lead to processing # the tasks inline. This effectively causes these tasks to skip over the # tasks which are waiting in the queue. for i in range(10, 20): self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True) res.sort() self.assertEqual(res, list(range(10, 20))) # This should release all the busy tasks. It will also cause the workers # to process all the Insert tasks in the queue. done_event.set() self.test_pool.Join() # Now the rest of the tasks should have been processed as well. self.assertEqual(sorted(res[10:]), list(range(20)))
def d_logS_d_expX(S, X, j, p, grad, thresh, eps=1e-5): """ Compute the gradient of log S[j] w.r.t. exp(X). For unstable cases, use p-th order approximnation. """ # ------------------------------------------------------------------------ # Detect unstabilites # ------------------------------------------------------------------------ _X_ = LogTensor(X) _S_ = [LogTensor(S[i]) for i in range(S.size(0))] # recursion of gradient formula (separate terms for stability) _N_, _P_ = recursion(_S_, _X_, j) # detect instability: small relative difference in log-space P = _P_.torch() if isinstance(_N_, int): N = torch.zeros(P.size()).cuda() else: N = _N_.torch() diff = (P - N) / (N.abs() + eps) # split into stable and unstable indices u_indices = torch.lt(diff, thresh) # unstable s_indices = u_indices.eq(0) # stable # ------------------------------------------------------------------------ # Compute d S[j] / d X # ------------------------------------------------------------------------ # make grad match size and type of X grad = grad.type_as(X).resize_as_(X) # exact gradient for s_indices (stable) elements if s_indices.sum(): # re-use positive and negative parts of recursion (separate for stability) _N_ = LogTensor(N[s_indices]) _P_ = LogTensor(P[s_indices]) _X_ = LogTensor(X[s_indices]) _S_ = [LogTensor(S[i][s_indices]) for i in range(S.size(0))] # d log S[j] / d exp(X) = (d S[j] / d X) * X / S[j] _SG_ = (_P_ - _N_) * _X_ / _S_[j] grad.masked_scatter_(s_indices, _SG_.torch().exp()) # approximate gradients for u_indices (unstable) elements if u_indices.sum(): _X_ = LogTensor(X[u_indices]) _S_ = [LogTensor(S[i][u_indices]) for i in range(S.size(0))] # positive and negative parts of approximation (separate for stability) _N_, _P_ = approximation(_S_, _X_, j, p) # d log S[j] / d exp(X) = (d S[j] / d X) * X / S[j] _UG_ = (_P_ - _N_) * _X_ / _S_[j] grad.masked_scatter_(u_indices, _UG_.torch().exp()) return grad
def test_multi_player(self): players = 2 run_config = run_configs.get() parallel = run_parallel.RunParallel() map_inst = maps.get("Simple64") screen_size_px = point.Point(64, 64) minimap_size_px = point.Point(32, 32) interface = sc_pb.InterfaceOptions() screen_size_px.assign_to(interface.feature_layer.resolution) minimap_size_px.assign_to(interface.feature_layer.minimap_resolution) # Reserve a whole bunch of ports for the weird multiplayer implementation. ports = [portpicker.pick_unused_port() for _ in range(1 + players * 2)] logging.info("Valid Ports: %s", ports) # Actually launch the game processes. print_stage("start") sc2_procs = [ run_config.start(extra_ports=ports) for _ in range(players) ] controllers = [p.controller for p in sc2_procs] try: # Save the maps so they can access it. map_path = os.path.basename(map_inst.path) print_stage("save_map") parallel.run((c.save_map, map_path, map_inst.data(run_config)) for c in controllers) # Create the create request. create = sc_pb.RequestCreateGame(local_map=sc_pb.LocalMap( map_path=map_path)) for _ in range(players): create.player_setup.add(type=sc_pb.Participant) # Create the join request. join = sc_pb.RequestJoinGame(race=sc_common.Random, options=interface) join.shared_port = ports.pop() join.server_ports.game_port = ports.pop() join.server_ports.base_port = ports.pop() for _ in range(players - 1): join.client_ports.add(game_port=ports.pop(), base_port=ports.pop()) # Play a few short games. for _ in range(2): # 2 episodes # Create and Join print_stage("create") controllers[0].create_game(create) print_stage("join") parallel.run((c.join_game, join) for c in controllers) print_stage("run") for game_loop in range(1, 10): # steps per episode # Step the game parallel.run(c.step for c in controllers) # Observe obs = parallel.run(c.observe for c in controllers) for p_id, o in enumerate(obs): self.assertEqual(o.observation.game_loop, game_loop) self.assertEqual(o.observation.player_common.player_id, p_id + 1) # Act actions = [sc_pb.Action() for _ in range(players)] for action in actions: pt = (point.Point.unit_rand() * minimap_size_px).floor() pt.assign_to(action.action_feature_layer.camera_move. center_minimap) parallel.run( (c.act, a) for c, a in zip(controllers, actions)) # Done this game. print_stage("leave") parallel.run(c.leave for c in controllers) finally: print_stage("quit") # Done, shut down. Don't depend on parallel since it might be broken. for c in controllers: c.quit() for p in sc2_procs: p.close()
*counter block*, which is a byte string of `block_size` bytes. For better performance, use `Crypto.Util.Counter`. segment_size : integer (*Only* `MODE_CFB`).The number of bits the plaintext and ciphertext are segmented in. It must be a multiple of 8. If 0 or not specified, it will be assumed to be 8. :Return: an `CAST128Cipher` object """ return CAST128Cipher(key, *args, **kwargs) #: Electronic Code Book (ECB). See `blockalgo.MODE_ECB`. MODE_ECB = 1 #: Cipher-Block Chaining (CBC). See `blockalgo.MODE_CBC`. MODE_CBC = 2 #: Cipher FeedBack (CFB). See `blockalgo.MODE_CFB`. MODE_CFB = 3 #: This mode should not be used. MODE_PGP = 4 #: Output FeedBack (OFB). See `blockalgo.MODE_OFB`. MODE_OFB = 5 #: CounTer Mode (CTR). See `blockalgo.MODE_CTR`. MODE_CTR = 6 #: OpenPGP Mode. See `blockalgo.MODE_OPENPGP`. MODE_OPENPGP = 7 #: Size of a data block (in bytes) block_size = 8 #: Size of a key (in bytes) key_size = range(5, 16 + 1)
def compute_symmetric_inner_product_mat(self, vec_handles): """Computes symmetric matrix whose elements are inner products of the vector objects in ``vec_handles`` with each other. Args: ``vec_handles``: List of handles for vector objects corresponding to both rows and columns. For example, in POD this is the snapshot matrix :math:`X`. Returns: ``IP_mat``: 2D array of inner products. See the documentation for :py:meth:`compute_inner_product_mat` for an idea how this works. Efficiency is achieved by only computing the upper-triangular elements, since the matrix is symmetric. Within the upper-triangular portion, there are rectangular chunks and triangular chunks. The rectangular chunks are divided up among MPI workers (processors) as weighted tasks. Once those have been computed, the triangular chunks are dealt with. """ # TODO: JON, write detailed documentation similar to # :py:meth:`compute_inner_product_mat`. self._check_inner_product() vec_handles = util.make_iterable(vec_handles) num_vecs = len(vec_handles) # num_cols_per_chunk is the number of cols each proc gets at once. # Columns are retrieved if the matrix must be broken up into sets of # chunks. Then symmetric upper triangular portions will be computed, # followed by a rectangular piece that uses columns not already in # memory. num_cols_per_proc_chunk = 1 num_rows_per_proc_chunk = self.max_vecs_per_proc -\ num_cols_per_proc_chunk # <nprocs> chunks are computed simulaneously, making up a set. num_cols_per_chunk = num_cols_per_proc_chunk * parallel.get_num_procs() num_rows_per_chunk = num_rows_per_proc_chunk * parallel.get_num_procs() # <num_row_chunks> is the number of sets that must be computed. num_row_chunks = int(np.ceil(num_vecs * 1. / num_rows_per_chunk)) if num_row_chunks > 1: self.print_msg( 'Warning: The vecs, of which ' 'there are %d, will be retrieved %d times each. Increase ' 'number of nodes or max_vecs_per_node to reduce redundant ' '"get"s for a speedup.' % (num_vecs, num_row_chunks)) # Estimate the time this will take and determine matrix datatype # (real or complex). test_vec = vec_handles[0].get() # Burn the first, it sometimes contains slow imports IP_burn = self.inner_product(test_vec, test_vec) start_time = time() test_vec = vec_handles[0].get() get_time = time() - start_time start_time = time() IP = self.inner_product(test_vec, test_vec) IP_time = time() - start_time IP_type = type(IP) total_IP_time = (num_vecs**2 * IP_time / 2. / parallel.get_num_procs()) vecs_per_proc = self.max_vecs_per_node * parallel.get_num_nodes() / \ parallel.get_num_procs() num_gets = (num_vecs**2 /2.) / ((vecs_per_proc-2) * parallel.get_num_procs()**2) + \ num_vecs/parallel.get_num_procs()/2. total_get_time = num_gets * get_time self.print_msg('Computing the inner product matrix will take at least ' '%.1f minutes' % ((total_IP_time + total_get_time) / 60.)) del test_vec # Use the same trick as in compute_IP_mat, having each proc # fill in elements of a num_rows x num_rows sized matrix, rather than # assembling small chunks. This is done for the triangular portions. # For the rectangular portions, the inner product mat is filled # in directly. IP_mat = np.mat(np.zeros((num_vecs, num_vecs), dtype=IP_type)) for start_row_index in range(0, num_vecs, num_rows_per_chunk): end_row_index = min(num_vecs, start_row_index + num_rows_per_chunk) proc_row_tasks_all = parallel.find_assignments( list(range(start_row_index, end_row_index))) num_active_procs = len([task for task in \ proc_row_tasks_all if task != []]) proc_row_tasks = proc_row_tasks_all[parallel.get_rank()] if len(proc_row_tasks) != 0: row_vecs = [ vec_handle.get() for vec_handle in vec_handles[proc_row_tasks[0]:proc_row_tasks[-1] + 1] ] else: row_vecs = [] # Triangular chunks if len(proc_row_tasks) > 0: # Test that indices are consecutive if proc_row_tasks[0:] != list( range(proc_row_tasks[0], proc_row_tasks[-1] + 1)): raise ValueError('Indices are not consecutive.') # Per-processor triangles (using only vecs in memory) for row_index in range(proc_row_tasks[0], proc_row_tasks[-1] + 1): # Diagonal term IP_mat[row_index, row_index] = self.\ inner_product(row_vecs[row_index - proc_row_tasks[ 0]], row_vecs[row_index - proc_row_tasks[0]]) # Off-diagonal terms for col_index in range(row_index + 1, proc_row_tasks[-1] + 1): IP_mat[row_index, col_index] = self.\ inner_product(row_vecs[row_index -\ proc_row_tasks[0]], row_vecs[col_index -\ proc_row_tasks[0]]) # Number of square chunks to fill in is n * (n-1) / 2. At each # iteration we fill in n of them, so we need (n-1) / 2 # iterations (round up). for set_index in range(int(np.ceil((num_active_procs - 1.) / 2))): # The current proc is "sender" my_rank = parallel.get_rank() my_row_indices = proc_row_tasks my_num_rows = len(my_row_indices) # The proc to send to is "destination" dest_rank = (my_rank + set_index + 1) % num_active_procs # This is unused? #dest_row_indices = proc_row_tasks_all[dest_rank] # The proc that data is received from is the "source" source_rank = (my_rank - set_index - 1) % num_active_procs # Find the maximum number of sends/recv to be done by any proc max_num_to_send = int(np.ceil(1. * max([len(tasks) for \ tasks in proc_row_tasks_all]) /\ num_cols_per_proc_chunk)) """ # Pad tasks with nan so that everyone has the same # number of things to send. Same for list of vecs with None. # The empty lists will not do anything when enumerated, so no # inner products will be taken. nan is inserted into the # indices because then min/max of the indices can be taken. if my_num_rows != len(row_vecs): raise ValueError('Number of rows assigned does not ' +\ 'match number of vecs in memory.') if my_num_rows > 0 and my_num_rows < max_num_to_send: my_row_indices += [np.nan] * (max_num_to_send - my_num_rows) row_vecs += [[]] * (max_num_to_send - my_num_rows) """ for send_index in range(max_num_to_send): # Only processors responsible for rows communicate if my_num_rows > 0: # Send row vecs, in groups of num_cols_per_proc_chunk # These become columns in the ensuing computation start_col_index = send_index * num_cols_per_proc_chunk end_col_index = min( start_col_index + num_cols_per_proc_chunk, my_num_rows) col_vecs_send = ( row_vecs[start_col_index:end_col_index], my_row_indices[start_col_index:end_col_index]) # Create unique tags based on ranks send_tag = my_rank * (parallel.get_num_procs() + 1) + dest_rank recv_tag = source_rank * (parallel.get_num_procs() + 1) + my_rank # Send and receieve data. The Wait() command after the # receive prevents a race condition not fixed by sync(). # The Wait() is very important for the non- # blocking send (though we are unsure why). request = parallel.comm.isend(col_vecs_send, dest=dest_rank, tag=send_tag) col_vecs_recv = parallel.comm.recv(source=source_rank, tag=recv_tag) request.Wait() col_vecs = col_vecs_recv[0] my_col_indices = col_vecs_recv[1] for row_index in range(my_row_indices[0], my_row_indices[-1] + 1): for col_vec_index, col_vec in enumerate(col_vecs): IP_mat[row_index, my_col_indices[ col_vec_index]] = self.inner_product( row_vecs[row_index - my_row_indices[0]], col_vec) if (time() - self.prev_print_time) > \ self.print_interval: num_completed_IPs = (np.abs(IP_mat) > 0).sum() percent_completed_IPs = \ (100.*2*num_completed_IPs * \ parallel.get_num_MPI_workers())/\ (num_vecs**2) self.print_msg( ('Completed %.1f%% of inner products') % percent_completed_IPs, sys.stderr) self.prev_print_time = time() # Sync after send/receive parallel.barrier() # Fill in the rectangular portion next to each triangle (if nec.). # Start at index after last row, continue to last column. This part # of the code is the same as in compute_IP_mat, as of # revision 141. for start_col_index in range(end_row_index, num_vecs, num_cols_per_chunk): end_col_index = min(start_col_index + num_cols_per_chunk, num_vecs) proc_col_tasks = parallel.find_assignments( list(range(start_col_index, end_col_index)))[parallel.get_rank()] # Pass the col vecs to proc with rank -> mod(rank+1,numProcs) # Must do this for each processor, until data makes a circle col_vecs_recv = (None, None) if len(proc_col_tasks) > 0: col_indices = list( range(proc_col_tasks[0], proc_col_tasks[-1] + 1)) else: col_indices = [] for num_passes in range(parallel.get_num_procs()): # If on the first pass, get the col vecs, no send/recv # This is all that is called when in serial, loop iterates # once. if num_passes == 0: if len(col_indices) > 0: col_vecs = [col_handle.get() \ for col_handle in vec_handles[col_indices[0]:\ col_indices[-1] + 1]] else: col_vecs = [] else: # Determine whom to communicate with dest = (parallel.get_rank() + 1) % parallel.\ get_num_procs() source = (parallel.get_rank() - 1) % parallel.\ get_num_procs() # Create unique tag based on ranks send_tag = parallel.get_rank() * (parallel.\ get_num_procs() + 1) + dest recv_tag = source*(parallel.get_num_procs() + 1) +\ parallel.get_rank() # Collect data and send/receive col_vecs_send = (col_vecs, col_indices) request = parallel.comm.isend(col_vecs_send, dest=\ dest, tag=send_tag) col_vecs_recv = parallel.comm.recv(source=source, tag=recv_tag) request.Wait() parallel.barrier() col_indices = col_vecs_recv[1] col_vecs = col_vecs_recv[0] # Compute the IPs for this set of data col_indices stores # the indices of the IP_mat columns to be # filled in. if len(proc_row_tasks) > 0: for row_index in range(proc_row_tasks[0], proc_row_tasks[-1] + 1): for col_vec_index, col_vec in enumerate(col_vecs): IP_mat[row_index, col_indices[ col_vec_index]] = self.inner_product( row_vecs[row_index - proc_row_tasks[0]], col_vec) if ((time() - self.prev_print_time) > self.print_interval): num_completed_IPs = (np.abs(IP_mat) > 0).sum() percent_completed_IPs = ( 100. * 2 * num_completed_IPs * parallel.get_num_MPI_workers()) / (num_vecs**2) self.print_msg( ('Completed %.1f%% of inner ' + 'products') % percent_completed_IPs, sys.stderr) self.prev_print_time = time() # Completed a chunk of rows and all columns on all processors. # Finished row_vecs loop, delete memory used del row_vecs # Assign the triangular portion chunks into IP_mat. if parallel.is_distributed(): IP_mat = parallel.custom_comm.allreduce(IP_mat) # Create a mask for the repeated values. Select values that are zero # in the upper triangular portion (not computed there) but nonzero in # the lower triangular portion (computed there). For the case where # the inner product is not perfectly symmetric, this will select the # computation done in the upper triangular portion. mask = np.multiply(IP_mat == 0, IP_mat.T != 0) # Collect values below diagonal IP_mat += np.multiply(np.triu(IP_mat.T, 1), mask) # Symmetrize matrix IP_mat = np.triu(IP_mat) + np.triu(IP_mat, 1).T percent_completed_IPs = 100. self.print_msg(('Completed %.1f%% of inner ' + 'products') % percent_completed_IPs, sys.stderr) self.prev_print_time = time() parallel.barrier() return IP_mat
def compute_inner_product_mat(self, row_vec_handles, col_vec_handles): """Computes matrix whose elements are inner products of the vector objects in ``row_vec_handles`` and ``col_vec_handles``. Args: ``row_vec_handles``: List of handles for vector objects corresponding to rows of the inner product matrix. For example, in BPOD this is the adjoint snapshot matrix :math:`Y`. ``col_vec_handles``: List of handles for vector objects corresponding to columns of the inner product matrix. For example, in BPOD this is the direct snapshot matrix :math:`X`. Returns: ``IP_mat``: 2D array of inner products. The vectors are retrieved in memory-efficient chunks and are not all in memory at once. The row vectors and column vectors are assumed to be different. When they are the same, use :py:meth:`compute_symmetric_inner_product` for a 2x speedup. Each MPI worker (processor) is responsible for retrieving a subset of the rows and columns. The processors then send/receive columns via MPI so they can be used to compute all inner products for the rows on each MPI worker. This is repeated until all MPI workers are done with all of their row chunks. If there are 2 processors:: | x o | rank0 | x o | | x o | - | o x | rank1 | o x | | o x | In the next step, rank 0 sends column 0 to rank 1 and rank 1 sends column 1 to rank 0. The remaining inner products are filled in:: | x x | rank0 | x x | | x x | - | x x | rank1 | x x | | x x | When the number of columns and rows is not divisible by the number of processors, the processors are assigned unequal numbers of tasks. However, all processors are always part of the passing cycle. The scaling is: - num gets / processor ~ :math:`(n_r*n_c/((max-2)*n_p*n_p)) + n_r/n_p` - num MPI sends / processor ~ :math:`(n_p-1)*(n_r/((max-2)*n_p))*n_c/n_p` - num inner products / processor ~ :math:`n_r*n_c/n_p` where :math:`n_r` is number of rows, :math:`n_c` number of columns, :math:`max` is ``max_vecs_per_proc = max_vecs_per_node/num_procs_per_node``, and :math:`n_p` is the number of MPI workers (processors). If there are more rows than columns, then an internal transpose and un-transpose is performed to improve efficiency (since :math:`n_c` only appears in the scaling in the quadratic term). """ self._check_inner_product() row_vec_handles = util.make_iterable(row_vec_handles) col_vec_handles = util.make_iterable(col_vec_handles) num_cols = len(col_vec_handles) num_rows = len(row_vec_handles) if num_rows > num_cols: transpose = True temp = row_vec_handles row_vec_handles = col_vec_handles col_vec_handles = temp temp = num_rows num_rows = num_cols num_cols = temp else: transpose = False # convenience rank = parallel.get_rank() ## Old way that worked # num_cols_per_proc_chunk is the number of cols each proc gets at once num_cols_per_proc_chunk = 1 num_rows_per_proc_chunk = self.max_vecs_per_proc - \ num_cols_per_proc_chunk # Determine how the retrieving and inner products will be split up. row_tasks = parallel.find_assignments(list(range(num_rows))) col_tasks = parallel.find_assignments(list(range(num_cols))) # Find max number of col tasks among all processors max_num_row_tasks = max([len(tasks) for tasks in row_tasks]) max_num_col_tasks = max([len(tasks) for tasks in col_tasks]) ## New way #if self.max_vecs_per_node > max_num_row_tasks: # num_cols_per_proc_chunk = #num_rows_per_proc_chunk = self.max_vecs_per_proc - \ # num_cols_per_proc_chunk # These variables are the number of iters through loops that retrieve # ("get") row and column vecs. num_row_get_loops = \ int(np.ceil(max_num_row_tasks*1./num_rows_per_proc_chunk)) num_col_get_loops = \ int(np.ceil(max_num_col_tasks*1./num_cols_per_proc_chunk)) if num_row_get_loops > 1: self.print_msg( 'Warning: The column vecs, of which ' 'there are %d, will be retrieved %d times each. Increase ' 'number of nodes or max_vecs_per_node to reduce redundant ' '"get"s for a speedup.' % (num_cols, num_row_get_loops)) # Estimate the time this will take and determine matrix datatype # (real or complex). row_vec = row_vec_handles[0].get() col_vec = col_vec_handles[0].get() # Burn the first, it sometimes contains slow imports IP_burn = self.inner_product(row_vec, col_vec) start_time = time() row_vec = row_vec_handles[0].get() get_time = time() - start_time start_time = time() IP = self.inner_product(row_vec, col_vec) IP_time = time() - start_time IP_type = type(IP) total_IP_time = (num_rows * num_cols * IP_time / parallel.get_num_procs()) vecs_per_proc = self.max_vecs_per_node * parallel.get_num_nodes() / \ parallel.get_num_procs() num_gets = (num_rows * num_cols) / ( (vecs_per_proc - 2) * parallel.get_num_procs()**2) + num_rows / parallel.get_num_procs() total_get_time = num_gets * get_time self.print_msg('Computing the inner product matrix will take at least ' '%.1f minutes' % ((total_IP_time + total_get_time) / 60.)) del row_vec, col_vec # To find all of the inner product mat chunks, each # processor has a full IP_mat with size # num_rows x num_cols even though each processor is not responsible for # filling in all of these entries. After each proc fills in what it is # responsible for, the other entries remain 0's. Then, an allreduce # is done and all the IP_mats are summed. This is simpler # concatenating chunks of the IPmats. # The efficiency is not an issue, the size of the mats # are small compared to the size of the vecs for large data. IP_mat = np.mat(np.zeros((num_rows, num_cols), dtype=IP_type)) for row_get_index in range(num_row_get_loops): if len(row_tasks[rank]) > 0: start_row_index = min( row_tasks[rank][0] + row_get_index * num_rows_per_proc_chunk, row_tasks[rank][-1] + 1) end_row_index = min(row_tasks[rank][-1] + 1, start_row_index + num_rows_per_proc_chunk) row_vecs = [ row_vec_handle.get() for row_vec_handle in row_vec_handles[start_row_index:end_row_index] ] else: row_vecs = [] for col_get_index in range(num_col_get_loops): if len(col_tasks[rank]) > 0: start_col_index = min( col_tasks[rank][0] + col_get_index * num_cols_per_proc_chunk, col_tasks[rank][-1] + 1) end_col_index = min( col_tasks[rank][-1] + 1, start_col_index + num_cols_per_proc_chunk) else: start_col_index = 0 end_col_index = 0 # Cycle the col vecs to proc with rank -> mod(rank+1,num_procs) # Must do this for each processor, until data makes a circle col_vecs_recv = (None, None) col_indices = list(range(start_col_index, end_col_index)) for pass_index in range(parallel.get_num_procs()): #if rank==0: print 'starting pass index=',pass_index # If on the first pass, get the col vecs, no send/recv # This is all that is called when in serial, loop iterates # once. if pass_index == 0: col_vecs = [ col_handle.get() for col_handle in col_vec_handles[start_col_index:end_col_index] ] else: # Determine with whom to communicate dest = (rank + 1) % parallel.get_num_procs() source = (rank - 1) % parallel.get_num_procs() # Create unique tag based on send/recv ranks send_tag = rank * \ (parallel.get_num_procs() + 1) + dest recv_tag = source * \ (parallel.get_num_procs() + 1) + rank # Collect data and send/receive col_vecs_send = (col_vecs, col_indices) request = parallel.comm.isend(col_vecs_send, dest=dest, tag=send_tag) col_vecs_recv = parallel.comm.recv(source=source, tag=recv_tag) request.Wait() parallel.barrier() col_indices = col_vecs_recv[1] col_vecs = col_vecs_recv[0] # Compute the IPs for this set of data col_indices stores # the indices of the IP_mat columns to be # filled in. if len(row_vecs) > 0: for row_index in range(start_row_index, end_row_index): for col_vec_index, col_vec in enumerate(col_vecs): IP_mat[row_index, col_indices[ col_vec_index]] = self.inner_product( row_vecs[row_index - start_row_index], col_vec) if (time() - self.prev_print_time) > \ self.print_interval: num_completed_IPs = (np.abs(IP_mat) > 0).sum() percent_completed_IPs = ( 100. * num_completed_IPs * parallel.get_num_MPI_workers()) / (num_cols * num_rows) self.print_msg( ('Completed %.1f%% of inner ' + 'products') % percent_completed_IPs, sys.stderr) self.prev_print_time = time() # Clear the retrieved column vecs after done this pass cycle del col_vecs # Completed a chunk of rows and all columns on all processors. del row_vecs # Assign these chunks into IP_mat. if parallel.is_distributed(): IP_mat = parallel.custom_comm.allreduce(IP_mat) if transpose: IP_mat = IP_mat.T percent_completed_IPs = 100. self.print_msg(('Completed %.1f%% of inner ' + 'products') % percent_completed_IPs, sys.stderr) self.prev_print_time = time() parallel.barrier() return IP_mat
def __getitem__(self, indices): """Get by indexing lookup.""" indices = self._indices(indices) obj = super(NamedNumpyArray, self).__getitem__(indices) if (isinstance(indices, np.ndarray) and len(indices.shape) > 1 and indices.dtype == bool): # Is this a multi-dimensional mask, eg: obj[obj == 5] ? # Multi-dimensional masks return a single dimensional array, and it's # unclear what it means for the result to have names, so return a normal # numpy array. return np.array(obj) if isinstance(obj, np.ndarray): # If this is a view, index the names too. if not isinstance(indices, tuple): indices = (indices, ) new_names = [] dim = 0 for i, index in enumerate(indices): if isinstance(index, numbers.Integral): dim += 1 # Drop this dimension's names. elif index is Ellipsis: # Copy all the dimensions' names through. end = len(self.shape) - len(indices) + i + 1 for j in range(dim, end): new_names.append(self._index_names[j]) dim = end elif index is np.newaxis: # Add an unnamed dimension. new_names.append(None) # Don't modify dim, as we're still working on the same one. elif (self._index_names[dim] is None or (isinstance(index, slice) and index == _NULL_SLICE)): # Keep unnamed dimensions or ones where the slice is a no-op. new_names.append(self._index_names[dim]) dim += 1 elif isinstance(index, (slice, list, np.ndarray)): if isinstance(index, np.ndarray) and len(index.shape) > 1: raise TypeError( "What does it mean to index into a named array by " "a multidimensional array? %s" % index) # Rebuild the index of names for the various forms of slicing. names = sorted(self._index_names[dim].items(), key=lambda item: item[1]) names = np.array( names, dtype=object) # Support full numpy slicing. sliced = names[index] # Actually slice it. indexed = {n: j for j, (n, _) in enumerate(sliced)} # Reindex. if len(sliced) != len(indexed): # Names aren't unique, so drop the names for this dimension. indexed = None new_names.append(indexed) dim += 1 else: raise TypeError("Unknown index: %s; %s" % (type(index), index)) obj._index_names = new_names + self._index_names[dim:] if len(obj._index_names) != len(obj.shape): raise IndexError("Names don't match object shape: %s != %s" % (len(obj.shape), len(obj._index_names))) return obj
def compute_trajectory(dynamics, policy, initial_state, num_steps): """Compute a state trajectory given dynamics and a policy. Parameters ---------- dynamics : callable A function that takes the current state and action as input and returns the next state. policy : callable A function that takes the current state as input and returns the action. initial_state : Tensor or ndarray The initial state at which to start simulating. num_steps : int The number of steps for which to simulate the system. Returns ------- states : ndarray A (num_steps x state_dim) array with one state on each row. actions : ndarray A (num_steps x action_dim) array with the corresponding action on each row. """ initial_state = np.atleast_2d(initial_state) state_dim = initial_state.shape[1] # Get storage (indexed by dynamics and policy) index = (dynamics, policy) storage = get_storage(_STORAGE, index=index) if storage is None: # Compute next state under the policy tf_state = tf.placeholder(config.dtype, [1, state_dim]) tf_action = policy(tf_state) tf_next_state = dynamics(tf_state, tf_action) storage = [('tf_state', tf_state), ('tf_action', tf_action), ('tf_next_state', tf_next_state)] set_storage(_STORAGE, storage, index=index) else: tf_state, tf_action, tf_next_state = storage.values() # Initialize dtype = config.np_dtype states = np.empty((num_steps, state_dim), dtype=dtype) actions = np.empty((num_steps - 1, policy.output_dim), dtype=dtype) states[0, :] = initial_state # Get the feed dict session = tf.get_default_session() feed_dict = get_feed_dict(session.graph) next_data = [tf_next_state, tf_action] # Run simulation for i in range(num_steps - 1): feed_dict[tf_state] = states[[i], :] states[i + 1, :], actions[i, :] = session.run(next_data, feed_dict=feed_dict) return states, actions
def __init__(self, form, context, *args, **kwargs): """ Dynamically add each of the form fields for the given form model instance and its related field model instances. """ self.form = form self.form_fields = form.fields.visible() initial = kwargs.pop("initial", {}) # If a FormEntry instance is given to edit, stores it's field # values for using as initial data. field_entries = {} if kwargs.get("instance"): for field_entry in kwargs["instance"].fields.all(): field_entries[field_entry.field_id] = field_entry.value super(FormForForm, self).__init__(*args, **kwargs) # Create the form fields. for field in self.form_fields: field_key = field.slug field_class = fields.CLASSES[field.field_type] field_widget = fields.WIDGETS.get(field.field_type) field_args = { "label": field.label, "required": field.required, "help_text": field.help_text } arg_names = field_class.__init__.__code__.co_varnames if "max_length" in arg_names: field_args["max_length"] = settings.FIELD_MAX_LENGTH if "choices" in arg_names: field_args["choices"] = field.get_choices() if field_widget is not None: field_args["widget"] = field_widget # # Initial value for field, in order of preference: # # - If a form model instance is given (eg we're editing a # form response), then use the instance's value for the # field. # - If the developer has provided an explicit "initial" # dict, use it. # - The default value for the field instance as given in # the admin. # initial_val = None try: initial_val = field_entries[field.id] except KeyError: try: initial_val = initial[field_key] except KeyError: initial_val = Template(field.default).render(context) if initial_val: if field.is_a(*fields.MULTIPLE): initial_val = split_choices(initial_val) if field.field_type == fields.CHECKBOX: initial_val = initial_val != "False" self.initial[field_key] = initial_val self.fields[field_key] = field_class(**field_args) if field.field_type == fields.DOB: now = datetime.now() years = list(range(now.year, now.year - 120, -1)) self.fields[field_key].widget.years = years # Add identifying CSS classes to the field. css_class = field_class.__name__.lower() if field.required: css_class += " required" if (settings.USE_HTML5 and field.field_type != fields.CHECKBOX_MULTIPLE): self.fields[field_key].widget.attrs["required"] = "" self.fields[field_key].widget.attrs["class"] = css_class if field.placeholder_text and not field.default: text = field.placeholder_text self.fields[field_key].widget.attrs["placeholder"] = text
def straight(cells, prop, srcs, recs, velocity=None, par=False): """ Calculate the travel times inside a mesh of square cells between source and receiver pairs assuming the rays are straight lines (no refraction or reflection). .. note:: Don't care about the units as long they are compatible. For a homogeneous model, *cells* can be a list with only one big cell. Parameters: * cells : list of :func:`geoist.mesher.Square` The velocity model to use to trace the straight rays. Cells must have the physical property given in parameter *prop*. This will be used as the velocity of each cell. (*cells* can also be a :class:`~geoist.mesher.SquareMesh`) * prop : str Which physical property of the cells to use as velocity. Normaly one would choose ``'vp'`` or ``'vs'`` * srcs : list fo lists List with [x, y] coordinate pairs of the wave sources. * recs : list fo lists List with [x, y] coordinate pairs of the receivers sources * velocity : float or None If not None, will use this value instead of the prop of cells as the velocity. Useful when building sensitivity matrices (use velocity = 1). * par : True or False If True, will run the calculations in parallel using all the cores available. Not recommended for Jacobian matrix building! *srcs* and *recs* are lists of source-receiver pairs. Each source in *srcs* is associated with the corresponding receiver in *recs* for a given travel time. For example:: >>> # One source was recorded at 3 receivers. >>> # The medium is homogeneous and can be >>> # represented by a single Square >>> from geoist.mesher import Square >>> cells = [Square([0, 10, 0, 10], {'vp':2})] >>> src = (5, 0) >>> srcs = [src, src, src] >>> recs = [(0, 0), (5, 10), (10, 0)] >>> print straight(cells, 'vp', srcs, recs) [ 2.5 5. 2.5] Returns: * times : array The total times each ray took to get from a source to a receiver (in compatible units with *prop*) """ if len(srcs) != len(recs): raise ValueError("Must have the same number of sources and receivers") if not par: if _ttime2d is not None: x_src, y_src = numpy.transpose(srcs).astype(numpy.float) x_rec, y_rec = numpy.transpose(recs).astype(numpy.float) times = _ttime2d.straight(x_src, y_src, x_rec, y_rec, len(srcs), cells, velocity, prop) else: times = _straight(cells, prop, srcs, recs, velocity) return times # Divide the workload into jobs and run them in different processes jobs = multiprocessing.cpu_count() start = 0 size = len(srcs) perjob = size / jobs processes = [] pipes = [] for i in range(jobs): if i == jobs - 1: end = size else: end = start + perjob outpipe, inpipe = multiprocessing.Pipe() args = (inpipe, srcs[start:end], recs[ start:end], cells, velocity, prop) proc = multiprocessing.Process(target=_straight_job, args=args) proc.start() processes.append(proc) pipes.append(outpipe) start = end times = [] for proc, pipe in zip(processes, pipes): times.extend(pipe.recv()) proc.join() return numpy.array(times)
def GeneratePassphrase(length=20): """Create a 20 char passphrase with easily typeable chars.""" valid_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ" valid_chars += "0123456789 ,-_&$#" return "".join(random.choice(valid_chars) for i in range(length))
def test_contains(self): self.assertIn(1, range(2)) self.assertNotIn(10, range(2)) self.assertNotIn(None, range(2)) self.assertNotIn("", range(2))
def test_slice_overflow_range(self): r = range(8) self.assertRangesEqual(r[2:200], range(2, 8)) self.assertRangesEqual(r[-200:-2], range(0, 6))
def minimize(self, objective): stats = dict(method="Levemberg-Marquardt", iterations=0, objective=[], step_attempts=[], step_size=[]) p = np.array(self.initial) value = objective.value(p) lamb = self.lamb stats['objective'].append(value) stats['step_attempts'].append(0) stats['step_size'].append(lamb) for iteration in range(self.maxit): grad = objective.gradient(p) hess = objective.hessian(p) if self.precondition: diag = np.abs(safe_diagonal(hess)) diag[diag < 1e-10] = 1e-10 precond = sp.diags(1/diag, 0).tocsr() hess = safe_dot(precond, hess) grad = safe_dot(precond, grad) diag = sp.diags(safe_diagonal(hess), 0).tocsr() # Try to take a step took_step = False for step in range(self.maxsteps): newp = p + safe_solve(hess + lamb*diag, -grad) newvalue = objective.value(newp) decrease = newvalue < value if not decrease: if lamb < 1e15: lamb = lamb*self.dlamb else: if lamb > 1e-15: lamb = lamb/self.dlamb took_step = True break if not took_step: stop = True warnings.warn( "LevMarq optimizer exited because couldn't take a step " + 'without increasing the objective function. ' + 'Might not have achieved convergence. ' + 'Try increasing the max number of step attempts.', RuntimeWarning) else: stop = abs(newvalue - value) < self.tol*abs(value) p = newp value = newvalue stats['objective'].append(value) stats['iterations'] += 1 stats['step_attempts'].append(step + 1) stats['step_size'].append(lamb) if stop: break if iteration == self.maxit - 1: warnings.warn( 'LevMarq optmizer exited because maximum iterations reached. ' + 'Might not have achieved convergence. ' + 'Try increasing the maximum number of iterations allowed.', RuntimeWarning) self.stats = stats return p
def create_table(self, fromcell=None, tocell=None, nheader=0, sheet=0): """ Creates a table (as a list) based on given query and columns fromcell: The index of the cell where to begin. The default is from the beginning of the data set (0, 0). tocell: The index of the cell where to end the selection. Default is in the end of the data set. nheader: Number of lines which are considered as a header lines. Normally, the value is 0 (default) or 1. sheet: Name or index of the sheet as string/unicode. The index starts from the 0 and is the default value. If numeric value is given, provide it in format:: et.create_table(fromcell='A1', tocell='B2', sheet='2') """ rows = [] # Select sheet by given index or name if type(sheet) is int or sheet.isdigit(): sh1 = self.book.sheet_by_index(int(sheet)) else: sh1 = self.book.sheet_by_name(sheet) # Name selection, like: 'A1' or 'AB12' if isinstance(fromcell, basestring): match = re.match(r'(?P<chars>[A-Z]+)(?P<nums>[1-9]+[0-9]*)', fromcell) if match: parts = (match.group('chars'), int(match.group('nums'))) fromcell = toindex(*parts) else: fromcell = tuple([int(num) for num in fromcell.split(u',')]) # Name selection, like: 'A1' or 'AB12' if isinstance(tocell, basestring): match = re.match(r'(?P<chars>[A-Z]+)(?P<nums>[1-9]+[0-9]*)', tocell) if match: parts = (match.group('chars'), int(match.group('nums'))) tocell = toindex(*parts) else: tocell = tuple([int(num) for num in tocell.split(u',')]) if not fromcell: fromcell = (0, 0) # If ending cell is not given, calculate # it from rows and cols #print sh1.ncols, sh1.nrows #print (tocell[0] > (sh1.ncols -1)) or (tocell[1] > (sh1.nrows -1)) maxrow_index = sh1.nrows - 1 maxcol_index = sh1.ncols - 1 if not tocell: tocell = (maxcol_index, maxrow_index) # If the value is bigger than the value, default to max value if int(tocell[0]) > maxcol_index: tocell = (maxcol_index, tocell[1]) # If the value is bigger than the value, default to max value if int(tocell[1]) > maxrow_index: tocell = (tocell[0], maxrow_index) # Iterate columns rows = {'headers': [], 'rows': []} widths = [] for rnum in range(fromcell[1], tocell[1] + 1): # Iterate rows within column cols = [] for cnum in range(fromcell[0], tocell[0] + 1): cell = sh1.cell(rnum, cnum) width = sh1.computed_column_width(cnum) # Put data cell_data = { 'type': 'row', 'width': width, 'value': self._get_value(cell) } # If header row if rnum < nheader: cell_data['type'] = 'header' # Get more format info for the cell cell_data.update(self._get_formatting(cell)) cols.append(cell_data) # If first column is header, their all headers - i think if cols[0]['type'] == 'header': rows['headers'].append(cols) else: rows['rows'].append(cols) #widths_together = sum([cell['width'] for cols in rows]) #print widths_together #widths = [round(val * 100.0 / widths_together) for val in widths] # Store into object for validation purposes self.fromcell = fromcell self.tocell = tocell return rows
from future.builtins import range import numpy as np import modred as mr num_vecs = 30 # Arbitrary data vecs = np.random.random((100, num_vecs)) num_modes = 5 modes, eig_vals = mr.compute_POD_matrices_snaps_method(vecs, list(range(num_modes)))
def minimize(self, objective): stats = dict(method='Steepest Descent', iterations=0, objective=[], step_attempts=[]) p = np.array(self.initial) value = objective.value(p) stats['objective'].append(value) if self.linesearch: stats['step_attempts'].append(0) alpha = 1e-4 # This is a mystic parameter of the Armijo rule for iteration in range(self.maxit): grad = objective.gradient(p) if self.linesearch: # Calculate now to avoid computing inside the loop gradnorm = np.linalg.norm(grad)**2 # Determine the best step size took_step = False for i in range(self.maxsteps): stepsize = self.beta**i newp = p - stepsize*grad newvalue = objective.value(newp) decreased = newvalue < value good_step = newvalue - value < alpha*stepsize*gradnorm if decreased and good_step: took_step = True break else: newp = p - grad newvalue = objective.value(newp) decreased = newvalue < value if decreased: took_step = True else: took_step = False if not took_step: stop = True warnings.warn( "SteepestDescent optimizer exited because couldn't take a" + " step without increasing the objective function. " + 'Might not have achieved convergence. ' + 'Try increasing the max number of step attempts allowed.', RuntimeWarning) else: stop = abs(newvalue - value) < self.tol*abs(value) p = newp value = newvalue stats['objective'].append(value) stats['iterations'] += 1 if self.linesearch: stats['step_attempts'].append(i + 1) if stop: break if iteration == self.maxit - 1: warnings.warn( 'SteepestDescent optimizer exited because maximum iterations ' + 'reached. Might not have achieved convergence. ' + 'Try increasing the maximum number of iterations allowed.', RuntimeWarning) self.stats = stats return p
def __init__(self, zeros): """ Constructor """ QDialog.__init__(self) self.__zeros = zeros self.setWindowTitle(QCoreApplication.translate("VDLTools", "Zeros")) self.__layout = QGridLayout() self.__zeroLabels = [] self.__zeroChecks = [] displayButton = False self.__scrollLayout = QGridLayout() for i in range(len(self.__zeros)): msg = "- vertex " + str(self.__zeros[i][0]) msg += QCoreApplication.translate("VDLTools", ", elevation : '0', ") if self.__zeros[i][1] is not None: msg += QCoreApplication.translate("VDLTools", "interpolated elevation : ") msg += str(self.__zeros[i][1]) + "m" if self.__zeros[i][2] > 1: msg += QCoreApplication.translate("VDLTools", " (and apply to point)") msgCheck = QCheckBox() msgCheck.setChecked(True) self.__zeroChecks.append(msgCheck) self.__scrollLayout.addWidget(self.__zeroChecks[i], i + 1, 2) displayButton = True else: msg += QCoreApplication.translate("VDLTools", "no interpolated elevation") self.__zeroChecks.append(None) zeroLabel = QLabel(msg) self.__zeroLabels.append(zeroLabel) self.__scrollLayout.addWidget(self.__zeroLabels[i], i + 1, 0, 1, 2) widget = QWidget() widget.setLayout(self.__scrollLayout) scroll = QScrollArea() scroll.setWidgetResizable(True) scroll.setWidget(widget) self.__layout.addWidget(scroll, 1, 0, 1, 2) self.__passButton = QPushButton( QCoreApplication.translate("VDLTools", "Pass")) self.__passButton.setMinimumHeight(20) self.__passButton.setMinimumWidth(100) pos = len(self.__zeros) + 1 self.__layout.addWidget(self.__passButton, pos, 0) self.__applyButton = QPushButton( QCoreApplication.translate("VDLTools", "Apply interpolation")) self.__applyButton.setMinimumHeight(20) self.__applyButton.setMinimumWidth(100) if displayButton: self.__layout.addWidget(self.__applyButton, pos, 1) self.setLayout(self.__layout)
def get_lyapunov_region(lyapunov, discretization, init_node): """Get the region within which a function is a Lyapunov function. Parameters ---------- lyapunov : callable A tensorflow function. discretization : instance of `GridWorld` The discretization on which to check the increasing property. init_node : tuple The node at which to start the verification. Returns ------- region : ndarray A boolean array that contains all the states for which lyapunov is a Lyapunov function that can be used for stability verification. """ # Turn values into a multi-dim array feed_dict = lyapunov.feed_dict values = lyapunov(discretization.all_points).eval(feed_dict=feed_dict) lyapunov_values = values.reshape(discretization.num_points) # Starting point for the verification init_value = lyapunov_values[init_node] ndim = discretization.ndim num_points = discretization.num_points # Indices for generating neighbors index_generator = itertools.product(*[(0, -1, 1) for _ in range(ndim)]) neighbor_indeces = np.array(tuple(index_generator)[1:]) # Array keeping track of visited nodes visited = np.zeros(discretization.num_points, dtype=np.bool) visited[init_node] = True # Create priority queue tiebreaker = itertools.count() last_value = init_value priority_queue = [(init_value, tiebreaker.next(), init_node)] while priority_queue: value, _, next_node = heappop(priority_queue) # Check if we reached the boundary of the discretization if np.any(0 == next_node) or np.any(next_node == num_points - 1): visited[tuple(next_node)] = False break # Make sure we are in the positive definite part of the function. if value < last_value: break last_value = value # Get all neighbors neighbors = next_node + neighbor_indeces # Remove neighbors that are already part of the visited set is_new = ~visited[np.split(neighbors.T, ndim)] neighbors = neighbors[is_new[0]] if neighbors.size: indices = np.split(neighbors.T, ndim) # add to visited set visited[indices] = True # get values values = lyapunov_values[indices][0] # add to priority queue for value, neighbor in zip(values, neighbors): heappush(priority_queue, (value, next(tiebreaker), neighbor)) # Prune nodes that were neighbors, but haven't been visited for _, _, node in priority_queue: visited[tuple(node)] = False return visited
def _download(self, chunks, resume): if not resume: self.info.clear() self.info.add_chunk('{0}.chunk0'.format( self.path), (0, 0)) # create an initial entry self.chunks = [] # initial chunk that will load complete file (if needed) init = CurlChunk(0, self, None, resume) self.chunks.append(init) self.manager.add_handle(init.get_handle()) last_finish_check = 0 last_time_check = 0 chunks_done = set() # list of curl handles that are finished chunks_created = False done = False # This is a resume, if we were chunked originally assume still can if self.info.get_count() > 1: self.chunk_support = True while True: # need to create chunks # will be set later by first chunk if not chunks_created and self.chunk_support and self.size: self.flags ^= Connection.Resumable # TODO: Recheck... if not resume: self.info.set_size(self.size) self.info.create_chunks(chunks) self.info.save() chunks = self.info.get_count() init.set_range(self.info.get_chunk_range(0)) for i in range(1, chunks): c = CurlChunk( i, self, self.info.get_chunk_range(i), resume) handle = c.get_handle() if handle: self.chunks.append(c) self.manager.add_handle(handle) else: # close immediately self.log.debug('Invalid curl handle -> closed') c.close() chunks_created = True while True: ret, _ = self.manager.perform() if ret != pycurl.E_CALL_MULTI_PERFORM: break t = time.time() # reduce these calls # when num_q is 0, the loop is exited while last_finish_check + 0.5 < t: # list of failed curl handles failed = [] # TODO: Rewrite... # save only last exception, we can only raise one anyway exc = Exception() num_q, ok_list, err_list = self.manager.info_read() for c in ok_list: chunk = self.find_chunk(c) # check if the header implies success, # else add it to failed list try: chunk.verify_header() except ResponseException as exc: self.log.debug( 'Chunk {0:d} failed'.format( chunk.id + 1)) self.log.debug(exc, exc_info=True) failed.append(chunk) else: chunks_done.add(c) for c in err_list: curl, errno, msg = c chunk = self.find_chunk(curl) # test if chunk was finished if errno != 23 or '0 !=' not in msg: failed.append(chunk) exc = pycurl.error(errno, msg) self.log.debug( 'Chunk {0:d} failed'.format(chunk.id + 1)) self.log.debug(exc, exc_info=True) continue # check if the header implies success, # else add it to failed list try: chunk.verify_header() except ResponseException as exc: self.log.debug( 'Chunk {0:d} failed'.format( chunk.id + 1)) self.log.debug(exc, exc_info=True) failed.append(chunk) else: chunks_done.add(curl) if not num_q: # no more info to get # check if init is not finished so we reset download # connections # note that other chunks are closed and everything # downloaded with initial connection if failed: if init in failed or init.curl in chunks_done: raise exc self.log.error( 'Download chunks failed, fallback to ' 'single connection | {0}'.format(exc)) # list of chunks to clean and remove to_clean = [x for x in self.chunks if x is not init] for chunk in to_clean: self.close_chunk(chunk) self.chunks.remove(chunk) remove(self.info.get_chunk_name(chunk.id)) # let first chunk load the rest and update the # info file init.reset_range() self.info.clear() self.info.add_chunk('{0}.chunk0'.format( self.path), (0, self.size)) self.info.save() last_finish_check = t if len(chunks_done) >= len(self.chunks): if len(chunks_done) > len(self.chunks): self.log.warning( 'Finished download chunks size incorrect') done = True # all chunks loaded break if done: break # all chunks loaded # calc speed once per second, averaging over 3 seconds if last_time_check + 1 < t: len_la = len(self.last_arrived) diff = [c.arrived - (self.last_arrived[i] if len_la > i else 0) for i, c in enumerate(self.chunks)] self.last_speeds[1] = self.last_speeds[0] self.last_speeds[0] = self.speeds self.speeds = [float(a) // (t - last_time_check) for a in diff] self.last_arrived = [c.arrived for c in self.chunks] last_time_check = t if self._abort: raise Abort self.manager.select(1) for chunk in self.chunks: chunk.flush_file() # make sure downloads are written to disk self._copy_chunks()
def load_imitation_learning_network(input_image, input_data, mode): branches = [] x = input_image if mode == tf.estimator.ModeKeys.TRAIN: dropout = DROPOUT_VEC_TRAIN is_training = True else: dropout = DROPOUT_VEC_INFER is_training = False with tf.name_scope( 'Network' ): # for a nicer Tensorboard graph, use: `with tf.variable_scope('Network'):` network_manager = Network(dropout, tf.shape(x), is_training) """conv1""" # kernel sz, stride, num feature maps xc = network_manager.conv_block(x, 5, 2, 32, padding_in='VALID') xc = network_manager.conv_block(xc, 3, 1, 32, padding_in='VALID') """conv2""" xc = network_manager.conv_block(xc, 3, 2, 64, padding_in='VALID') xc = network_manager.conv_block(xc, 3, 1, 64, padding_in='VALID') """conv3""" xc = network_manager.conv_block(xc, 3, 2, 128, padding_in='VALID') xc = network_manager.conv_block(xc, 3, 1, 128, padding_in='VALID') """conv4""" xc = network_manager.conv_block(xc, 3, 1, 256, padding_in='VALID') xc = network_manager.conv_block(xc, 3, 1, 256, padding_in='VALID') """mp3 (default values)""" """ reshape """ x = tf.reshape(xc, [-1, int(np.prod(xc.get_shape()[1:]))], name='reshape') """ fc1 """ x = network_manager.fc_block(x, 512) """ fc2 """ x = network_manager.fc_block(x, 512) """Process Control""" """ Speed (measurements)""" with tf.name_scope("Speed"): speed = input_data[1] # get the speed from input data speed = network_manager.fc_block(speed, 128) speed = network_manager.fc_block(speed, 128) """ Joint sensory """ j = tf.concat([x, speed], 1) j = network_manager.fc_block(j, 512) """Start BRANCHING""" branch_config = [ [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE], [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE], [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE], [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE], [ilc.TGT_SPEED], ] for i in range(0, len(branch_config)): with tf.name_scope("Branch_{}".format(i)): if branch_config[i][0] == ilc.TGT_SPEED: # we only use the image as input to speed prediction branch_output = network_manager.fc_block(x, 256) branch_output = network_manager.fc_block( branch_output, 256) else: branch_output = network_manager.fc_block(j, 256) branch_output = network_manager.fc_block( branch_output, 256) branches.append( network_manager.fc(branch_output, len(branch_config[i]))) print(branch_output) return branches
def SetupClients(self, nr_clients, *args, **kwargs): """Prepares nr_clients test client mocks to be used.""" return self.SetupClientsWithIndices(range(nr_clients), *args, **kwargs)
def test_slice_zero_step(self): msg = '^slice step cannot be zero$' with self.assertRaisesRegexp(ValueError, msg): range(8)[::0]
def lin_combine(self, sum_vec_handles, basis_vec_handles, coeff_mat, coeff_mat_col_indices=None): """Computes linear combination(s) of basis vector objects and calls ``put`` on result(s), using handles. Args: ``sum_vec_handles``: List of handles for the sum vector objects. ``basis_vec_handles``: List of handles for the basis vector objects. ``coeff_mat``: Matrix whose rows correspond to basis vectors and whose columns correspond to sum vectors. The rows and columns correspond, by index, to the lists ``basis_vec_handles`` and ``sum_vec_handles``. In matrix notation, we can write ``sums = basis * coeff_mat`` Kwargs: ``coeff_mat_col_indices``: List of column indices. Only the ``sum_vecs`` corresponding to these columns of the coefficient matrix are computed. Each MPI worker (processor) retrieves a subset of the basis vectors to compute as many outputs as an MPI worker (processor) can have in memory at once. Each MPI worker (processor) computes the "layers" from the basis it is responsible for, and for as many modes as it can fit in memory. The layers from all MPI workers (processors) are summed together to form the ``sum_vecs`` and ``put`` is called on each. Scaling is: num gets/worker = :math:`n_s/(n_p*(max-2)) * n_b/n_p` passes/worker = :math:`(n_p-1) * n_s/(n_p*(max-2)) * (n_b/n_p)` scalar multiplies/worker = :math:`n_s*n_b/n_p` where :math:`n_s` is number of sum vecs, :math:`n_b` is number of basis vecs, :math:`n_p` is number of processors, :math:`max` = ``max_vecs_per_node``. """ sum_vec_handles = util.make_iterable(sum_vec_handles) basis_vec_handles = util.make_iterable(basis_vec_handles) num_bases = len(basis_vec_handles) num_sums = len(sum_vec_handles) if coeff_mat_col_indices is not None: coeff_mat = coeff_mat[:, coeff_mat_col_indices] if num_bases != coeff_mat.shape[0]: raise ValueError(('Number of coeff_mat rows (%d) does not equal ' 'number of basis handles (%d)' % (coeff_mat.shape[0], num_bases))) if num_sums != coeff_mat.shape[1]: raise ValueError(('Number of coeff_mat cols (%d) does not equal ' 'number of output handles (%d)') % (coeff_mat.shape[1], num_sums)) # Estimate time it will take # Burn the first one for slow imports test_vec_burn = basis_vec_handles[0].get() test_vec_burn_3 = test_vec_burn + 2. * test_vec_burn del test_vec_burn, test_vec_burn_3 start_time = time() test_vec = basis_vec_handles[0].get() get_time = time() - start_time start_time = time() test_vec_3 = test_vec + 2. * test_vec add_scale_time = time() - start_time del test_vec, test_vec_3 vecs_per_worker = self.max_vecs_per_node * parallel.get_num_nodes() / \ parallel.get_num_MPI_workers() num_gets = num_sums/(parallel.get_num_MPI_workers()*(\ vecs_per_worker-2)) + \ num_bases/parallel.get_num_MPI_workers() num_add_scales = num_sums * num_bases / parallel.get_num_MPI_workers() self.print_msg('Linear combinations will take at least %.1f minutes' % (num_gets * get_time / 60. + num_add_scales * add_scale_time / 60.)) # convenience rank = parallel.get_rank() # num_bases_per_proc_chunk is the num of bases each proc gets at once. num_bases_per_proc_chunk = 1 num_sums_per_proc_chunk = self.max_vecs_per_proc - \ num_bases_per_proc_chunk basis_tasks = parallel.find_assignments(list(range(num_bases))) sum_tasks = parallel.find_assignments(list(range(num_sums))) # Find max number tasks among all processors max_num_basis_tasks = max([len(tasks) for tasks in basis_tasks]) max_num_sum_tasks = max([len(tasks) for tasks in sum_tasks]) # These variables are the number of iters through loops that retrieve # ("get") # and "put" basis and sum vecs. num_basis_get_iters = int( np.ceil(max_num_basis_tasks * 1. / num_bases_per_proc_chunk)) num_sum_put_iters = int( np.ceil(max_num_sum_tasks * 1. / num_sums_per_proc_chunk)) if num_sum_put_iters > 1: self.print_msg( 'Warning: The basis vecs, ' 'of which there are %d, will be retrieved %d times each. ' 'If possible, increase number of nodes or ' 'max_vecs_per_node to reduce redundant retrieves and get a ' 'big speedup.' % (num_bases, num_sum_put_iters)) for sum_put_index in range(num_sum_put_iters): if len(sum_tasks[rank]) > 0: start_sum_index = min( sum_tasks[rank][0] + sum_put_index * num_sums_per_proc_chunk, sum_tasks[rank][-1] + 1) end_sum_index = min(start_sum_index + num_sums_per_proc_chunk, sum_tasks[rank][-1] + 1) # Create empty list on each processor sum_layers = [None] * (end_sum_index - start_sum_index) else: start_sum_index = 0 end_sum_index = 0 sum_layers = [] for basis_get_index in range(num_basis_get_iters): if len(basis_tasks[rank]) > 0: start_basis_index = min( basis_tasks[rank][0] + basis_get_index * num_bases_per_proc_chunk, basis_tasks[rank][-1] + 1) end_basis_index = min( start_basis_index + num_bases_per_proc_chunk, basis_tasks[rank][-1] + 1) basis_indices = list( range(start_basis_index, end_basis_index)) else: basis_indices = [] # Pass the basis vecs to proc with rank -> mod(rank+1,num_procs) # Must do this for each processor, until data makes a circle basis_vecs_recv = (None, None) for pass_index in range(parallel.get_num_procs()): # If on the first pass, retrieve the basis vecs, # no send/recv. # This is all that is called when in serial, # loop iterates once. if pass_index == 0: if len(basis_indices) > 0: basis_vecs = [basis_handle.get() \ for basis_handle in basis_vec_handles[ basis_indices[0]:basis_indices[-1]+1]] else: basis_vecs = [] else: # Figure out with whom to communicate source = (parallel.get_rank()-1) % \ parallel.get_num_procs() dest = (parallel.get_rank()+1) % \ parallel.get_num_procs() #Create unique tags based on ranks send_tag = parallel.get_rank() * \ (parallel.get_num_procs()+1) + dest recv_tag = source*(parallel.get_num_procs()+1) + \ parallel.get_rank() # Send/receive data basis_vecs_send = (basis_vecs, basis_indices) request = parallel.comm.isend(basis_vecs_send, dest=dest, tag=send_tag) basis_vecs_recv = parallel.comm.recv(source=source, tag=recv_tag) request.Wait() parallel.barrier() basis_indices = basis_vecs_recv[1] basis_vecs = basis_vecs_recv[0] # Compute the scalar multiplications for this set of data. # basis_indices stores the indices of the coeff_mat to # use. for sum_index in range(start_sum_index, end_sum_index): for basis_index, basis_vec in enumerate(basis_vecs): sum_layer = basis_vec * \ coeff_mat[basis_indices[basis_index],\ sum_index] if sum_layers[sum_index - start_sum_index] is None: sum_layers[sum_index-start_sum_index] = \ sum_layer else: sum_layers[sum_index-start_sum_index] += \ sum_layer if ((time() - self.prev_print_time) > self.print_interval): self.print_msg( 'Completed %.1f%% of linear combinations' % (sum_index * 100. / len(sum_tasks[rank]))) self.prev_print_time = time() # Completed this set of sum vecs, puts them to memory or file for sum_index in range(start_sum_index, end_sum_index): sum_vec_handles[sum_index].put(sum_layers[sum_index - start_sum_index]) del sum_layers self.print_msg('Completed %.1f%% of linear combinations' % 100.) self.prev_print_time = time() parallel.barrier()
def grouped_impl(wrap, size, sequence): for i in range(0, len(sequence), size): yield wrap(sequence[i:i + size])