Пример #1
0
    def __checkSituations(self):
        """
        To check if point with no elevation on line, and one or more elevation from other layers,
        and if there are different elevations at the same point
        """
        situations = []
        differences = []
        for p in range(len(self.__points)):
            pt = self.__points[p]
            num_lines = len(self.__selectedIds)
            zz = []
            for i in range(num_lines):
                if pt['z'][i] is not None:
                    zz.append(i)
            if len(zz) == 0:
                self.__iface.messageBar().pushMessage(
                    QCoreApplication.translate("VDLTools", "No line z ?!?"), level=QgsMessageBar.WARNING)
            elif len(zz) == 1:
                z0 = pt['z'][zz[0]]
                for i in range(num_lines, len(pt['z'])):
                    if pt['z'][i] is None:
                        continue
                    if abs(pt['z'][i]-z0) > self.ALT_TOLERANCE:
                        situations.append({'point': p, 'layer': (i-num_lines+1), 'vertex': z0})
            elif len(zz) == 2:
                z0 = pt['z'][zz[0]]
                if abs(pt['z'][zz[1]] - z0) > self.ALT_TOLERANCE:
                    differences.append({'point': p, 'v1': z0, 'v2': pt['z'][zz[1]]})
                else:
                    for i in range(num_lines, len(pt['z'])):
                        if pt['z'][i] is None:
                            continue
                        if abs(pt['z'][i]-z0) > self.ALT_TOLERANCE:
                            situations.append({'point': p, 'layer': (i-num_lines+1), 'vertex': z0})
            else:
                self.__iface.messageBar().pushMessage(
                    QCoreApplication.translate("VDLTools", "More than 2 lines z ?!?"), level=QgsMessageBar.WARNING)

        if (len(situations) > 0) or (len(differences) > 0):
            self.__setMessageDialog(situations, differences, self.__getNames())
            self.__rubberSit.reset()
            self.__rubberDif.reset()
            for situation in situations:
                pt = self.__points[situation['point']]
                point = QgsPoint(pt['x'], pt['y'])
                if self.__rubberSit.numberOfVertices() == 0:
                    self.__rubberSit.setToGeometry(QgsGeometry().fromPoint(point), None)
                else:
                    self.__rubberSit.addPoint(point)
            for difference in differences:
                pt = self.__points[difference['point']]
                point = QgsPoint(pt['x'], pt['y'])
                if self.__rubberDif.numberOfVertices() == 0:
                    self.__rubberDif.setToGeometry(QgsGeometry().fromPoint(point), None)
                else:
                    self.__rubberDif.addPoint(point)

            self.__msgDlg.show()
        else:
            self.__checkZeros()
Пример #2
0
def english_to_key (s):
    """english_to_key(string):string(2.x)/bytes(2.x)
    Transform a string into a corresponding key.
    The string must contain words separated by whitespace; the number
    of words must be a multiple of 6.
    """

    L=s.upper().split() ; key=b('')
    for index in range(0, len(L), 6):
        sublist=L[index:index+6] ; char=9*[0] ; bits=0
        for i in sublist:
            index = wordlist.index(i)
            shift = (8-(bits+11)%8) %8
            y = index << shift
            cl, cc, cr = (y>>16), (y>>8)&0xff, y & 0xff
            if (shift>5):
                char[bits>>3] = char[bits>>3] | cl
                char[(bits>>3)+1] = char[(bits>>3)+1] | cc
                char[(bits>>3)+2] = char[(bits>>3)+2] | cr
            elif shift>-3:
                char[bits>>3] = char[bits>>3] | cc
                char[(bits>>3)+1] = char[(bits>>3)+1] | cr
            else: char[bits>>3] = char[bits>>3] | cr
            bits=bits+11
        subkey=reduce(lambda x,y:x+bchr(y), char, b(''))

        # Check the parity of the resulting key
        skbin=_key2bin(subkey)
        p=0
        for i in range(0, 64, 2): p=p+_extract(skbin, i, 2)
        if (p&3) != _extract(skbin, 64, 2):
            raise ValueError("Parity error in resulting key")
        key=key+subkey[0:8]
    return key
Пример #3
0
  def Run(self):
    if data_store.RelationalDBEnabled():
      replace = {}
      for i in range(0, 2):
        with test_lib.FakeTime((1 + i) * 1000):
          hunt_id = self.CreateHunt(description="hunt_%d" % i)
          if i % 2:
            hunt.StopHunt(hunt_id)

          replace[hunt_id] = "H:00000%d" % i
    else:
      replace = {}
      for i in range(0, 2):
        with test_lib.FakeTime((1 + i) * 1000):
          with self.CreateHunt(description="hunt_%d" % i) as hunt_obj:
            if i % 2:
              hunt_obj.Stop()

            replace[hunt_obj.urn.Basename()] = "H:00000%d" % i

    self.Check(
        "ListHunts", args=hunt_plugin.ApiListHuntsArgs(), replace=replace)
    self.Check(
        "ListHunts",
        args=hunt_plugin.ApiListHuntsArgs(count=1),
        replace=replace)
    self.Check(
        "ListHunts",
        args=hunt_plugin.ApiListHuntsArgs(offset=1, count=1),
        replace=replace)
Пример #4
0
  def testReturnsAllDataByDefault(self):
    """Checks that stats collection works."""
    stats_collector = stats_collector_instance.Get()
    stats_collector.IncrementCounter("grr_client_received_bytes", 1566)
    stats_collector.IncrementCounter("grr_client_sent_bytes", 2000)

    results = self.RunAction(
        admin.GetClientStats,
        grr_worker=MockClientWorker(),
        arg=rdf_client_action.GetClientStatsRequest())

    response = results[0]
    self.assertEqual(response.bytes_received, 1566)
    self.assertEqual(response.bytes_sent, 2000)

    self.assertLen(response.cpu_samples, 3)
    for i in range(3):
      self.assertEqual(response.cpu_samples[i].timestamp,
                       rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100 + i * 10))
      self.assertAlmostEqual(response.cpu_samples[i].user_cpu_time, 0.1)
      self.assertAlmostEqual(response.cpu_samples[i].system_cpu_time,
                             0.1 * (i + 1))
      self.assertAlmostEqual(response.cpu_samples[i].cpu_percent, 10.0 + 5 * i)

    self.assertLen(response.io_samples, 3)
    for i in range(3):
      self.assertEqual(response.io_samples[i].timestamp,
                       rdfvalue.RDFDatetime.FromSecondsSinceEpoch(100 + i * 10))
      self.assertEqual(response.io_samples[i].read_bytes, 100 * (i + 1))
      self.assertEqual(response.io_samples[i].write_bytes, 100 * (i + 1))

    self.assertEqual(response.boot_time, 100 * 1e6)
Пример #5
0
  def testNotificationsAreDeletedFromAllShards(self):
    manager = queue_manager.QueueManager(token=self.token)
    manager.QueueNotification(
        session_id=rdfvalue.SessionID(
            base="aff4:/hunts", queue=queues.HUNTS, flow_name="42"))
    manager.Flush()
    manager.QueueNotification(
        session_id=rdfvalue.SessionID(
            base="aff4:/hunts", queue=queues.HUNTS, flow_name="43"))
    manager.Flush()
    # There should be two notifications in two different shards.
    shards_with_data = 0
    for _ in range(manager.num_notification_shards):
      shard_sessions = manager.GetNotifications(queues.HUNTS)
      if shard_sessions:
        shards_with_data += 1
        self.assertLen(shard_sessions, 1)
    self.assertEqual(shards_with_data, 2)

    # This should still work, as we delete notifications from all shards.
    manager.DeleteNotification(
        rdfvalue.SessionID(
            base="aff4:/hunts", queue=queues.HUNTS, flow_name="43"))
    manager.DeleteNotification(
        rdfvalue.SessionID(
            base="aff4:/hunts", queue=queues.HUNTS, flow_name="42"))
    for _ in range(manager.num_notification_shards):
      shard_sessions = manager.GetNotifications(queues.HUNTS)
      self.assertFalse(shard_sessions)
Пример #6
0
def inner_product_mat(num_states, num_rows, num_cols, max_vecs_per_node, 
    verbosity=1):
    """
    Computes inner products from known vecs.
    
    Remember that rows correspond to adjoint modes and cols to direct modes
    """
    col_vec_handles = [mr.VecHandlePickle(join(data_dir, col_vec_name%col_num))
        for col_num in range(num_cols)]
    row_vec_handles = [mr.VecHandlePickle(join(data_dir, row_vec_name%row_num))
        for row_num in range(num_rows)]
    
    generate_vecs(data_dir, num_states, row_vec_handles+col_vec_handles)
    
    my_VS = mr.VectorSpaceHandles(np.vdot, max_vecs_per_node=max_vecs_per_node,
        verbosity=verbosity) 
    
    prof = cProfile.Profile()
    start_time = T.time()
    prof.runcall(my_VS.compute_inner_product_mat, *(col_vec_handles,  
        row_vec_handles))
    total_time = T.time() - start_time
    prof.dump_stats('IP_mat_r%d.prof'%_parallel.get_rank())

    return total_time
Пример #7
0
def lin_combine(num_states, num_bases, num_products, max_vecs_per_node,
    verbosity=1):
    """
    Computes linear combination of vecs from saved vecs and random coeffs
    
    num_bases is number of vecs to be linearly combined
    num_products is the resulting number of vecs
    """

    basis_handles = [mr.VecHandlePickle(join(data_dir, basis_name%basis_num))
        for basis_num in range(num_bases)]
    product_handles = [mr.VecHandlePickle(join(data_dir, 
        product_name%product_num))
        for product_num in range(num_products)]

    generate_vecs(data_dir, num_states, basis_handles)
    my_VS = mr.VectorSpaceHandles(np.vdot, max_vecs_per_node=max_vecs_per_node,
        verbosity=verbosity)
    coeff_mat = np.random.random((num_bases, num_products))
    _parallel.barrier()

    prof = cProfile.Profile()
    start_time = T.time()
    prof.runcall(my_VS.lin_combine, *(product_handles, basis_handles, 
        coeff_mat)) 
    total_time = T.time() - start_time
    prof.dump_stats('lincomb_r%d.prof'%_parallel.get_rank())
    return total_time
Пример #8
0
Файл: klib.py Проект: LUMC/kPAL
    def _ratios_matrix(self):
        """
        Calculate all relative frequencies of *k*-mers. If a division by 0
        occurs, the frequency will be set to -1.0.

        :return: A matrix with relative frequencies.
        :rtype: float[][]
        """
        # Perhaps we can use something like this for a future normalization
        # operation.
        # Todo: Do this directly with NumPy.
        ratios = []
        for i in range(self.number):
            ratios.append(self.number * [0.0])

        # Fill the matrix.
        for i in range(self.number):
            for j in range(self.number):
                if self.counts[j]:
                    ratios[i][j] = (self.counts[i] /
                                    self.counts[j]) / self.total
                else:
                    ratios[i][j] = -1.0

        return ratios
Пример #9
0
  def TestOffsetAndCount(self,
                         fetch_all_fn,
                         fetch_range_fn,
                         error_desc = None):
    """Tests a DB API method with different offset/count combinations.

    This helper method works by first fetching all available objects with
    fetch_all_fn and then fetching all possible ranges using fetch_fn. The test
    passes if subranges returned by fetch_fn match subranges of values in
    the list returned by fetch_all_fn.

    Args:
      fetch_all_fn: Function without arguments that fetches all available
        objects using the API method that's being tested.
      fetch_range_fn: Function that calls an API method that's being tested
        passing 2 positional arguments: offset and count. It should return a
          list of objects.
      error_desc: Optional string to be used in error messages. May be useful to
        identify errors from a particular test.
    """
    all_objects = fetch_all_fn()
    self.assertNotEmpty(all_objects,
                        "Fetched objects can't be empty (%s)." % error_desc)

    for i in range(len(all_objects)):
      for l in range(1, len(all_objects) + 1):
        results = fetch_range_fn(i, l)
        expected = all_objects[i:i + l]

        self.assertListEqual(
            results, expected,
            "Results differ from expected (offset %d, count %d%s): %s vs %s" %
            (i, l,
             (", " + error_desc) if error_desc else "", results, expected))
Пример #10
0
def plot_lstsq(rec, ax=None, fname=None, base=np.e):
    err, g0, b, W, R, info = rec
    tcoda, tbulk, Ecoda, Ebulk, Gcoda, Gbulk = info
    fig = None
    if ax is None:
        fig = plt.figure()
        ax = fig.add_subplot(111)
    tmin = min(tcoda[i][0] for i in range(len(tcoda)))
    tmax = max(tcoda[i][-1] for i in range(len(tcoda)))
    for i in range(len(tcoda)):
        offset = R[i % len(R)] * W[i // len(R)] / W[0]
        #offset = R[i] if len(W) == 1 else C[i]
        #Bci = np.log(Ecoda[i]) - np.log(FS * Gcoda[i]) - np.log(offset)  # + R0
        Bci = np.log(Ecoda[i]) - np.log(Gcoda[i]) - np.log(offset)
        ax.plot(tcoda[i], Bci / np.log(base), color='0.7')
    for i in range(len(tbulk)):
        #offset = R[i] if len(W) == 1 else C[i]
        offset = R[i % len(R)] * W[i // len(R)] / W[0]
        #Bbi = np.log(Ebulk[i]) - np.log(FS * Gbulk[i]) - np.log(offset)  # + R0
        Bbi = np.log(Ebulk[i]) - np.log(Gbulk[i]) - np.log(offset)
        ax.plot(tbulk[i], Bbi / np.log(base), 'o', color='0.4', mec='0.4',
                ms=MS)
        tmin = min(tmin, tbulk[i])
    t = np.linspace(tmin, tmax, 100)
    ax.plot(t, (np.log(W[0]) - b * t) / np.log(base), color='k')
    ax.set_xlim(right=tmax)
    if fig and fname:
        _savefig(fig, fname=fname)
Пример #11
0
  def testFilterConsidersOffsetAndCount(self):
    client_id = self.client_ids[0]

    # Create five approval requests without granting them.
    for i in range(10):
      with test_lib.FakeTime(42 + i):
        self.RequestClientApproval(
            client_id.Basename(), reason="Request reason %d" % i)

    args = user_plugin.ApiListClientApprovalsArgs(
        client_id=client_id, offset=0, count=5)
    result = self.handler.Handle(args, token=self.token)

    # Approvals are returned newest to oldest, so the first five approvals
    # have reason 9 to 5.
    self.assertLen(result.items, 5)
    for item, i in zip(result.items, reversed(range(6, 10))):
      self.assertEqual(item.reason, "Request reason %d" % i)

    # When no count is specified, take all items from offset to the end.
    args = user_plugin.ApiListClientApprovalsArgs(client_id=client_id, offset=7)
    result = self.handler.Handle(args, token=self.token)

    self.assertLen(result.items, 3)
    for item, i in zip(result.items, reversed(range(0, 3))):
      self.assertEqual(item.reason, "Request reason %d" % i)
Пример #12
0
def _create_legend(plot_axes, distribution_markers, distribution_labels,
                   marker_type):
    """Creates a legend on the supplied axes."""
    # We have to use a proxy artist for the legend because box plots currently
    # don't have a very useful legend in matplotlib, and using the default
    # legend for bar/scatterplots chokes on empty/null distributions.
    #
    # Note: This code is based on the following examples:
    #   http://matplotlib.sourceforge.net/users/legend_guide.html
    #   http://stackoverflow.com/a/11423554
    if len(distribution_markers) != len(distribution_labels):
        raise ValueError("The number of distribution markers does not match "
                         "the number of distribution labels.")
    if marker_type == 'colors':
        legend_proxy = [Rectangle((0, 0), 1, 1, fc=marker)
                        for marker in distribution_markers]
        plot_axes.legend(legend_proxy, distribution_labels, loc='best')
    elif marker_type == 'symbols':
        legend_proxy = [Line2D(range(1), range(1), color='white',
                        markerfacecolor='black', marker=marker)
                        for marker in distribution_markers]
        plot_axes.legend(legend_proxy, distribution_labels, numpoints=3,
                         scatterpoints=3, loc='best')
    else:
        raise ValueError("Invalid marker_type: '%s'. marker_type must be "
                         "either 'colors' or 'symbols'." % marker_type)
Пример #13
0
        def thdGetLogLines(conn):
            # get a set of chunks that completely cover the requested range
            tbl = self.db.model.logchunks
            q = sa.select([tbl.c.first_line, tbl.c.last_line,
                           tbl.c.content, tbl.c.compressed])
            q = q.where(tbl.c.logid == logid)
            q = q.where(tbl.c.first_line <= last_line)
            q = q.where(tbl.c.last_line >= first_line)
            q = q.order_by(tbl.c.first_line)
            rv = []
            for row in conn.execute(q):
                # Retrieve associated "reader" and extract the data
                # Note that row.content is stored as bytes, and our caller expects unicode
                data = self.COMPRESSION_BYID[
                    row.compressed]["read"](row.content)
                content = data.decode('utf-8')

                if row.first_line < first_line:
                    idx = -1
                    count = first_line - row.first_line
                    for _ in range(count):
                        idx = content.index('\n', idx + 1)
                    content = content[idx + 1:]
                if row.last_line > last_line:
                    idx = len(content) + 1
                    count = row.last_line - last_line
                    for _ in range(count):
                        idx = content.rindex('\n', 0, idx)
                    content = content[:idx]
                rv.append(content)
            return u'\n'.join(rv) + u'\n' if rv else u''
Пример #14
0
    def distances(self, distance_fn):
        """Compute distances between all pairs of sequences

        Parameters
        ----------
        distance_fn : function
            Function for computing the distance between a pair of sequences.
            This must take two sequences as input (as `skbio.Sequence` objects)
            and return a single integer or float value.

        Returns
        -------
        skbio.DistanceMatrix
            Matrix containing the distances between all pairs of sequences.

        """
        sequence_count = self.sequence_count()
        dm = np.zeros((sequence_count, sequence_count))
        ids = []
        for i in range(sequence_count):
            self_i = self[i]
            ids.append(self_i.metadata['id'])
            for j in range(i):
                dm[i, j] = dm[j, i] = self_i.distance(self[j], distance_fn)
        return DistanceMatrix(dm, ids)
Пример #15
0
    def __lt__(self, o):
        """less than `o`

        :param Route o: other rule to compare with
        """
        if self == o:
            return False

        variables, strings = self._sort_struct()
        variables_o, strings_o = o._sort_struct()
        if len(variables) != len(variables_o):
            return len(variables) < len(variables_o)

        if len(''.join(strings)) != len(''.join(strings_o)):
            return len(''.join(strings)) > len(''.join(strings_o))

        for i in range(len(strings)):
            if strings[i] != strings_o[i]:
                return strings[i] < strings_o[i]

        # strings are the same, so check variables for non-default parser
        for i in range(len(variables)):
            if variables[i] != 'str' and variables_o[i] == 'str':
                return True
        return False
Пример #16
0
  def setUp(self):
    super(SystemCronTestMixin, self).setUp()

    one_hour_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("1h")
    eight_day_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("8d")
    ancient_ping = rdfvalue.RDFDatetime.Now() - rdfvalue.Duration("61d")

    self.SetupClientsWithIndices(
        range(0, 10), system="Windows", ping=eight_day_ping)
    self.SetupClientsWithIndices(
        range(10, 20), system="Linux", ping=eight_day_ping)
    self.SetupClientsWithIndices(
        range(20, 22),
        system="Darwin",
        fleetspeak_enabled=True,
        ping=one_hour_ping)
    # These clients shouldn't be analyzed by any of the stats cronjobs.
    self.SetupClientsWithIndices(
        range(22, 24), system="Linux", ping=ancient_ping)

    for i in range(0, 10):
      client_id = u"C.1%015x" % i
      if data_store.AFF4Enabled():
        with aff4.FACTORY.Open(
            client_id, mode="rw", token=self.token) as client:
          client.AddLabels([u"Label1", u"Label2"], owner=u"GRR")
          client.AddLabel(u"UserLabel", owner=u"jim")

      if data_store.RelationalDBEnabled():
        data_store.REL_DB.AddClientLabels(client_id, u"GRR",
                                          [u"Label1", u"Label2"])
        data_store.REL_DB.AddClientLabels(client_id, u"jim", [u"UserLabel"])
Пример #17
0
def fd1d(size):
    """
    Produce a 1D finite difference matrix.

    Parameters:

    * size : int
        The number of points

    Returns:

    * fd : sparse CSR matrix
        The finite difference matrix

    Examples:

    >>> fd1d(2).todense()
    matrix([[ 1, -1]])
    >>> fd1d(3).todense()
    matrix([[ 1, -1,  0],
            [ 0,  1, -1]])
    >>> fd1d(4).todense()
    matrix([[ 1, -1,  0,  0],
            [ 0,  1, -1,  0],
            [ 0,  0,  1, -1]])

    """
    i = list(range(size - 1)) + list(range(size - 1))
    j = list(range(size - 1)) + list(range(1, size))
    v = [1] * (size - 1) + [-1] * (size - 1)
    return scipy.sparse.coo_matrix((v, (i, j)), (size - 1, size)).tocsr()
Пример #18
0
  def testOffsetIsRelativeToFilteredResultsWhenFilterIsPresent(self):
    for i in range(5):
      self.CreateHunt(description="foo_hunt_%d" % i)

    for i in range(3):
      self.CreateHunt(description="bar_hunt_%d" % i)

    result = self.handler.Handle(
        hunt_plugin.ApiListHuntsArgs(
            description_contains="bar", active_within="1d", offset=1),
        token=self.token)
    self.assertLen(result.items, 2)
    for item in result.items:
      self.assertIn("bar", item.description)

    result = self.handler.Handle(
        hunt_plugin.ApiListHuntsArgs(
            description_contains="bar", active_within="1d", offset=2),
        token=self.token)
    self.assertLen(result.items, 1)
    for item in result.items:
      self.assertIn("bar", item.description)

    result = self.handler.Handle(
        hunt_plugin.ApiListHuntsArgs(
            description_contains="bar", active_within="1d", offset=3),
        token=self.token)
    self.assertEmpty(result.items)
Пример #19
0
    def do_addLogLines_huge_log(self, NUM_CHUNKS=3000, chunk=(u'xy' * 70 + u'\n') * 3):
        if chunk.endswith("\n"):
            chunk = chunk[:-1]
        linesperchunk = chunk.count("\n") + 1
        test_data = [
            fakedb.LogChunk(logid=201, first_line=i * linesperchunk,
                            last_line=i * linesperchunk + linesperchunk - 1, compressed=0,
                            content=chunk)
            for i in range(NUM_CHUNKS)
        ]
        yield self.insertTestData(
            self.backgroundData + [
                fakedb.Log(id=201, stepid=101, name=u'stdio', slug=u'stdio',
                           complete=0, num_lines=NUM_CHUNKS * 3, type=u's')] +
            test_data)
        wholeLog = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3)
        for i in range(10):
            yield self.db.logs.compressLog(201)
            wholeLog2 = yield self.db.logs.getLogLines(201, 0, NUM_CHUNKS * 3)
            self.assertEqual(wholeLog, wholeLog2)
        self.assertEqual(wholeLog, wholeLog2)

        def countChunk(conn):
            tbl = self.db.model.logchunks
            q = sa.select([sa.func.count(tbl.c.content)])
            q = q.where(tbl.c.logid == 201)
            return conn.execute(q).fetchone()[0]

        chunks = yield self.db.pool.do(countChunk)
        # make sure MAX_CHUNK_LINES is taken in account
        self.assertGreaterEqual(
            chunks, NUM_CHUNKS * linesperchunk / logs.LogsConnectorComponent.MAX_CHUNK_LINES)
Пример #20
0
def _fasta_sniffer(fh):
    # Strategy:
    #   Ignore up to 5 blank/whitespace-only lines at the beginning of the
    #   file. Read up to 10 FASTA records. If at least one record is read (i.e.
    #   the file isn't empty) and no errors are thrown during reading, assume
    #   the file is in FASTA format. Next, try to parse the file as QUAL, which
    #   has stricter requirements. If this succeeds, do *not* identify the file
    #   as FASTA since we don't want to sniff QUAL files as FASTA (technically
    #   they can be read as FASTA since the sequences aren't validated but it
    #   probably isn't what the user wanted). Also, if we add QUAL as its own
    #   file format in the future, we wouldn't want the FASTA and QUAL sniffers
    #   to both identify a QUAL file.
    if _too_many_blanks(fh, 5):
        return False, {}

    num_records = 10
    try:
        not_empty = False
        for _ in zip(range(num_records), _fasta_to_generator(fh)):
            not_empty = True

        if not_empty:
            fh.seek(0)
            try:
                list(zip(range(num_records),
                         _parse_fasta_raw(fh, _parse_quality_scores, 'QUAL')))
            except FASTAFormatError:
                return True, {}
            else:
                return False, {}
        else:
            return False, {}
    except FASTAFormatError:
        return False, {}
 def test_filter(self):
     levPairs = [('Mammalia', 'Primates'),
                 ('Vertebrata', 'Euarchontoglires'),
                 ('Euarchontoglires', 'Rodents'),
                 ('Vertebrata', 'HUMAN'),
                 ('Primates', 'PANTR'),
                 ('Primates', 'HUMAN')]
     expRes = [
               [
                [fa.FamIdent('1'), fa.FamIdent('2')],
                [fa.FamIdent('1')],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')],
                [fa.FamIdent('1')],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a')]
               ],
               [
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamDupl('3', ['3.1a', '3.1b'])],
                [fa.FamIdent('1'), fa.FamNovel('2'), fa.FamDupl('3', ['3.1a', '3.1b'])],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')],
                [fa.FamIdent('1'), fa.FamNovel('2'), fa.FamDupl('3', '3.1a'), fa.FamSingleton('5')],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamIdent('3.1b')],
                [fa.FamIdent('1'), fa.FamIdent('2'), fa.FamIdent('3.1a'), fa.FamLost('3.1b'), fa.FamSingleton('5')]
               ]
              ]
     filters = ['identical', {'identical', 'lost', 'singleton', 'novel', 'duplicated'}]
     res = [ [], [] ]
     for i in range(len(levPairs)):
         lev1, lev2 = levPairs[i]
         comp = self.compareLevelsSingletonAware(lev1, lev2)
         for j in range(len(filters)):
             fams = comp.filter(filters[j])
             res[j].append(fams)
             self.assertListEqual(res[j][i], expRes[j][i], "failed for {} vs {}".format(lev1, lev2))
Пример #22
0
 def _gen_rrg_block(self, xmlgen, block):
     """Generate a <block_type> tag for the given ``block``.
 
     Args:
         xmlgen (`XMLGenerator`): the generator to be used
         block (:obj:`dict`): a `dict` satisfying the JSON schema 'schema/block.schema.json'
     """
     # 1. validate argument
     validate(instance = block, schema = _block_schema)
     # 2. generate tag
     with xmlgen.element("block_type", {
         "name": block["name"],
         "id": block["id"],
         "width": block.get("width", 1),
         "height": block.get("height", 1), }):
         ptc_it = count()
         capacity = block.get("capacity", 1)
         for z, key in product(range(capacity), ("input", "output", "clock")):
             for port in block.get(key, []):
                 for bit in range(port["num_pins"]):
                     with xmlgen.element("pin_class", {
                         "type": "OUTPUT" if key == "output" else "INPUT", }):
                         if capacity == 1:
                             xmlgen.element_leaf("pin", {
                                 "ptc": next(ptc_it), },
                                 "{}.{}[{}]".format(block["name"], port["name"], bit))
                         else:
                             xmlgen.element_leaf("pin", {
                                 "ptc": next(ptc_it), },
                                 "{}[{}].{}[{}]".format(block["name"], z, port["name"], bit))
    def test_which_pools(self):
        """FortunaAccumulator.which_pools"""

        # which_pools(0) should fail
        self.assertRaises(AssertionError, FortunaAccumulator.which_pools, 0)

        self.assertEqual(FortunaAccumulator.which_pools(1), [0])
        self.assertEqual(FortunaAccumulator.which_pools(2), [0, 1])
        self.assertEqual(FortunaAccumulator.which_pools(3), [0])
        self.assertEqual(FortunaAccumulator.which_pools(4), [0, 1, 2])
        self.assertEqual(FortunaAccumulator.which_pools(5), [0])
        self.assertEqual(FortunaAccumulator.which_pools(6), [0, 1])
        self.assertEqual(FortunaAccumulator.which_pools(7), [0])
        self.assertEqual(FortunaAccumulator.which_pools(8), [0, 1, 2, 3])
        for i in range(1, 32):
            self.assertEqual(FortunaAccumulator.which_pools(2**i-1), [0])
            self.assertEqual(FortunaAccumulator.which_pools(2**i), list(range(i+1)))
            self.assertEqual(FortunaAccumulator.which_pools(2**i+1), [0])
        self.assertEqual(FortunaAccumulator.which_pools(2**31), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**32), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**33), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**34), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**35), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**36), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**64), list(range(32)))
        self.assertEqual(FortunaAccumulator.which_pools(2**128), list(range(32)))
Пример #24
0
def herdif(n, m, b):
    """Computes differentiation matrices D1, D2, ..., Dm on Hermite points.
    
    Args:
        n: np.mber of points, which is also the order of accuracy.
        m: np.mber of derivative matrices to return.
        b: Scaling parameter. Real and positive.
    
    Returns:
        x: Array of nodes, zeros of Hermite polynomial of degree n, scaled by b.
        Dm: A list s.t. Dm[i] is the (i+1)-th derivative matrix, i=0...m-1.
    
    np.te: 0 < m < n-1.
    """
    x = herroots(n)
    # Compute weights
    alpha = np.exp(-x**2 / 2.)
    # Set up beta matrix s.t. beta[i,j] = 
    #  ( (i+1)-th derivative of alpha(x) )/alpha(x), evaluated at x = x(j).
    beta = np.zeros((m+1, x.shape[0]))
    beta[0] = 1.0
    beta[1] = -x
    for i in range(2, m+1):
        beta[i] = -x * beta[i-1] - (i-1) * beta[i-2]
    # Remove initializing row from beta
    beta = np.delete(beta, 0, 0)
    # Compute differentiation matrix (b=1).
    Dm = poldif(x, alpha=alpha, B=beta)
    # Scale nodes by the factor b.
    x = x/b
    # Adjust derivatives for b not equal to 1.
    for i in range(1, m+1):
        Dm[i-1] *= b**i
        
    return x, Dm
Пример #25
0
    def encrypt(self, plaintext):
        AES.encrypt(self, plaintext)

        plaintextBytes = stringToBytes(plaintext)
        chainBytes = stringToBytes(self.IV)

        #CBC Mode: For each block...
        for x in range(len(plaintextBytes)/16):

            #XOR with the chaining block
            blockBytes = plaintextBytes[x*16 : (x*16)+16]
            for y in range(16):
                blockBytes[y] ^= chainBytes[y]
            blockString = bytesToString(blockBytes)

            #Encrypt it
            encryptedBytes = stringToBytes(self.rijndael.encrypt(blockString))

            #Overwrite the input with the output
            for y in range(16):
                plaintextBytes[(x*16)+y] = encryptedBytes[y]

            #Set the next chaining block
            chainBytes = encryptedBytes

        self.IV = bytesToString(chainBytes)
        return bytesToString(plaintextBytes)
Пример #26
0
    def test_codes(self):
        # Codes that mean uneventful copies (including no copy at all).
        for i in [0, 1]:
            yield self._run_simple_test(
                r'D:\source', r'E:\dest', expected_code=i,
                expected_res=SUCCESS
            )

        # Codes that mean some mismatched or extra files were found.
        for i in range(2, 8):
            yield self._run_simple_test(
                r'D:\source', r'E:\dest', expected_code=i,
                expected_res=WARNINGS
            )
        # Codes that mean errors have been encountered.
        for i in range(8, 32):
            yield self._run_simple_test(
                r'D:\source', r'E:\dest', expected_code=i,
                expected_res=FAILURE
            )

        # bit 32 is meaningless
        yield self._run_simple_test(
            r'D:\source', r'E:\dest', expected_code=32,
            expected_res=EXCEPTION
        )
Пример #27
0
 def snapCurvedIntersections(mapPoint, mapCanvas, mapTool, featureId=None):
     """
     To snap on curved intersections
     :param mapPoint: the map position
     :param mapCanvas: the used QgsMapCanvas
     :param mapTool: a QgsMapTool instance
     :param featureId: if we want to snap on a given feature
     :return: intersection point
     """
     snap_layers = Finder.getLayersSettings(mapCanvas, [QGis.Line, QGis.Polygon, QGis.Point])
     features = Finder.findFeaturesLayersAt(mapPoint, snap_layers, mapTool)
     inter = None
     if len(features) > 1:
         if len(features) > 2:
             for i in range(len(features)):
                 for j in range(i, len(features)):
                     feat1 = features[i]
                     feat2 = features[j]
                     if feat1 != feat2:
                         interP = Finder.intersect(featureId, feat1, feat2, mapPoint)
                         if interP is not None:
                             if inter is None or mapPoint.sqrDist(interP) < mapPoint.sqrDist(inter):
                                 inter = interP
         else:
             feat1 = features[0]
             feat2 = features[1]
             inter = Finder.intersect(featureId, feat1, feat2, mapPoint)
     return inter
Пример #28
0
  def testMaxSize(self):
    mocks = []

    def gen_mock():
      c = mock.MagicMock()
      mocks.append(c)
      return c

    proxies = []
    pool = mysql_pool.Pool(gen_mock, max_size=5)
    for _ in builtins.range(5):
      c = pool.get(blocking=False)
      self.assertIsNotNone(c)
      proxies.append(c)

    self.assertIsNone(pool.get(blocking=False))
    for p in proxies:
      p.close()

    for m in mocks:
      # Should be returned to the pool.
      m.close.assert_not_called()

    for _ in builtins.range(5):
      c = pool.get(blocking=False)
      self.assertIsNotNone(c)
      proxies.append(c)
    for p in proxies:
      p.close()
    self.assertLen(mocks, 5, 'Should have created only 5 mocks.')
Пример #29
0
    def test_array_horizontal_shear(self):
        # tests function array_rotation_strain with synthetic data with pure
        # horizontal shear strain, no rotation or dilation
        array_coords = self.array_coords
        subarray = self.subarray
        ts1 = self.ts1
        ts2 = self.ts2
        sigmau = self.sigmau
        Vp = self.Vp
        Vs = self.Vs

        shear_strainh = (
            0.00001
            * np.exp(-1 * np.square(np.linspace(-2, 2, 1000)))
            * np.sin(np.linspace(-10 * np.pi, 10 * np.pi, 1000))
        )

        ts3 = np.zeros((1000, 7))

        for stat in range(7):
            for t in range(1000):
                ts1[t, stat] = array_coords[stat, 1] * shear_strainh[t]
                ts2[t, stat] = array_coords[stat, 0] * shear_strainh[t]

        out = array_rotation_strain(subarray, ts1, ts2, ts3, Vp, Vs, array_coords, sigmau)

        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_d"], decimal=12)
        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_dh"], decimal=12)
        np.testing.assert_array_almost_equal(abs(shear_strainh), out["ts_s"], decimal=12)
        np.testing.assert_array_almost_equal(abs(shear_strainh), out["ts_sh"], decimal=12)
        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w1"], decimal=12)
        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w2"], decimal=12)
        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_w3"], decimal=12)
        np.testing.assert_array_almost_equal(np.zeros(1000), out["ts_M"], decimal=12)
Пример #30
0
  def testYamlPluginWithValuesOfSameType(self):
    responses = []
    for i in range(10):
      responses.append(
          rdf_client_fs.StatEntry(
              pathspec=rdf_paths.PathSpec(
                  path="/foo/bar/%d" % i, pathtype="OS"),
              st_mode=33184,  # octal = 100640 => u=rw,g=r,o= => -rw-r-----
              st_ino=1063090,
              st_dev=64512,
              st_nlink=1 + i,
              st_uid=139592,
              st_gid=5000,
              st_size=0,
              st_atime=1336469177,
              st_mtime=1336129892,
              st_ctime=1336129892))

    zip_fd, prefix = self.ProcessValuesToZip(
        {rdf_client_fs.StatEntry: responses})
    self.assertEqual(
        set(zip_fd.namelist()), {
            "%s/MANIFEST" % prefix,
            "%s/ExportedFile/from_StatEntry.yaml" % prefix
        })

    parsed_manifest = yaml.load(zip_fd.read("%s/MANIFEST" % prefix))
    self.assertEqual(parsed_manifest,
                     {"export_stats": {
                         "StatEntry": {
                             "ExportedFile": 10
                         }
                     }})

    parsed_output = yaml.load(
        zip_fd.read("%s/ExportedFile/from_StatEntry.yaml" % prefix))
    self.assertLen(parsed_output, 10)
    for i in range(10):
      # Only the client_urn is filled in by the plugin. Doing lookups for
      # all the clients metadata is possible but expensive. It doesn't seem to
      # be worth it.
      self.assertEqual(parsed_output[i]["metadata"]["client_urn"],
                       str(self.client_id))
      self.assertEqual(parsed_output[i]["metadata"]["source_urn"],
                       str(self.results_urn))
      self.assertEqual(parsed_output[i]["urn"],
                       self.client_id.Add("/fs/os/foo/bar").Add(str(i)))
      self.assertEqual(parsed_output[i]["st_mode"], "-rw-r-----")
      self.assertEqual(parsed_output[i]["st_ino"], "1063090")
      self.assertEqual(parsed_output[i]["st_dev"], "64512")
      self.assertEqual(parsed_output[i]["st_nlink"], str(1 + i))
      self.assertEqual(parsed_output[i]["st_uid"], "139592")
      self.assertEqual(parsed_output[i]["st_gid"], "5000")
      self.assertEqual(parsed_output[i]["st_size"], "0")
      self.assertEqual(parsed_output[i]["st_atime"], "2012-05-08 09:26:17")
      self.assertEqual(parsed_output[i]["st_mtime"], "2012-05-04 11:11:32")
      self.assertEqual(parsed_output[i]["st_ctime"], "2012-05-04 11:11:32")
      self.assertEqual(parsed_output[i]["st_blksize"], "0")
      self.assertEqual(parsed_output[i]["st_rdev"], "0")
      self.assertEqual(parsed_output[i]["symlink"], "")
Пример #31
0
 def test_slice_range(self):
     r = range(8)
     self.assertEqual(r[:], range(8))
     self.assertEqual(r[:2], range(2))
     self.assertEqual(r[:-2], range(6))
     self.assertEqual(r[2:], range(2, 8))
     self.assertEqual(r[-2:], range(6, 8))
     self.assertEqual(r[2:-2], range(2, 6))
     self.assertEqual(r[-2:2:-1], range(6, 2, -1))
     r = r[::-1]
     self.assertEqual(r, range(7, -1, -1))
     self.assertEqual(r[:], range(7, -1, -1))
     self.assertEqual(r[:2], range(7, 5, -1))
     self.assertEqual(r[:-2], range(7, 1, -1))
     self.assertEqual(r[2:], range(5, -1, -1))
     self.assertEqual(r[-2:], range(1, -1, -1))
     self.assertEqual(r[2:-2], range(5, 1, -1))
     self.assertEqual(r[-2:2:-1], range(1, 5))
Пример #32
0
 def test_slice_empty_range(self):
     self.assertEqual(range(0)[:], range(0))
     self.assertEqual(range(0)[::-1], range(-1, -1, -1))
Пример #33
0
 def test_equality_range(self):
     self.assertEqual(range(7), range(7))
     self.assertEqual(range(0), range(1, 1))
     self.assertEqual(range(0, 10, 3), range(0, 11, 3))
Пример #34
0
 def test_bool_range(self):
     self.assertFalse(range(0))
     self.assertTrue(range(1))
     self.assertFalse(range(1, 1))
     self.assertFalse(range(5, 2))
Пример #35
0
 def test_range(self):
     self.assertTrue(isinstance(range(0), Sequence))
     self.assertTrue(isinstance(reversed(range(0)), Iterator))
Пример #36
0
    def testBlockingTasks(self):
        # The pool starts off with the minimum number of threads.
        self.assertLen(self.test_pool, self.NUMBER_OF_THREADS)

        done_event = threading.Event()
        self.lock = threading.Lock()
        res = []

        def Block(done):
            done.wait()

        def Insert(list_obj, element):
            with self.lock:
                list_obj.append(element)

        # Schedule the maximum number of threads of blocking tasks and the same of
        # insert tasks. The threads are now all blocked, and the inserts are
        # waiting in the queue.
        for _ in range(self.MAXIMUM_THREADS):
            self.test_pool.AddTask(Block, (done_event, ), "Blocking")

        # Wait until the threadpool picks up tasks.
        self.WaitUntil(
            lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS)

        # Now there's maximum number of threads active and the queue is empty.
        self.assertEqual(self.test_pool.pending_tasks, 0)

        # Now we push these tasks on the queue, but they're not going to be
        # processed, since all threads are busy.
        for i in range(self.MAXIMUM_THREADS):
            self.test_pool.AddTask(Insert, (res, i),
                                   "Insert",
                                   blocking=True,
                                   inline=False)

        # There should be 20 workers created and they should consume all the
        # blocking tasks.
        self.WaitUntil(
            lambda: self.test_pool.busy_threads == self.MAXIMUM_THREADS)

        # No Insert tasks are running yet.
        self.assertEqual(res, [])

        # There are 20 tasks waiting on the queue.
        self.assertEqual(self.test_pool.pending_tasks, self.MAXIMUM_THREADS)

        # Inserting more tasks than the queue can hold should lead to processing
        # the tasks inline. This effectively causes these tasks to skip over the
        # tasks which are waiting in the queue.
        for i in range(10, 20):
            self.test_pool.AddTask(Insert, (res, i), "Insert", inline=True)

        res.sort()
        self.assertEqual(res, list(range(10, 20)))

        # This should release all the busy tasks. It will also cause the workers
        # to process all the Insert tasks in the queue.
        done_event.set()

        self.test_pool.Join()

        # Now the rest of the tasks should have been processed as well.
        self.assertEqual(sorted(res[10:]), list(range(20)))
Пример #37
0
def d_logS_d_expX(S, X, j, p, grad, thresh, eps=1e-5):
    """
    Compute the gradient of log S[j] w.r.t. exp(X).
    For unstable cases, use p-th order approximnation.
    """

    # ------------------------------------------------------------------------
    # Detect unstabilites
    # ------------------------------------------------------------------------

    _X_ = LogTensor(X)
    _S_ = [LogTensor(S[i]) for i in range(S.size(0))]

    # recursion of gradient formula (separate terms for stability)
    _N_, _P_ = recursion(_S_, _X_, j)

    # detect instability: small relative difference in log-space

    P = _P_.torch()
    if isinstance(_N_, int):
        N = torch.zeros(P.size()).cuda()
    else:
        N = _N_.torch()

    diff = (P - N) / (N.abs() + eps)

    # split into stable and unstable indices
    u_indices = torch.lt(diff, thresh)  # unstable
    s_indices = u_indices.eq(0)  # stable

    # ------------------------------------------------------------------------
    # Compute d S[j] / d X
    # ------------------------------------------------------------------------

    # make grad match size and type of X
    grad = grad.type_as(X).resize_as_(X)

    # exact gradient for s_indices (stable) elements
    if s_indices.sum():
        # re-use positive and negative parts of recursion (separate for stability)
        _N_ = LogTensor(N[s_indices])
        _P_ = LogTensor(P[s_indices])
        _X_ = LogTensor(X[s_indices])
        _S_ = [LogTensor(S[i][s_indices]) for i in range(S.size(0))]

        # d log S[j] / d exp(X) = (d S[j] / d X) * X / S[j]
        _SG_ = (_P_ - _N_) * _X_ / _S_[j]
        grad.masked_scatter_(s_indices, _SG_.torch().exp())

    # approximate gradients for u_indices (unstable) elements
    if u_indices.sum():
        _X_ = LogTensor(X[u_indices])
        _S_ = [LogTensor(S[i][u_indices]) for i in range(S.size(0))]

        # positive and negative parts of approximation (separate for stability)
        _N_, _P_ = approximation(_S_, _X_, j, p)

        # d log S[j] / d exp(X) = (d S[j] / d X) * X / S[j]
        _UG_ = (_P_ - _N_) * _X_ / _S_[j]
        grad.masked_scatter_(u_indices, _UG_.torch().exp())

    return grad
Пример #38
0
    def test_multi_player(self):
        players = 2
        run_config = run_configs.get()
        parallel = run_parallel.RunParallel()
        map_inst = maps.get("Simple64")

        screen_size_px = point.Point(64, 64)
        minimap_size_px = point.Point(32, 32)
        interface = sc_pb.InterfaceOptions()
        screen_size_px.assign_to(interface.feature_layer.resolution)
        minimap_size_px.assign_to(interface.feature_layer.minimap_resolution)

        # Reserve a whole bunch of ports for the weird multiplayer implementation.
        ports = [portpicker.pick_unused_port() for _ in range(1 + players * 2)]
        logging.info("Valid Ports: %s", ports)

        # Actually launch the game processes.
        print_stage("start")
        sc2_procs = [
            run_config.start(extra_ports=ports) for _ in range(players)
        ]
        controllers = [p.controller for p in sc2_procs]

        try:
            # Save the maps so they can access it.
            map_path = os.path.basename(map_inst.path)
            print_stage("save_map")
            parallel.run((c.save_map, map_path, map_inst.data(run_config))
                         for c in controllers)

            # Create the create request.
            create = sc_pb.RequestCreateGame(local_map=sc_pb.LocalMap(
                map_path=map_path))
            for _ in range(players):
                create.player_setup.add(type=sc_pb.Participant)

            # Create the join request.
            join = sc_pb.RequestJoinGame(race=sc_common.Random,
                                         options=interface)
            join.shared_port = ports.pop()
            join.server_ports.game_port = ports.pop()
            join.server_ports.base_port = ports.pop()
            for _ in range(players - 1):
                join.client_ports.add(game_port=ports.pop(),
                                      base_port=ports.pop())

            # Play a few short games.
            for _ in range(2):  # 2 episodes
                # Create and Join
                print_stage("create")
                controllers[0].create_game(create)
                print_stage("join")
                parallel.run((c.join_game, join) for c in controllers)

                print_stage("run")
                for game_loop in range(1, 10):  # steps per episode
                    # Step the game
                    parallel.run(c.step for c in controllers)

                    # Observe
                    obs = parallel.run(c.observe for c in controllers)
                    for p_id, o in enumerate(obs):
                        self.assertEqual(o.observation.game_loop, game_loop)
                        self.assertEqual(o.observation.player_common.player_id,
                                         p_id + 1)

                    # Act
                    actions = [sc_pb.Action() for _ in range(players)]
                    for action in actions:
                        pt = (point.Point.unit_rand() *
                              minimap_size_px).floor()
                        pt.assign_to(action.action_feature_layer.camera_move.
                                     center_minimap)
                    parallel.run(
                        (c.act, a) for c, a in zip(controllers, actions))

                # Done this game.
                print_stage("leave")
                parallel.run(c.leave for c in controllers)
        finally:
            print_stage("quit")
            # Done, shut down. Don't depend on parallel since it might be broken.
            for c in controllers:
                c.quit()
            for p in sc2_procs:
                p.close()
Пример #39
0
        *counter block*, which is a byte string of `block_size` bytes.
        For better performance, use `Crypto.Util.Counter`.
      segment_size : integer
        (*Only* `MODE_CFB`).The number of bits the plaintext and ciphertext
        are segmented in.
        It must be a multiple of 8. If 0 or not specified, it will be assumed to be 8.

    :Return: an `CAST128Cipher` object
    """
    return CAST128Cipher(key, *args, **kwargs)


#: Electronic Code Book (ECB). See `blockalgo.MODE_ECB`.
MODE_ECB = 1
#: Cipher-Block Chaining (CBC). See `blockalgo.MODE_CBC`.
MODE_CBC = 2
#: Cipher FeedBack (CFB). See `blockalgo.MODE_CFB`.
MODE_CFB = 3
#: This mode should not be used.
MODE_PGP = 4
#: Output FeedBack (OFB). See `blockalgo.MODE_OFB`.
MODE_OFB = 5
#: CounTer Mode (CTR). See `blockalgo.MODE_CTR`.
MODE_CTR = 6
#: OpenPGP Mode. See `blockalgo.MODE_OPENPGP`.
MODE_OPENPGP = 7
#: Size of a data block (in bytes)
block_size = 8
#: Size of a key (in bytes)
key_size = range(5, 16 + 1)
Пример #40
0
    def compute_symmetric_inner_product_mat(self, vec_handles):
        """Computes symmetric matrix whose elements are inner products of the
        vector objects in ``vec_handles`` with each other.

        Args:
            ``vec_handles``: List of handles for vector objects corresponding
            to both rows and columns.  For example, in POD this is the snapshot
            matrix :math:`X`.

        Returns:
            ``IP_mat``: 2D array of inner products.

        See the documentation for :py:meth:`compute_inner_product_mat` for an
        idea how this works.  Efficiency is achieved by only computing the
        upper-triangular elements, since the matrix is symmetric.  Within the
        upper-triangular portion, there are rectangular chunks and triangular
        chunks.  The rectangular chunks are divided up among MPI workers
        (processors) as weighted tasks.  Once those have been computed, the
        triangular chunks are dealt with.
        """
        # TODO: JON, write detailed documentation similar to
        # :py:meth:`compute_inner_product_mat`.
        self._check_inner_product()
        vec_handles = util.make_iterable(vec_handles)

        num_vecs = len(vec_handles)

        # num_cols_per_chunk is the number of cols each proc gets at once.
        # Columns are retrieved if the matrix must be broken up into sets of
        # chunks.  Then symmetric upper triangular portions will be computed,
        # followed by a rectangular piece that uses columns not already in
        # memory.
        num_cols_per_proc_chunk = 1
        num_rows_per_proc_chunk = self.max_vecs_per_proc -\
            num_cols_per_proc_chunk

        # <nprocs> chunks are computed simulaneously, making up a set.
        num_cols_per_chunk = num_cols_per_proc_chunk * parallel.get_num_procs()
        num_rows_per_chunk = num_rows_per_proc_chunk * parallel.get_num_procs()

        # <num_row_chunks> is the number of sets that must be computed.
        num_row_chunks = int(np.ceil(num_vecs * 1. / num_rows_per_chunk))
        if num_row_chunks > 1:
            self.print_msg(
                'Warning: The vecs, of which '
                'there are %d, will be retrieved %d times each. Increase '
                'number of nodes or max_vecs_per_node to reduce redundant '
                '"get"s for a speedup.' % (num_vecs, num_row_chunks))

        # Estimate the time this will take and determine matrix datatype
        # (real or complex).
        test_vec = vec_handles[0].get()
        # Burn the first, it sometimes contains slow imports
        IP_burn = self.inner_product(test_vec, test_vec)

        start_time = time()
        test_vec = vec_handles[0].get()
        get_time = time() - start_time

        start_time = time()
        IP = self.inner_product(test_vec, test_vec)
        IP_time = time() - start_time
        IP_type = type(IP)

        total_IP_time = (num_vecs**2 * IP_time / 2. / parallel.get_num_procs())
        vecs_per_proc = self.max_vecs_per_node * parallel.get_num_nodes() / \
            parallel.get_num_procs()
        num_gets =  (num_vecs**2 /2.) / ((vecs_per_proc-2) *
            parallel.get_num_procs()**2) + \
            num_vecs/parallel.get_num_procs()/2.
        total_get_time = num_gets * get_time
        self.print_msg('Computing the inner product matrix will take at least '
                       '%.1f minutes' %
                       ((total_IP_time + total_get_time) / 60.))
        del test_vec

        # Use the same trick as in compute_IP_mat, having each proc
        # fill in elements of a num_rows x num_rows sized matrix, rather than
        # assembling small chunks. This is done for the triangular portions.
        # For the rectangular portions, the inner product mat is filled
        # in directly.
        IP_mat = np.mat(np.zeros((num_vecs, num_vecs), dtype=IP_type))
        for start_row_index in range(0, num_vecs, num_rows_per_chunk):
            end_row_index = min(num_vecs, start_row_index + num_rows_per_chunk)
            proc_row_tasks_all = parallel.find_assignments(
                list(range(start_row_index, end_row_index)))
            num_active_procs = len([task for task in \
                proc_row_tasks_all if task != []])
            proc_row_tasks = proc_row_tasks_all[parallel.get_rank()]
            if len(proc_row_tasks) != 0:
                row_vecs = [
                    vec_handle.get() for vec_handle in
                    vec_handles[proc_row_tasks[0]:proc_row_tasks[-1] + 1]
                ]
            else:
                row_vecs = []

            # Triangular chunks
            if len(proc_row_tasks) > 0:
                # Test that indices are consecutive
                if proc_row_tasks[0:] != list(
                        range(proc_row_tasks[0], proc_row_tasks[-1] + 1)):
                    raise ValueError('Indices are not consecutive.')

                # Per-processor triangles (using only vecs in memory)
                for row_index in range(proc_row_tasks[0],
                                       proc_row_tasks[-1] + 1):
                    # Diagonal term
                    IP_mat[row_index, row_index] = self.\
                        inner_product(row_vecs[row_index - proc_row_tasks[
                        0]], row_vecs[row_index - proc_row_tasks[0]])

                    # Off-diagonal terms
                    for col_index in range(row_index + 1,
                                           proc_row_tasks[-1] + 1):
                        IP_mat[row_index, col_index] = self.\
                            inner_product(row_vecs[row_index -\
                            proc_row_tasks[0]], row_vecs[col_index -\
                            proc_row_tasks[0]])

            # Number of square chunks to fill in is n * (n-1) / 2.  At each
            # iteration we fill in n of them, so we need (n-1) / 2
            # iterations (round up).
            for set_index in range(int(np.ceil((num_active_procs - 1.) / 2))):
                # The current proc is "sender"
                my_rank = parallel.get_rank()
                my_row_indices = proc_row_tasks
                my_num_rows = len(my_row_indices)

                # The proc to send to is "destination"
                dest_rank = (my_rank + set_index + 1) % num_active_procs
                # This is unused?
                #dest_row_indices = proc_row_tasks_all[dest_rank]

                # The proc that data is received from is the "source"
                source_rank = (my_rank - set_index - 1) % num_active_procs

                # Find the maximum number of sends/recv to be done by any proc
                max_num_to_send = int(np.ceil(1. * max([len(tasks) for \
                    tasks in proc_row_tasks_all]) /\
                    num_cols_per_proc_chunk))
                """
                # Pad tasks with nan so that everyone has the same
                # number of things to send.  Same for list of vecs with None.
                # The empty lists will not do anything when enumerated, so no
                # inner products will be taken.  nan is inserted into the
                # indices because then min/max of the indices can be taken.

                if my_num_rows != len(row_vecs):
                    raise ValueError('Number of rows assigned does not ' +\
                        'match number of vecs in memory.')
                if my_num_rows > 0 and my_num_rows < max_num_to_send:
                    my_row_indices += [np.nan] * (max_num_to_send - my_num_rows)
                    row_vecs += [[]] * (max_num_to_send - my_num_rows)
                """
                for send_index in range(max_num_to_send):
                    # Only processors responsible for rows communicate
                    if my_num_rows > 0:
                        # Send row vecs, in groups of num_cols_per_proc_chunk
                        # These become columns in the ensuing computation
                        start_col_index = send_index * num_cols_per_proc_chunk
                        end_col_index = min(
                            start_col_index + num_cols_per_proc_chunk,
                            my_num_rows)
                        col_vecs_send = (
                            row_vecs[start_col_index:end_col_index],
                            my_row_indices[start_col_index:end_col_index])

                        # Create unique tags based on ranks
                        send_tag = my_rank * (parallel.get_num_procs() +
                                              1) + dest_rank
                        recv_tag = source_rank * (parallel.get_num_procs() +
                                                  1) + my_rank

                        # Send and receieve data.  The Wait() command after the
                        # receive prevents a race condition not fixed by sync().
                        # The Wait() is very important for the non-
                        # blocking send (though we are unsure why).
                        request = parallel.comm.isend(col_vecs_send,
                                                      dest=dest_rank,
                                                      tag=send_tag)
                        col_vecs_recv = parallel.comm.recv(source=source_rank,
                                                           tag=recv_tag)
                        request.Wait()
                        col_vecs = col_vecs_recv[0]
                        my_col_indices = col_vecs_recv[1]

                        for row_index in range(my_row_indices[0],
                                               my_row_indices[-1] + 1):
                            for col_vec_index, col_vec in enumerate(col_vecs):
                                IP_mat[row_index, my_col_indices[
                                    col_vec_index]] = self.inner_product(
                                        row_vecs[row_index -
                                                 my_row_indices[0]], col_vec)
                            if (time() - self.prev_print_time) > \
                                self.print_interval:
                                num_completed_IPs = (np.abs(IP_mat) > 0).sum()
                                percent_completed_IPs = \
                                    (100.*2*num_completed_IPs * \
                                    parallel.get_num_MPI_workers())/\
                                    (num_vecs**2)
                                self.print_msg(
                                    ('Completed %.1f%% of inner products') %
                                    percent_completed_IPs, sys.stderr)
                                self.prev_print_time = time()

                    # Sync after send/receive
                    parallel.barrier()

            # Fill in the rectangular portion next to each triangle (if nec.).
            # Start at index after last row, continue to last column. This part
            # of the code is the same as in compute_IP_mat, as of
            # revision 141.
            for start_col_index in range(end_row_index, num_vecs,
                                         num_cols_per_chunk):
                end_col_index = min(start_col_index + num_cols_per_chunk,
                                    num_vecs)
                proc_col_tasks = parallel.find_assignments(
                    list(range(start_col_index,
                               end_col_index)))[parallel.get_rank()]

                # Pass the col vecs to proc with rank -> mod(rank+1,numProcs)
                # Must do this for each processor, until data makes a circle
                col_vecs_recv = (None, None)
                if len(proc_col_tasks) > 0:
                    col_indices = list(
                        range(proc_col_tasks[0], proc_col_tasks[-1] + 1))
                else:
                    col_indices = []

                for num_passes in range(parallel.get_num_procs()):
                    # If on the first pass, get the col vecs, no send/recv
                    # This is all that is called when in serial, loop iterates
                    # once.
                    if num_passes == 0:
                        if len(col_indices) > 0:
                            col_vecs = [col_handle.get() \
                                for col_handle in vec_handles[col_indices[0]:\
                                    col_indices[-1] + 1]]
                        else:
                            col_vecs = []
                    else:
                        # Determine whom to communicate with
                        dest = (parallel.get_rank() + 1) % parallel.\
                            get_num_procs()
                        source = (parallel.get_rank() - 1) % parallel.\
                            get_num_procs()

                        # Create unique tag based on ranks
                        send_tag = parallel.get_rank() * (parallel.\
                            get_num_procs() + 1) + dest
                        recv_tag = source*(parallel.get_num_procs() + 1) +\
                            parallel.get_rank()

                        # Collect data and send/receive
                        col_vecs_send = (col_vecs, col_indices)
                        request = parallel.comm.isend(col_vecs_send, dest=\
                            dest, tag=send_tag)
                        col_vecs_recv = parallel.comm.recv(source=source,
                                                           tag=recv_tag)
                        request.Wait()
                        parallel.barrier()
                        col_indices = col_vecs_recv[1]
                        col_vecs = col_vecs_recv[0]

                    # Compute the IPs for this set of data col_indices stores
                    # the indices of the IP_mat columns to be
                    # filled in.
                    if len(proc_row_tasks) > 0:
                        for row_index in range(proc_row_tasks[0],
                                               proc_row_tasks[-1] + 1):
                            for col_vec_index, col_vec in enumerate(col_vecs):
                                IP_mat[row_index, col_indices[
                                    col_vec_index]] = self.inner_product(
                                        row_vecs[row_index -
                                                 proc_row_tasks[0]], col_vec)
                        if ((time() - self.prev_print_time) >
                                self.print_interval):
                            num_completed_IPs = (np.abs(IP_mat) > 0).sum()
                            percent_completed_IPs = (
                                100. * 2 * num_completed_IPs *
                                parallel.get_num_MPI_workers()) / (num_vecs**2)
                            self.print_msg(
                                ('Completed %.1f%% of inner ' + 'products') %
                                percent_completed_IPs, sys.stderr)
                            self.prev_print_time = time()

            # Completed a chunk of rows and all columns on all processors.
            # Finished row_vecs loop, delete memory used
            del row_vecs

        # Assign the triangular portion chunks into IP_mat.
        if parallel.is_distributed():
            IP_mat = parallel.custom_comm.allreduce(IP_mat)

        # Create a mask for the repeated values.  Select values that are zero
        # in the upper triangular portion (not computed there) but nonzero in
        # the lower triangular portion (computed there).  For the case where
        # the inner product is not perfectly symmetric, this will select the
        # computation done in the upper triangular portion.
        mask = np.multiply(IP_mat == 0, IP_mat.T != 0)

        # Collect values below diagonal
        IP_mat += np.multiply(np.triu(IP_mat.T, 1), mask)

        # Symmetrize matrix
        IP_mat = np.triu(IP_mat) + np.triu(IP_mat, 1).T

        percent_completed_IPs = 100.
        self.print_msg(('Completed %.1f%% of inner ' + 'products') %
                       percent_completed_IPs, sys.stderr)
        self.prev_print_time = time()

        parallel.barrier()
        return IP_mat
Пример #41
0
    def compute_inner_product_mat(self, row_vec_handles, col_vec_handles):
        """Computes matrix whose elements are inner products of the vector
        objects in ``row_vec_handles`` and ``col_vec_handles``.

        Args:
            ``row_vec_handles``: List of handles for vector objects
            corresponding to rows of the inner product matrix.  For example, in
            BPOD this is the adjoint snapshot matrix :math:`Y`.

            ``col_vec_handles``: List of handles for vector objects
            corresponding to columns of the inner product matrix.  For example,
            in BPOD this is the direct snapshot matrix :math:`X`.

        Returns:
            ``IP_mat``: 2D array of inner products.

        The vectors are retrieved in memory-efficient chunks and are not all in
        memory at once.  The row vectors and column vectors are assumed to be
        different.  When they are the same, use
        :py:meth:`compute_symmetric_inner_product` for a 2x speedup.

        Each MPI worker (processor) is responsible for retrieving a subset of
        the rows and columns. The processors then send/receive columns via MPI
        so they can be used to compute all inner products for the rows on each
        MPI worker.  This is repeated until all MPI workers are done with all
        of their row chunks.  If there are 2 processors::

                | x o |
          rank0 | x o |
                | x o |
            -
                | o x |
          rank1 | o x |
                | o x |

        In the next step, rank 0 sends column 0 to rank 1 and rank 1 sends
        column 1 to rank 0. The remaining inner products are filled in::

                | x x |
          rank0 | x x |
                | x x |
            -
                | x x |
          rank1 | x x |
                | x x |

        When the number of columns and rows is not divisible by the number of
        processors, the processors are assigned unequal numbers of tasks.
        However, all processors are always part of the passing cycle.

        The scaling is:

        - num gets / processor ~ :math:`(n_r*n_c/((max-2)*n_p*n_p)) + n_r/n_p`
        - num MPI sends / processor ~
          :math:`(n_p-1)*(n_r/((max-2)*n_p))*n_c/n_p`
        - num inner products / processor ~ :math:`n_r*n_c/n_p`

        where :math:`n_r` is number of rows, :math:`n_c` number of columns,
        :math:`max` is
        ``max_vecs_per_proc = max_vecs_per_node/num_procs_per_node``,
        and :math:`n_p` is the number of MPI workers (processors).

        If there are more rows than columns, then an internal transpose and
        un-transpose is performed to improve efficiency (since :math:`n_c` only
        appears in the scaling in the quadratic term).
        """
        self._check_inner_product()
        row_vec_handles = util.make_iterable(row_vec_handles)
        col_vec_handles = util.make_iterable(col_vec_handles)

        num_cols = len(col_vec_handles)
        num_rows = len(row_vec_handles)

        if num_rows > num_cols:
            transpose = True
            temp = row_vec_handles
            row_vec_handles = col_vec_handles
            col_vec_handles = temp
            temp = num_rows
            num_rows = num_cols
            num_cols = temp
        else:
            transpose = False

        # convenience
        rank = parallel.get_rank()

        ## Old way that worked
        # num_cols_per_proc_chunk is the number of cols each proc gets at once
        num_cols_per_proc_chunk = 1
        num_rows_per_proc_chunk = self.max_vecs_per_proc - \
            num_cols_per_proc_chunk

        # Determine how the retrieving and inner products will be split up.
        row_tasks = parallel.find_assignments(list(range(num_rows)))
        col_tasks = parallel.find_assignments(list(range(num_cols)))

        # Find max number of col tasks among all processors
        max_num_row_tasks = max([len(tasks) for tasks in row_tasks])
        max_num_col_tasks = max([len(tasks) for tasks in col_tasks])

        ## New way
        #if self.max_vecs_per_node > max_num_row_tasks:
        #    num_cols_per_proc_chunk =
        #num_rows_per_proc_chunk = self.max_vecs_per_proc - \
        #    num_cols_per_proc_chunk

        # These variables are the number of iters through loops that retrieve
        # ("get") row and column vecs.
        num_row_get_loops = \
            int(np.ceil(max_num_row_tasks*1./num_rows_per_proc_chunk))
        num_col_get_loops = \
            int(np.ceil(max_num_col_tasks*1./num_cols_per_proc_chunk))
        if num_row_get_loops > 1:
            self.print_msg(
                'Warning: The column vecs, of which '
                'there are %d, will be retrieved %d times each. Increase '
                'number of nodes or max_vecs_per_node to reduce redundant '
                '"get"s for a speedup.' % (num_cols, num_row_get_loops))

        # Estimate the time this will take and determine matrix datatype
        # (real or complex).
        row_vec = row_vec_handles[0].get()
        col_vec = col_vec_handles[0].get()
        # Burn the first, it sometimes contains slow imports
        IP_burn = self.inner_product(row_vec, col_vec)

        start_time = time()
        row_vec = row_vec_handles[0].get()
        get_time = time() - start_time

        start_time = time()
        IP = self.inner_product(row_vec, col_vec)
        IP_time = time() - start_time
        IP_type = type(IP)

        total_IP_time = (num_rows * num_cols * IP_time /
                         parallel.get_num_procs())
        vecs_per_proc = self.max_vecs_per_node * parallel.get_num_nodes() / \
            parallel.get_num_procs()
        num_gets = (num_rows * num_cols) / (
            (vecs_per_proc - 2) *
            parallel.get_num_procs()**2) + num_rows / parallel.get_num_procs()
        total_get_time = num_gets * get_time
        self.print_msg('Computing the inner product matrix will take at least '
                       '%.1f minutes' %
                       ((total_IP_time + total_get_time) / 60.))
        del row_vec, col_vec

        # To find all of the inner product mat chunks, each
        # processor has a full IP_mat with size
        # num_rows x num_cols even though each processor is not responsible for
        # filling in all of these entries. After each proc fills in what it is
        # responsible for, the other entries remain 0's. Then, an allreduce
        # is done and all the IP_mats are summed. This is simpler
        # concatenating chunks of the IPmats.
        # The efficiency is not an issue, the size of the mats
        # are small compared to the size of the vecs for large data.
        IP_mat = np.mat(np.zeros((num_rows, num_cols), dtype=IP_type))
        for row_get_index in range(num_row_get_loops):
            if len(row_tasks[rank]) > 0:
                start_row_index = min(
                    row_tasks[rank][0] +
                    row_get_index * num_rows_per_proc_chunk,
                    row_tasks[rank][-1] + 1)
                end_row_index = min(row_tasks[rank][-1] + 1,
                                    start_row_index + num_rows_per_proc_chunk)
                row_vecs = [
                    row_vec_handle.get() for row_vec_handle in
                    row_vec_handles[start_row_index:end_row_index]
                ]
            else:
                row_vecs = []

            for col_get_index in range(num_col_get_loops):
                if len(col_tasks[rank]) > 0:
                    start_col_index = min(
                        col_tasks[rank][0] +
                        col_get_index * num_cols_per_proc_chunk,
                        col_tasks[rank][-1] + 1)
                    end_col_index = min(
                        col_tasks[rank][-1] + 1,
                        start_col_index + num_cols_per_proc_chunk)
                else:
                    start_col_index = 0
                    end_col_index = 0
                # Cycle the col vecs to proc with rank -> mod(rank+1,num_procs)
                # Must do this for each processor, until data makes a circle
                col_vecs_recv = (None, None)
                col_indices = list(range(start_col_index, end_col_index))
                for pass_index in range(parallel.get_num_procs()):
                    #if rank==0: print 'starting pass index=',pass_index
                    # If on the first pass, get the col vecs, no send/recv
                    # This is all that is called when in serial, loop iterates
                    # once.
                    if pass_index == 0:
                        col_vecs = [
                            col_handle.get() for col_handle in
                            col_vec_handles[start_col_index:end_col_index]
                        ]
                    else:
                        # Determine with whom to communicate
                        dest = (rank + 1) % parallel.get_num_procs()
                        source = (rank - 1) % parallel.get_num_procs()

                        # Create unique tag based on send/recv ranks
                        send_tag = rank * \
                                (parallel.get_num_procs() + 1) + dest
                        recv_tag = source * \
                            (parallel.get_num_procs() + 1) + rank

                        # Collect data and send/receive
                        col_vecs_send = (col_vecs, col_indices)
                        request = parallel.comm.isend(col_vecs_send,
                                                      dest=dest,
                                                      tag=send_tag)
                        col_vecs_recv = parallel.comm.recv(source=source,
                                                           tag=recv_tag)
                        request.Wait()
                        parallel.barrier()
                        col_indices = col_vecs_recv[1]
                        col_vecs = col_vecs_recv[0]

                    # Compute the IPs for this set of data col_indices stores
                    # the indices of the IP_mat columns to be
                    # filled in.
                    if len(row_vecs) > 0:
                        for row_index in range(start_row_index, end_row_index):
                            for col_vec_index, col_vec in enumerate(col_vecs):
                                IP_mat[row_index, col_indices[
                                    col_vec_index]] = self.inner_product(
                                        row_vecs[row_index - start_row_index],
                                        col_vec)
                        if (time() - self.prev_print_time) > \
                            self.print_interval:
                            num_completed_IPs = (np.abs(IP_mat) > 0).sum()
                            percent_completed_IPs = (
                                100. * num_completed_IPs *
                                parallel.get_num_MPI_workers()) / (num_cols *
                                                                   num_rows)
                            self.print_msg(
                                ('Completed %.1f%% of inner ' + 'products') %
                                percent_completed_IPs, sys.stderr)
                            self.prev_print_time = time()

                # Clear the retrieved column vecs after done this pass cycle
                del col_vecs
            # Completed a chunk of rows and all columns on all processors.
            del row_vecs

        # Assign these chunks into IP_mat.
        if parallel.is_distributed():
            IP_mat = parallel.custom_comm.allreduce(IP_mat)

        if transpose:
            IP_mat = IP_mat.T

        percent_completed_IPs = 100.
        self.print_msg(('Completed %.1f%% of inner ' + 'products') %
                       percent_completed_IPs, sys.stderr)
        self.prev_print_time = time()

        parallel.barrier()
        return IP_mat
Пример #42
0
    def __getitem__(self, indices):
        """Get by indexing lookup."""
        indices = self._indices(indices)
        obj = super(NamedNumpyArray, self).__getitem__(indices)

        if (isinstance(indices, np.ndarray) and len(indices.shape) > 1
                and indices.dtype == bool):
            # Is this a multi-dimensional mask, eg: obj[obj == 5] ?
            # Multi-dimensional masks return a single dimensional array, and it's
            # unclear what it means for the result to have names, so return a normal
            # numpy array.
            return np.array(obj)

        if isinstance(obj,
                      np.ndarray):  # If this is a view, index the names too.
            if not isinstance(indices, tuple):
                indices = (indices, )
            new_names = []
            dim = 0
            for i, index in enumerate(indices):
                if isinstance(index, numbers.Integral):
                    dim += 1  # Drop this dimension's names.
                elif index is Ellipsis:
                    # Copy all the dimensions' names through.
                    end = len(self.shape) - len(indices) + i + 1
                    for j in range(dim, end):
                        new_names.append(self._index_names[j])
                    dim = end
                elif index is np.newaxis:  # Add an unnamed dimension.
                    new_names.append(None)
                    # Don't modify dim, as we're still working on the same one.
                elif (self._index_names[dim] is None
                      or (isinstance(index, slice) and index == _NULL_SLICE)):
                    # Keep unnamed dimensions or ones where the slice is a no-op.
                    new_names.append(self._index_names[dim])
                    dim += 1
                elif isinstance(index, (slice, list, np.ndarray)):
                    if isinstance(index, np.ndarray) and len(index.shape) > 1:
                        raise TypeError(
                            "What does it mean to index into a named array by "
                            "a multidimensional array? %s" % index)
                    # Rebuild the index of names for the various forms of slicing.
                    names = sorted(self._index_names[dim].items(),
                                   key=lambda item: item[1])
                    names = np.array(
                        names, dtype=object)  # Support full numpy slicing.
                    sliced = names[index]  # Actually slice it.
                    indexed = {n: j
                               for j, (n, _) in enumerate(sliced)}  # Reindex.
                    if len(sliced) != len(indexed):
                        # Names aren't unique, so drop the names for this dimension.
                        indexed = None
                    new_names.append(indexed)
                    dim += 1
                else:
                    raise TypeError("Unknown index: %s; %s" %
                                    (type(index), index))
            obj._index_names = new_names + self._index_names[dim:]
            if len(obj._index_names) != len(obj.shape):
                raise IndexError("Names don't match object shape: %s != %s" %
                                 (len(obj.shape), len(obj._index_names)))
        return obj
Пример #43
0
def compute_trajectory(dynamics, policy, initial_state, num_steps):
    """Compute a state trajectory given dynamics and a policy.

    Parameters
    ----------
    dynamics : callable
        A function that takes the current state and action as input and returns
        the next state.
    policy : callable
        A function that takes the current state as input and returns the
        action.
    initial_state : Tensor or ndarray
        The initial state at which to start simulating.
    num_steps : int
        The number of steps for which to simulate the system.

    Returns
    -------
    states : ndarray
        A (num_steps x state_dim) array with one state on each row.
    actions : ndarray
        A (num_steps x action_dim) array with the corresponding action on each
        row.
    """
    initial_state = np.atleast_2d(initial_state)
    state_dim = initial_state.shape[1]

    # Get storage (indexed by dynamics and policy)
    index = (dynamics, policy)
    storage = get_storage(_STORAGE, index=index)

    if storage is None:
        # Compute next state under the policy
        tf_state = tf.placeholder(config.dtype, [1, state_dim])
        tf_action = policy(tf_state)
        tf_next_state = dynamics(tf_state, tf_action)

        storage = [('tf_state', tf_state), ('tf_action', tf_action),
                   ('tf_next_state', tf_next_state)]

        set_storage(_STORAGE, storage, index=index)
    else:
        tf_state, tf_action, tf_next_state = storage.values()

    # Initialize
    dtype = config.np_dtype
    states = np.empty((num_steps, state_dim), dtype=dtype)
    actions = np.empty((num_steps - 1, policy.output_dim), dtype=dtype)

    states[0, :] = initial_state

    # Get the feed dict
    session = tf.get_default_session()
    feed_dict = get_feed_dict(session.graph)

    next_data = [tf_next_state, tf_action]

    # Run simulation
    for i in range(num_steps - 1):
        feed_dict[tf_state] = states[[i], :]
        states[i + 1, :], actions[i, :] = session.run(next_data,
                                                      feed_dict=feed_dict)

    return states, actions
Пример #44
0
    def __init__(self, form, context, *args, **kwargs):
        """
        Dynamically add each of the form fields for the given form model
        instance and its related field model instances.
        """
        self.form = form
        self.form_fields = form.fields.visible()
        initial = kwargs.pop("initial", {})
        # If a FormEntry instance is given to edit, stores it's field
        # values for using as initial data.
        field_entries = {}
        if kwargs.get("instance"):
            for field_entry in kwargs["instance"].fields.all():
                field_entries[field_entry.field_id] = field_entry.value
        super(FormForForm, self).__init__(*args, **kwargs)
        # Create the form fields.
        for field in self.form_fields:
            field_key = field.slug
            field_class = fields.CLASSES[field.field_type]
            field_widget = fields.WIDGETS.get(field.field_type)
            field_args = {
                "label": field.label,
                "required": field.required,
                "help_text": field.help_text
            }
            arg_names = field_class.__init__.__code__.co_varnames
            if "max_length" in arg_names:
                field_args["max_length"] = settings.FIELD_MAX_LENGTH
            if "choices" in arg_names:
                field_args["choices"] = field.get_choices()
            if field_widget is not None:
                field_args["widget"] = field_widget
            #
            #   Initial value for field, in order of preference:
            #
            # - If a form model instance is given (eg we're editing a
            #   form response), then use the instance's value for the
            #   field.
            # - If the developer has provided an explicit "initial"
            #   dict, use it.
            # - The default value for the field instance as given in
            #   the admin.
            #
            initial_val = None
            try:
                initial_val = field_entries[field.id]
            except KeyError:
                try:
                    initial_val = initial[field_key]
                except KeyError:
                    initial_val = Template(field.default).render(context)
            if initial_val:
                if field.is_a(*fields.MULTIPLE):
                    initial_val = split_choices(initial_val)
                if field.field_type == fields.CHECKBOX:
                    initial_val = initial_val != "False"
                self.initial[field_key] = initial_val
            self.fields[field_key] = field_class(**field_args)

            if field.field_type == fields.DOB:
                now = datetime.now()
                years = list(range(now.year, now.year - 120, -1))
                self.fields[field_key].widget.years = years

            # Add identifying CSS classes to the field.
            css_class = field_class.__name__.lower()
            if field.required:
                css_class += " required"
                if (settings.USE_HTML5
                        and field.field_type != fields.CHECKBOX_MULTIPLE):
                    self.fields[field_key].widget.attrs["required"] = ""
            self.fields[field_key].widget.attrs["class"] = css_class
            if field.placeholder_text and not field.default:
                text = field.placeholder_text
                self.fields[field_key].widget.attrs["placeholder"] = text
Пример #45
0
def straight(cells, prop, srcs, recs, velocity=None, par=False):
    """
    Calculate the travel times inside a mesh of square cells between source and
    receiver pairs assuming the rays are straight lines (no refraction or
    reflection).

    .. note:: Don't care about the units as long they are compatible.

    For a homogeneous model, *cells* can be a list with only one big cell.

    Parameters:

    * cells : list of :func:`geoist.mesher.Square`
        The velocity model to use to trace the straight rays. Cells must have
        the physical property given in parameter *prop*. This will be used
        as the velocity of each cell. (*cells* can also be a
        :class:`~geoist.mesher.SquareMesh`)
    * prop : str
        Which physical property of the cells to use as velocity.
        Normaly one would choose ``'vp'`` or ``'vs'``
    * srcs : list fo lists
        List with [x, y] coordinate pairs of the wave sources.
    * recs : list fo lists
        List with [x, y] coordinate pairs of the receivers sources
    * velocity : float or None
        If not None, will use this value instead of the prop of cells as the
        velocity. Useful when building sensitivity matrices (use velocity = 1).
    * par : True or False
        If True, will run the calculations in parallel using all the cores
        available. Not recommended for Jacobian matrix building!

    *srcs* and *recs* are lists of source-receiver pairs. Each source in *srcs*
    is associated with the corresponding receiver in *recs* for a given travel
    time.

    For example::

        >>> # One source was recorded at 3 receivers.
        >>> # The medium is homogeneous and can be
        >>> # represented by a single Square
        >>> from geoist.mesher import Square
        >>> cells = [Square([0, 10, 0, 10], {'vp':2})]
        >>> src = (5, 0)
        >>> srcs = [src, src, src]
        >>> recs = [(0, 0), (5, 10), (10, 0)]
        >>> print straight(cells, 'vp', srcs, recs)
        [ 2.5  5.   2.5]

    Returns:

    * times : array
        The total times each ray took to get from a source to a receiver (in
        compatible units with *prop*)

    """
    if len(srcs) != len(recs):
        raise ValueError("Must have the same number of sources and receivers")
    if not par:
        if _ttime2d is not None:
            x_src, y_src = numpy.transpose(srcs).astype(numpy.float)
            x_rec, y_rec = numpy.transpose(recs).astype(numpy.float)
            times = _ttime2d.straight(x_src, y_src, x_rec, y_rec, len(srcs),
                                      cells, velocity, prop)
        else:
            times = _straight(cells, prop, srcs, recs, velocity)
        return times
    # Divide the workload into jobs and run them in different processes
    jobs = multiprocessing.cpu_count()
    start = 0
    size = len(srcs)
    perjob = size / jobs
    processes = []
    pipes = []
    for i in range(jobs):
        if i == jobs - 1:
            end = size
        else:
            end = start + perjob
        outpipe, inpipe = multiprocessing.Pipe()
        args = (inpipe, srcs[start:end], recs[
                start:end], cells, velocity, prop)
        proc = multiprocessing.Process(target=_straight_job, args=args)
        proc.start()
        processes.append(proc)
        pipes.append(outpipe)
        start = end
    times = []
    for proc, pipe in zip(processes, pipes):
        times.extend(pipe.recv())
        proc.join()
    return numpy.array(times)
Пример #46
0
def GeneratePassphrase(length=20):
  """Create a 20 char passphrase with easily typeable chars."""
  valid_chars = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ"
  valid_chars += "0123456789 ,-_&$#"
  return "".join(random.choice(valid_chars) for i in range(length))
Пример #47
0
 def test_contains(self):
     self.assertIn(1, range(2))
     self.assertNotIn(10, range(2))
     self.assertNotIn(None, range(2))
     self.assertNotIn("", range(2))
Пример #48
0
 def test_slice_overflow_range(self):
     r = range(8)
     self.assertRangesEqual(r[2:200], range(2, 8))
     self.assertRangesEqual(r[-200:-2], range(0, 6))
Пример #49
0
 def minimize(self, objective):
     stats = dict(method="Levemberg-Marquardt",
                  iterations=0,
                  objective=[],
                  step_attempts=[],
                  step_size=[])
     p = np.array(self.initial)
     value = objective.value(p)
     lamb = self.lamb
     stats['objective'].append(value)
     stats['step_attempts'].append(0)
     stats['step_size'].append(lamb)
     for iteration in range(self.maxit):
         grad = objective.gradient(p)
         hess = objective.hessian(p)
         if self.precondition:
             diag = np.abs(safe_diagonal(hess))
             diag[diag < 1e-10] = 1e-10
             precond = sp.diags(1/diag, 0).tocsr()
             hess = safe_dot(precond, hess)
             grad = safe_dot(precond, grad)
         diag = sp.diags(safe_diagonal(hess), 0).tocsr()
         # Try to take a step
         took_step = False
         for step in range(self.maxsteps):
             newp = p + safe_solve(hess + lamb*diag, -grad)
             newvalue = objective.value(newp)
             decrease = newvalue < value
             if not decrease:
                 if lamb < 1e15:
                     lamb = lamb*self.dlamb
             else:
                 if lamb > 1e-15:
                     lamb = lamb/self.dlamb
                 took_step = True
                 break
         if not took_step:
             stop = True
             warnings.warn(
                 "LevMarq optimizer exited because couldn't take a step "
                 + 'without increasing the objective function. '
                 + 'Might not have achieved convergence. '
                 + 'Try increasing the max number of step attempts.',
                 RuntimeWarning)
         else:
             stop = abs(newvalue - value) < self.tol*abs(value)
             p = newp
             value = newvalue
             stats['objective'].append(value)
             stats['iterations'] += 1
             stats['step_attempts'].append(step + 1)
             stats['step_size'].append(lamb)
         if stop:
             break
     if iteration == self.maxit - 1:
         warnings.warn(
             'LevMarq optmizer exited because maximum iterations reached. '
             + 'Might not have achieved convergence. '
             + 'Try increasing the maximum number of iterations allowed.',
             RuntimeWarning)
     self.stats = stats
     return p
Пример #50
0
    def create_table(self, fromcell=None, tocell=None, nheader=0, sheet=0):
        """
    Creates a table (as a list) based on given query and columns

    fromcell:
      The index of the cell where to begin. The default
      is from the beginning of the data set (0, 0).

    tocell:
      The index of the cell where to end the selection.
      Default is in the end of the data set.

    nheader:
      Number of lines which are considered as a header lines.
      Normally, the value is 0 (default) or 1.

    sheet:
      Name or index of the sheet as string/unicode. The index starts from the 0
      and is the default value. If numeric value is given, provide it in format::

        et.create_table(fromcell='A1', tocell='B2', sheet='2')

    """
        rows = []

        # Select sheet by given index or name
        if type(sheet) is int or sheet.isdigit():
            sh1 = self.book.sheet_by_index(int(sheet))
        else:
            sh1 = self.book.sheet_by_name(sheet)

        # Name selection, like: 'A1' or 'AB12'
        if isinstance(fromcell, basestring):
            match = re.match(r'(?P<chars>[A-Z]+)(?P<nums>[1-9]+[0-9]*)',
                             fromcell)
            if match:
                parts = (match.group('chars'), int(match.group('nums')))
                fromcell = toindex(*parts)
            else:
                fromcell = tuple([int(num) for num in fromcell.split(u',')])

        # Name selection, like: 'A1' or 'AB12'
        if isinstance(tocell, basestring):
            match = re.match(r'(?P<chars>[A-Z]+)(?P<nums>[1-9]+[0-9]*)',
                             tocell)
            if match:
                parts = (match.group('chars'), int(match.group('nums')))
                tocell = toindex(*parts)
            else:
                tocell = tuple([int(num) for num in tocell.split(u',')])

        if not fromcell:
            fromcell = (0, 0)

        # If ending cell is not given, calculate
        # it from rows and cols
        #print sh1.ncols, sh1.nrows
        #print (tocell[0] > (sh1.ncols -1)) or (tocell[1] > (sh1.nrows -1))
        maxrow_index = sh1.nrows - 1
        maxcol_index = sh1.ncols - 1
        if not tocell:
            tocell = (maxcol_index, maxrow_index)

        # If the value is bigger than the value, default to max value
        if int(tocell[0]) > maxcol_index:
            tocell = (maxcol_index, tocell[1])

        # If the value is bigger than the value, default to max value
        if int(tocell[1]) > maxrow_index:
            tocell = (tocell[0], maxrow_index)

        # Iterate columns
        rows = {'headers': [], 'rows': []}
        widths = []
        for rnum in range(fromcell[1], tocell[1] + 1):

            # Iterate rows within column
            cols = []
            for cnum in range(fromcell[0], tocell[0] + 1):
                cell = sh1.cell(rnum, cnum)
                width = sh1.computed_column_width(cnum)

                # Put data
                cell_data = {
                    'type': 'row',
                    'width': width,
                    'value': self._get_value(cell)
                }

                # If header row
                if rnum < nheader:
                    cell_data['type'] = 'header'

                # Get more format info for the cell
                cell_data.update(self._get_formatting(cell))

                cols.append(cell_data)

            # If first column is header, their all headers - i think
            if cols[0]['type'] == 'header':
                rows['headers'].append(cols)
            else:
                rows['rows'].append(cols)

        #widths_together = sum([cell['width'] for cols in rows])
        #print widths_together
        #widths = [round(val * 100.0 / widths_together) for val in widths]

        # Store into object for validation purposes
        self.fromcell = fromcell
        self.tocell = tocell

        return rows
from future.builtins import range
import numpy as np
import modred as mr

num_vecs = 30
# Arbitrary data
vecs = np.random.random((100, num_vecs))
num_modes = 5
modes, eig_vals = mr.compute_POD_matrices_snaps_method(vecs,
                                                       list(range(num_modes)))
Пример #52
0
 def minimize(self, objective):
     stats = dict(method='Steepest Descent',
                  iterations=0,
                  objective=[],
                  step_attempts=[])
     p = np.array(self.initial)
     value = objective.value(p)
     stats['objective'].append(value)
     if self.linesearch:
         stats['step_attempts'].append(0)
     alpha = 1e-4  # This is a mystic parameter of the Armijo rule
     for iteration in range(self.maxit):
         grad = objective.gradient(p)
         if self.linesearch:
             # Calculate now to avoid computing inside the loop
             gradnorm = np.linalg.norm(grad)**2
             # Determine the best step size
             took_step = False
             for i in range(self.maxsteps):
                 stepsize = self.beta**i
                 newp = p - stepsize*grad
                 newvalue = objective.value(newp)
                 decreased = newvalue < value
                 good_step = newvalue - value < alpha*stepsize*gradnorm
                 if decreased and good_step:
                     took_step = True
                     break
         else:
             newp = p - grad
             newvalue = objective.value(newp)
             decreased = newvalue < value
             if decreased:
                 took_step = True
             else:
                 took_step = False
         if not took_step:
             stop = True
             warnings.warn(
                 "SteepestDescent optimizer exited because couldn't take a"
                 + " step without increasing the objective function. "
                 + 'Might not have achieved convergence. '
                 + 'Try increasing the max number of step attempts allowed.',
                 RuntimeWarning)
         else:
             stop = abs(newvalue - value) < self.tol*abs(value)
             p = newp
             value = newvalue
             stats['objective'].append(value)
             stats['iterations'] += 1
             if self.linesearch:
                 stats['step_attempts'].append(i + 1)
         if stop:
             break
     if iteration == self.maxit - 1:
         warnings.warn(
             'SteepestDescent optimizer exited because maximum iterations '
             + 'reached. Might not have achieved convergence. '
             + 'Try increasing the maximum number of iterations allowed.',
             RuntimeWarning)
     self.stats = stats
     return p
Пример #53
0
    def __init__(self, zeros):
        """
        Constructor
        """
        QDialog.__init__(self)
        self.__zeros = zeros
        self.setWindowTitle(QCoreApplication.translate("VDLTools", "Zeros"))
        self.__layout = QGridLayout()

        self.__zeroLabels = []
        self.__zeroChecks = []

        displayButton = False

        self.__scrollLayout = QGridLayout()

        for i in range(len(self.__zeros)):
            msg = "- vertex " + str(self.__zeros[i][0])
            msg += QCoreApplication.translate("VDLTools",
                                              ", elevation : '0', ")
            if self.__zeros[i][1] is not None:
                msg += QCoreApplication.translate("VDLTools",
                                                  "interpolated elevation : ")
                msg += str(self.__zeros[i][1]) + "m"
                if self.__zeros[i][2] > 1:
                    msg += QCoreApplication.translate("VDLTools",
                                                      " (and apply to point)")
                msgCheck = QCheckBox()
                msgCheck.setChecked(True)
                self.__zeroChecks.append(msgCheck)
                self.__scrollLayout.addWidget(self.__zeroChecks[i], i + 1, 2)
                displayButton = True
            else:
                msg += QCoreApplication.translate("VDLTools",
                                                  "no interpolated elevation")
                self.__zeroChecks.append(None)

            zeroLabel = QLabel(msg)
            self.__zeroLabels.append(zeroLabel)
            self.__scrollLayout.addWidget(self.__zeroLabels[i], i + 1, 0, 1, 2)

        widget = QWidget()
        widget.setLayout(self.__scrollLayout)

        scroll = QScrollArea()
        scroll.setWidgetResizable(True)
        scroll.setWidget(widget)

        self.__layout.addWidget(scroll, 1, 0, 1, 2)

        self.__passButton = QPushButton(
            QCoreApplication.translate("VDLTools", "Pass"))
        self.__passButton.setMinimumHeight(20)
        self.__passButton.setMinimumWidth(100)

        pos = len(self.__zeros) + 1
        self.__layout.addWidget(self.__passButton, pos, 0)

        self.__applyButton = QPushButton(
            QCoreApplication.translate("VDLTools", "Apply interpolation"))
        self.__applyButton.setMinimumHeight(20)
        self.__applyButton.setMinimumWidth(100)
        if displayButton:
            self.__layout.addWidget(self.__applyButton, pos, 1)

        self.setLayout(self.__layout)
Пример #54
0
def get_lyapunov_region(lyapunov, discretization, init_node):
    """Get the region within which a function is a Lyapunov function.

    Parameters
    ----------
    lyapunov : callable
        A tensorflow function.
    discretization : instance of `GridWorld`
        The discretization on which to check the increasing property.
    init_node : tuple
        The node at which to start the verification.

    Returns
    -------
    region : ndarray
        A boolean array that contains all the states for which lyapunov is a
        Lyapunov function that can be used for stability verification.

    """
    # Turn values into a multi-dim array
    feed_dict = lyapunov.feed_dict

    values = lyapunov(discretization.all_points).eval(feed_dict=feed_dict)
    lyapunov_values = values.reshape(discretization.num_points)

    # Starting point for the verification
    init_value = lyapunov_values[init_node]

    ndim = discretization.ndim
    num_points = discretization.num_points

    # Indices for generating neighbors
    index_generator = itertools.product(*[(0, -1, 1) for _ in range(ndim)])
    neighbor_indeces = np.array(tuple(index_generator)[1:])

    # Array keeping track of visited nodes
    visited = np.zeros(discretization.num_points, dtype=np.bool)
    visited[init_node] = True

    # Create priority queue
    tiebreaker = itertools.count()
    last_value = init_value
    priority_queue = [(init_value, tiebreaker.next(), init_node)]

    while priority_queue:
        value, _, next_node = heappop(priority_queue)

        # Check if we reached the boundary of the discretization
        if np.any(0 == next_node) or np.any(next_node == num_points - 1):
            visited[tuple(next_node)] = False
            break

        # Make sure we are in the positive definite part of the function.
        if value < last_value:
            break

        last_value = value

        # Get all neighbors
        neighbors = next_node + neighbor_indeces

        # Remove neighbors that are already part of the visited set
        is_new = ~visited[np.split(neighbors.T, ndim)]
        neighbors = neighbors[is_new[0]]

        if neighbors.size:
            indices = np.split(neighbors.T, ndim)
            # add to visited set
            visited[indices] = True
            # get values
            values = lyapunov_values[indices][0]

            # add to priority queue
            for value, neighbor in zip(values, neighbors):
                heappush(priority_queue, (value, next(tiebreaker), neighbor))

    # Prune nodes that were neighbors, but haven't been visited
    for _, _, node in priority_queue:
        visited[tuple(node)] = False

    return visited
Пример #55
0
    def _download(self, chunks, resume):
        if not resume:
            self.info.clear()
            self.info.add_chunk('{0}.chunk0'.format(
                self.path), (0, 0))  # create an initial entry

        self.chunks = []

        # initial chunk that will load complete file (if needed)
        init = CurlChunk(0, self, None, resume)

        self.chunks.append(init)
        self.manager.add_handle(init.get_handle())

        last_finish_check = 0
        last_time_check = 0
        chunks_done = set()  # list of curl handles that are finished
        chunks_created = False
        done = False
        # This is a resume, if we were chunked originally assume still can
        if self.info.get_count() > 1:
            self.chunk_support = True

        while True:
            # need to create chunks
            # will be set later by first chunk
            if not chunks_created and self.chunk_support and self.size:

                self.flags ^= Connection.Resumable  # TODO: Recheck...
                if not resume:
                    self.info.set_size(self.size)
                    self.info.create_chunks(chunks)
                    self.info.save()

                chunks = self.info.get_count()

                init.set_range(self.info.get_chunk_range(0))

                for i in range(1, chunks):
                    c = CurlChunk(
                        i, self, self.info.get_chunk_range(i), resume)

                    handle = c.get_handle()
                    if handle:
                        self.chunks.append(c)
                        self.manager.add_handle(handle)
                    else:
                        # close immediately
                        self.log.debug('Invalid curl handle -> closed')
                        c.close()

                chunks_created = True

            while True:
                ret, _ = self.manager.perform()
                if ret != pycurl.E_CALL_MULTI_PERFORM:
                    break

            t = time.time()

            # reduce these calls
            # when num_q is 0, the loop is exited
            while last_finish_check + 0.5 < t:
                # list of failed curl handles
                failed = []

                # TODO: Rewrite...
                # save only last exception, we can only raise one anyway
                exc = Exception()

                num_q, ok_list, err_list = self.manager.info_read()
                for c in ok_list:
                    chunk = self.find_chunk(c)
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as exc:
                        self.log.debug(
                            'Chunk {0:d} failed'.format(
                                chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        failed.append(chunk)
                    else:
                        chunks_done.add(c)

                for c in err_list:
                    curl, errno, msg = c
                    chunk = self.find_chunk(curl)
                    # test if chunk was finished
                    if errno != 23 or '0 !=' not in msg:
                        failed.append(chunk)
                        exc = pycurl.error(errno, msg)
                        self.log.debug(
                            'Chunk {0:d} failed'.format(chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        continue
                    # check if the header implies success,
                    # else add it to failed list
                    try:
                        chunk.verify_header()
                    except ResponseException as exc:
                        self.log.debug(
                            'Chunk {0:d} failed'.format(
                                chunk.id + 1))
                        self.log.debug(exc, exc_info=True)
                        failed.append(chunk)
                    else:
                        chunks_done.add(curl)
                if not num_q:  # no more info to get

                    # check if init is not finished so we reset download
                    # connections
                    # note that other chunks are closed and everything
                    # downloaded with initial connection
                    if failed:
                        if init in failed or init.curl in chunks_done:
                            raise exc
                        self.log.error(
                            'Download chunks failed, fallback to '
                            'single connection | {0}'.format(exc))

                        # list of chunks to clean and remove
                        to_clean = [x for x in self.chunks if x is not init]
                        for chunk in to_clean:
                            self.close_chunk(chunk)
                            self.chunks.remove(chunk)
                            remove(self.info.get_chunk_name(chunk.id))

                        # let first chunk load the rest and update the
                        # info file
                        init.reset_range()
                        self.info.clear()
                        self.info.add_chunk('{0}.chunk0'.format(
                            self.path), (0, self.size))
                        self.info.save()

                    last_finish_check = t

                    if len(chunks_done) >= len(self.chunks):
                        if len(chunks_done) > len(self.chunks):
                            self.log.warning(
                                'Finished download chunks size incorrect')
                        done = True  # all chunks loaded

                    break

            if done:
                break  # all chunks loaded

            # calc speed once per second, averaging over 3 seconds
            if last_time_check + 1 < t:
                len_la = len(self.last_arrived)
                diff = [c.arrived - (self.last_arrived[i] if len_la > i else 0)
                        for i, c in enumerate(self.chunks)]

                self.last_speeds[1] = self.last_speeds[0]
                self.last_speeds[0] = self.speeds
                self.speeds = [float(a) // (t - last_time_check) for a in diff]
                self.last_arrived = [c.arrived for c in self.chunks]
                last_time_check = t

            if self._abort:
                raise Abort

            self.manager.select(1)

        for chunk in self.chunks:
            chunk.flush_file()  # make sure downloads are written to disk

        self._copy_chunks()
def load_imitation_learning_network(input_image, input_data, mode):
    branches = []

    x = input_image
    if mode == tf.estimator.ModeKeys.TRAIN:
        dropout = DROPOUT_VEC_TRAIN
        is_training = True
    else:
        dropout = DROPOUT_VEC_INFER
        is_training = False

    with tf.name_scope(
            'Network'
    ):  # for a nicer Tensorboard graph, use: `with tf.variable_scope('Network'):`
        network_manager = Network(dropout, tf.shape(x), is_training)
        """conv1"""  # kernel sz, stride, num feature maps
        xc = network_manager.conv_block(x, 5, 2, 32, padding_in='VALID')
        xc = network_manager.conv_block(xc, 3, 1, 32, padding_in='VALID')
        """conv2"""
        xc = network_manager.conv_block(xc, 3, 2, 64, padding_in='VALID')
        xc = network_manager.conv_block(xc, 3, 1, 64, padding_in='VALID')
        """conv3"""
        xc = network_manager.conv_block(xc, 3, 2, 128, padding_in='VALID')
        xc = network_manager.conv_block(xc, 3, 1, 128, padding_in='VALID')
        """conv4"""
        xc = network_manager.conv_block(xc, 3, 1, 256, padding_in='VALID')
        xc = network_manager.conv_block(xc, 3, 1, 256, padding_in='VALID')
        """mp3 (default values)"""
        """ reshape """
        x = tf.reshape(xc, [-1, int(np.prod(xc.get_shape()[1:]))],
                       name='reshape')
        """ fc1 """
        x = network_manager.fc_block(x, 512)
        """ fc2 """
        x = network_manager.fc_block(x, 512)
        """Process Control"""
        """ Speed (measurements)"""
        with tf.name_scope("Speed"):
            speed = input_data[1]  # get the speed from input data
            speed = network_manager.fc_block(speed, 128)
            speed = network_manager.fc_block(speed, 128)
        """ Joint sensory """
        j = tf.concat([x, speed], 1)
        j = network_manager.fc_block(j, 512)
        """Start BRANCHING"""
        branch_config = [
            [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE],
            [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE],
            [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE],
            [ilc.TGT_STEER, ilc.TGT_GAS, ilc.TGT_BRAKE],
            [ilc.TGT_SPEED],
        ]
        for i in range(0, len(branch_config)):
            with tf.name_scope("Branch_{}".format(i)):
                if branch_config[i][0] == ilc.TGT_SPEED:
                    # we only use the image as input to speed prediction
                    branch_output = network_manager.fc_block(x, 256)
                    branch_output = network_manager.fc_block(
                        branch_output, 256)
                else:
                    branch_output = network_manager.fc_block(j, 256)
                    branch_output = network_manager.fc_block(
                        branch_output, 256)

                branches.append(
                    network_manager.fc(branch_output, len(branch_config[i])))

        print(branch_output)
        return branches
Пример #57
0
 def SetupClients(self, nr_clients, *args, **kwargs):
     """Prepares nr_clients test client mocks to be used."""
     return self.SetupClientsWithIndices(range(nr_clients), *args, **kwargs)
Пример #58
0
 def test_slice_zero_step(self):
     msg = '^slice step cannot be zero$'
     with self.assertRaisesRegexp(ValueError, msg):
         range(8)[::0]
Пример #59
0
    def lin_combine(self,
                    sum_vec_handles,
                    basis_vec_handles,
                    coeff_mat,
                    coeff_mat_col_indices=None):
        """Computes linear combination(s) of basis vector objects and calls
        ``put`` on result(s), using handles.

        Args:
            ``sum_vec_handles``: List of handles for the sum vector objects.

            ``basis_vec_handles``: List of handles for the basis vector objects.

            ``coeff_mat``: Matrix whose rows correspond to basis vectors and
            whose columns correspond to sum vectors.  The rows and columns
            correspond, by index, to the lists ``basis_vec_handles`` and
            ``sum_vec_handles``.  In matrix notation, we can write ``sums =
            basis * coeff_mat``

        Kwargs:
            ``coeff_mat_col_indices``: List of column indices.  Only the
            ``sum_vecs`` corresponding to these columns of the coefficient
            matrix are computed.

        Each MPI worker (processor) retrieves a subset of the basis vectors to
        compute as many outputs as an MPI worker (processor) can have in memory
        at once. Each MPI worker (processor) computes the "layers" from the
        basis it is responsible for, and for as many modes as it can fit in
        memory. The layers from all MPI workers (processors) are summed
        together to form the ``sum_vecs`` and ``put`` is called on each.

        Scaling is:

          num gets/worker = :math:`n_s/(n_p*(max-2)) * n_b/n_p`

          passes/worker = :math:`(n_p-1) * n_s/(n_p*(max-2)) * (n_b/n_p)`

          scalar multiplies/worker = :math:`n_s*n_b/n_p`

        where :math:`n_s` is number of sum vecs, :math:`n_b` is
        number of basis vecs,
        :math:`n_p` is number of processors,
        :math:`max` = ``max_vecs_per_node``.
        """
        sum_vec_handles = util.make_iterable(sum_vec_handles)
        basis_vec_handles = util.make_iterable(basis_vec_handles)
        num_bases = len(basis_vec_handles)
        num_sums = len(sum_vec_handles)
        if coeff_mat_col_indices is not None:
            coeff_mat = coeff_mat[:, coeff_mat_col_indices]
        if num_bases != coeff_mat.shape[0]:
            raise ValueError(('Number of coeff_mat rows (%d) does not equal '
                              'number of basis handles (%d)' %
                              (coeff_mat.shape[0], num_bases)))
        if num_sums != coeff_mat.shape[1]:
            raise ValueError(('Number of coeff_mat cols (%d) does not equal '
                              'number of output handles (%d)') %
                             (coeff_mat.shape[1], num_sums))

        # Estimate time it will take
        # Burn the first one for slow imports
        test_vec_burn = basis_vec_handles[0].get()
        test_vec_burn_3 = test_vec_burn + 2. * test_vec_burn
        del test_vec_burn, test_vec_burn_3
        start_time = time()
        test_vec = basis_vec_handles[0].get()
        get_time = time() - start_time
        start_time = time()
        test_vec_3 = test_vec + 2. * test_vec
        add_scale_time = time() - start_time
        del test_vec, test_vec_3

        vecs_per_worker = self.max_vecs_per_node * parallel.get_num_nodes() / \
            parallel.get_num_MPI_workers()
        num_gets = num_sums/(parallel.get_num_MPI_workers()*(\
            vecs_per_worker-2)) + \
            num_bases/parallel.get_num_MPI_workers()
        num_add_scales = num_sums * num_bases / parallel.get_num_MPI_workers()
        self.print_msg('Linear combinations will take at least %.1f minutes' %
                       (num_gets * get_time / 60. +
                        num_add_scales * add_scale_time / 60.))

        # convenience
        rank = parallel.get_rank()

        # num_bases_per_proc_chunk is the num of bases each proc gets at once.
        num_bases_per_proc_chunk = 1
        num_sums_per_proc_chunk = self.max_vecs_per_proc - \
            num_bases_per_proc_chunk

        basis_tasks = parallel.find_assignments(list(range(num_bases)))
        sum_tasks = parallel.find_assignments(list(range(num_sums)))

        # Find max number tasks among all processors
        max_num_basis_tasks = max([len(tasks) for tasks in basis_tasks])
        max_num_sum_tasks = max([len(tasks) for tasks in sum_tasks])

        # These variables are the number of iters through loops that retrieve
        # ("get")
        # and "put" basis and sum vecs.
        num_basis_get_iters = int(
            np.ceil(max_num_basis_tasks * 1. / num_bases_per_proc_chunk))
        num_sum_put_iters = int(
            np.ceil(max_num_sum_tasks * 1. / num_sums_per_proc_chunk))
        if num_sum_put_iters > 1:
            self.print_msg(
                'Warning: The basis vecs, '
                'of which there are %d, will be retrieved %d times each. '
                'If possible, increase number of nodes or '
                'max_vecs_per_node to reduce redundant retrieves and get a '
                'big speedup.' % (num_bases, num_sum_put_iters))

        for sum_put_index in range(num_sum_put_iters):
            if len(sum_tasks[rank]) > 0:
                start_sum_index = min(
                    sum_tasks[rank][0] +
                    sum_put_index * num_sums_per_proc_chunk,
                    sum_tasks[rank][-1] + 1)
                end_sum_index = min(start_sum_index + num_sums_per_proc_chunk,
                                    sum_tasks[rank][-1] + 1)

                # Create empty list on each processor
                sum_layers = [None] * (end_sum_index - start_sum_index)
            else:
                start_sum_index = 0
                end_sum_index = 0
                sum_layers = []

            for basis_get_index in range(num_basis_get_iters):
                if len(basis_tasks[rank]) > 0:
                    start_basis_index = min(
                        basis_tasks[rank][0] +
                        basis_get_index * num_bases_per_proc_chunk,
                        basis_tasks[rank][-1] + 1)
                    end_basis_index = min(
                        start_basis_index + num_bases_per_proc_chunk,
                        basis_tasks[rank][-1] + 1)
                    basis_indices = list(
                        range(start_basis_index, end_basis_index))
                else:
                    basis_indices = []

                # Pass the basis vecs to proc with rank -> mod(rank+1,num_procs)
                # Must do this for each processor, until data makes a circle
                basis_vecs_recv = (None, None)
                for pass_index in range(parallel.get_num_procs()):
                    # If on the first pass, retrieve the basis vecs,
                    # no send/recv.
                    # This is all that is called when in serial,
                    # loop iterates once.
                    if pass_index == 0:
                        if len(basis_indices) > 0:
                            basis_vecs = [basis_handle.get() \
                                for basis_handle in basis_vec_handles[
                                    basis_indices[0]:basis_indices[-1]+1]]
                        else:
                            basis_vecs = []
                    else:
                        # Figure out with whom to communicate
                        source = (parallel.get_rank()-1) % \
                            parallel.get_num_procs()
                        dest = (parallel.get_rank()+1) % \
                            parallel.get_num_procs()

                        #Create unique tags based on ranks
                        send_tag = parallel.get_rank() * \
                            (parallel.get_num_procs()+1) + dest
                        recv_tag = source*(parallel.get_num_procs()+1) + \
                            parallel.get_rank()

                        # Send/receive data
                        basis_vecs_send = (basis_vecs, basis_indices)
                        request = parallel.comm.isend(basis_vecs_send,
                                                      dest=dest,
                                                      tag=send_tag)
                        basis_vecs_recv = parallel.comm.recv(source=source,
                                                             tag=recv_tag)
                        request.Wait()
                        parallel.barrier()
                        basis_indices = basis_vecs_recv[1]
                        basis_vecs = basis_vecs_recv[0]

                    # Compute the scalar multiplications for this set of data.
                    # basis_indices stores the indices of the coeff_mat to
                    # use.
                    for sum_index in range(start_sum_index, end_sum_index):
                        for basis_index, basis_vec in enumerate(basis_vecs):
                            sum_layer = basis_vec * \
                                coeff_mat[basis_indices[basis_index],\
                                sum_index]
                            if sum_layers[sum_index - start_sum_index] is None:
                                sum_layers[sum_index-start_sum_index] = \
                                    sum_layer
                            else:
                                sum_layers[sum_index-start_sum_index] += \
                                    sum_layer
                        if ((time() - self.prev_print_time) >
                                self.print_interval):
                            self.print_msg(
                                'Completed %.1f%% of linear combinations' %
                                (sum_index * 100. / len(sum_tasks[rank])))
                            self.prev_print_time = time()

            # Completed this set of sum vecs, puts them to memory or file
            for sum_index in range(start_sum_index, end_sum_index):
                sum_vec_handles[sum_index].put(sum_layers[sum_index -
                                                          start_sum_index])
            del sum_layers

        self.print_msg('Completed %.1f%% of linear combinations' % 100.)
        self.prev_print_time = time()
        parallel.barrier()
Пример #60
0
def grouped_impl(wrap, size, sequence):
    for i in range(0, len(sequence), size):
        yield wrap(sequence[i:i + size])