def _getplotdata(format='png'):
    import io
    io = io.StringIO()
    plt.savefig(io, format=format)
    val = io.getvalue()
    io.close()
    return val
 def test_close(self):
     io = Interface(lambda input: None, "textbox", "label")
     _, local_url, _ = io.launch(prevent_thread_lock=True)
     response = requests.get(local_url)
     self.assertEqual(response.status_code, 200)
     io.close()
     with self.assertRaises(Exception):
         response = requests.get(local_url)
Beispiel #3
0
 def _do_close(self, io, closefd):
     try:
         io.close()
         # self.fileio already knows whether or not to close the
         # file descriptor
         self.fileio.close()
     finally:
         self._fobj = None
         self.fileio = None
Beispiel #4
0
 def _do_close(self, io, closefd):
     try:
         io.close()
         # self.fileio already knows whether or not to close the
         # file descriptor
         self.fileio.close()
     finally:
         self._fobj = None
         self.fileio = None
Beispiel #5
0
 def test_stream_io(self):
     import io
     f = gzip.open("tests/saint_barthelemy.osm.gz")
     io = io.BytesIO(f.read())
     i1 = OsmSaxReader(io, "tests/saint_barthelemy.state.txt")
     o1 = TestCountObjects()
     i1.CopyTo(o1)
     self.assertEquals(o1.num_nodes, 8076)
     self.assertEquals(o1.num_ways, 625)
     self.assertEquals(o1.num_rels, 16)
     io.close()
Beispiel #6
0
    def test_response_is_iterable(self):
        r = requests.Response()
        io = StringIO.StringIO('abc')
        read_ = io.read

        def read_mock(amt, decode_content=None):
            return read_(amt)
        setattr(io, 'read', read_mock)
        r.raw = io
        assert next(iter(r))
        io.close()
    def test_response_is_iterable(self):
        r = requests.Response()
        io = StringIO.StringIO('abc')
        read_ = io.read

        def read_mock(amt, decode_content=None):
            return read_(amt)
        setattr(io, 'read', read_mock)
        r.raw = io
        assert next(iter(r))
        io.close()
Beispiel #8
0
    def test_read(self):
        io = StringIO()
        writer = excelcsv.DictWriter(io, [u'1', u'2', u'3'])
        writer.writeheader()
        writer.writerow({u'1': u'a', u'2': u'b', u'3': u'c'})
        writer.writerow({u'1': u'd', u'2': u'e', u'3': u'f'})

        value = io.getvalue()
        self.assertIsInstance(value, text_type)
        self.assertEqual(value, u'1,2,3\r\na,b,c\r\nd,e,f\r\n')

        io.close()
Beispiel #9
0
 def test_create_tunnel(self):
     response = requests.get(networking.GRADIO_API_SERVER)
     payload = response.json()[0]
     io = Interface(lambda x: x, "text", "text")
     _, path_to_local_server, _ = io.launch(prevent_thread_lock=True,
                                            share=False)
     _, localhost, port = path_to_local_server.split(":")
     threading.Thread.start = mock.MagicMock(return_value=None)
     paramiko.SSHClient.connect = mock.MagicMock(return_value=None)
     tunneling.create_tunnel(payload, localhost, port)
     threading.Thread.start.assert_called_once()
     paramiko.SSHClient.connect.assert_called_once()
     io.close()
Beispiel #10
0
def test_limited_stream_disconnection():
    io = BytesIO(b'A bit of content')

    # disconnect detection on out of bytes
    stream = wsgi.LimitedStream(io, 255)
    with pytest.raises(ClientDisconnected):
        stream.read()

    # disconnect detection because file close
    io = BytesIO(b'x' * 255)
    io.close()
    stream = wsgi.LimitedStream(io, 255)
    with pytest.raises(ClientDisconnected):
        stream.read()
Beispiel #11
0
def test_limited_stream_disconnection():
    io = BytesIO(b'A bit of content')

    # disconnect detection on out of bytes
    stream = wsgi.LimitedStream(io, 255)
    with pytest.raises(ClientDisconnected):
        stream.read()

    # disconnect detection because file close
    io = BytesIO(b'x' * 255)
    io.close()
    stream = wsgi.LimitedStream(io, 255)
    with pytest.raises(ClientDisconnected):
        stream.read()
def gather_archives(mailbox_url, search_terms, destio):
    """Download the mailbox at mailbox_url, and write every message
    containing one of the terms in search_terms, or a message
    (transitively) in reply to one of those messages, to destio,
    the output mailbox stream."""

    io = cStringIO.StringIO()
    fetch_https_securely(mailbox_url, io,
                         username=passwords.get_w3c_username(),
                         password=passwords.get_w3c_password())
    month_message_str = io.getvalue()
    io.close()

    message_ids_included = set()
    for message in generate_messages(month_message_str):
        include = False
        message_id = None
        for [header_name, header_value] in generate_headers(message):
            header_name = header_name.lower()
            if header_name == "message-id":
                message_id = header_value
            elif header_name == "in-reply-to":
                if not include:
                    if header_value in message_ids_included:
                        # This is a reply to a message in our set.
                        include = True
            elif header_name == "references":
                if not include:
                    for reference in ws_re.split(header_value):
                        if reference in message_ids_included:
                            # This is a reply to a message in our set.
                            include = True
                            break
        if not include:
            for term in search_terms:
                if message.find(term) != -1:
                    include = True
                    break
        if include:
            message_ids_included.add(message_id)
            destio.write(message)
Beispiel #13
0
def prepare_data(argv=None):
    '''Aggregate sequence data GTDB using a file-of-files'''
    import argparse
    import io
    import sys
    import logging
    import h5py
    import pandas as pd

    from skbio import TreeNode

    from hdmf.common import get_hdf5io
    from hdmf.data_utils import DataChunkIterator

    from ..utils import get_faa_path, get_fna_path, get_genomic_path
    from exabiome.sequence.convert import AASeqIterator, DNASeqIterator, DNAVocabIterator, DNAVocabGeneIterator
    from exabiome.sequence.dna_table import AATable, DNATable, SequenceTable, TaxaTable, DeepIndexFile, NewickString, CondensedDistanceMatrix

    parser = argparse.ArgumentParser()
    parser.add_argument(
        'accessions',
        type=str,
        help='file of the NCBI accessions of the genomes to convert')
    parser.add_argument('fadir',
                        type=str,
                        help='directory with NCBI sequence files')
    parser.add_argument('metadata', type=str, help='metadata file from GTDB')
    parser.add_argument('tree', type=str, help='the distances file')
    parser.add_argument('out', type=str, help='output HDF5')
    grp = parser.add_mutually_exclusive_group()
    parser.add_argument('-e',
                        '--emb',
                        type=str,
                        help='embedding file',
                        default=None)
    grp.add_argument('-p',
                     '--protein',
                     action='store_true',
                     default=False,
                     help='get paths for protein files')
    grp.add_argument('-c',
                     '--cds',
                     action='store_true',
                     default=False,
                     help='get paths for CDS files')
    grp.add_argument('-g',
                     '--genomic',
                     action='store_true',
                     default=False,
                     help='get paths for genomic files (default)')
    parser.add_argument('-D',
                        '--dist_h5',
                        type=str,
                        help='the distances file',
                        default=None)
    parser.add_argument(
        '-d',
        '--max_deg',
        type=float,
        default=None,
        help='max number of degenerate characters in protein sequences')
    parser.add_argument('-l',
                        '--min_len',
                        type=float,
                        default=None,
                        help='min length of sequences')
    parser.add_argument('-V',
                        '--vocab',
                        action='store_true',
                        default=False,
                        help='store sequences as vocabulary data')

    if len(sys.argv) == 1:
        parser.print_help()
        sys.exit(1)

    args = parser.parse_args(args=argv)

    if not any([args.protein, args.cds, args.genomic]):
        args.genomic = True

    logging.basicConfig(stream=sys.stdout,
                        level=logging.INFO,
                        format='%(asctime)s - %(message)s')
    logger = logging.getLogger()

    # read accessions
    logger.info('reading accessions %s' % args.accessions)
    with open(args.accessions, 'r') as f:
        taxa_ids = [l[:-1] for l in f.readlines()]

    # get paths to Fasta Files
    fa_path_func = get_genomic_path
    if args.cds:
        fa_path_func = get_fna_path
    elif args.protein:
        fa_path_func = get_faa_path
    fapaths = [fa_path_func(acc, args.fadir) for acc in taxa_ids]

    di_kwargs = dict()
    # if a distance matrix file has been given, read and select relevant distances
    if args.dist_h5:
        #############################
        # read and filter distances
        #############################
        logger.info('reading distances from %s' % args.dist_h5)
        with h5py.File(args.dist_h5, 'r') as f:
            dist = f['distances'][:]
            dist_taxa = f['leaf_names'][:].astype('U')
        logger.info('selecting distances for taxa found in %s' %
                    args.accessions)
        dist = select_distances(taxa_ids, dist_taxa, dist)
        dist = CondensedDistanceMatrix('distances', data=dist)
        di_kwargs['distances'] = dist

    #############################
    # read and filter taxonomies
    #############################
    logger.info('reading taxonomies from %s' % args.metadata)
    taxlevels = [
        'domain', 'phylum', 'class', 'order', 'family', 'genus', 'species'
    ]

    def func(row):
        dat = dict(zip(taxlevels, row['gtdb_taxonomy'].split(';')))
        dat['species'] = dat['species'].split(' ')[1]
        dat['gtdb_genome_representative'] = row['gtdb_genome_representative'][
            3:]
        dat['accession'] = row['accession'][3:]
        return pd.Series(data=dat)

    logger.info('selecting GTDB taxonomy for taxa found in %s' %
                args.accessions)
    taxdf = pd.read_csv(args.metadata, header=0, sep='\t')[['accession', 'gtdb_taxonomy', 'gtdb_genome_representative']]\
                        .apply(func, axis=1)\
                        .set_index('accession')\
                        .filter(items=taxa_ids, axis=0)

    #############################
    # read and filter embeddings
    #############################
    emb = None
    if args.emb is not None:
        logger.info('reading embeddings from %s' % args.emb)
        with h5py.File(args.emb, 'r') as f:
            emb = f['embedding'][:]
            emb_taxa = f['leaf_names'][:]
        logger.info('selecting embeddings for taxa found in %s' %
                    args.accessions)
        emb = select_embeddings(taxa_ids, emb_taxa, emb)

    #############################
    # read and trim tree
    #############################
    logger.info('reading tree from %s' % args.tree)
    root = TreeNode.read(args.tree, format='newick')

    logger.info('transforming leaf names for shearing')
    for tip in root.tips():
        tip.name = tip.name[3:].replace(' ', '_')

    logger.info('shearing taxa not found in %s' % args.accessions)
    rep_ids = taxdf['gtdb_genome_representative'].values
    root = root.shear(rep_ids)

    logger.info('converting tree to Newick string')
    bytes_io = io.BytesIO()
    root.write(bytes_io, format='newick')
    tree_str = bytes_io.getvalue()
    tree = NewickString('tree', data=tree_str)

    if di_kwargs.get('distances') is None:
        from scipy.spatial.distance import squareform
        tt_dmat = root.tip_tip_distances()
        if (rep_ids != taxa_ids).any():
            tt_dmat = get_nonrep_matrix(taxa_ids, rep_ids, tt_dmat)
        dmat = tt_dmat.data
        di_kwargs['distances'] = CondensedDistanceMatrix('distances',
                                                         data=dmat)

    h5path = args.out

    logger.info("reading %d Fasta files" % len(fapaths))
    logger.info("Total size: %d", sum(os.path.getsize(f) for f in fapaths))

    if args.vocab:
        if args.protein:
            SeqTable = SequenceTable
            seqit = AAVocabIterator(fapaths,
                                    logger=logger,
                                    min_seq_len=args.min_len)
        else:
            SeqTable = DNATable
            if args.cds:
                logger.info("reading and writing CDS sequences")
                seqit = DNAVocabGeneIterator(fapaths,
                                             logger=logger,
                                             min_seq_len=args.min_len)
            else:
                seqit = DNAVocabIterator(fapaths,
                                         logger=logger,
                                         min_seq_len=args.min_len)
    else:
        if args.protein:
            logger.info("reading and writing protein sequences")
            seqit = AASeqIterator(fapaths,
                                  logger=logger,
                                  max_degenerate=args.max_deg,
                                  min_seq_len=args.min_len)
            SeqTable = AATable
        else:
            logger.info("reading and writing DNA sequences")
            seqit = DNASeqIterator(fapaths,
                                   logger=logger,
                                   min_seq_len=args.min_len)
            SeqTable = DNATable

    seqit_bsize = 2**25
    if args.protein:
        seqit_bsize = 2**15
    elif args.cds:
        seqit_bsize = 2**18

    # set up DataChunkIterators
    packed = DataChunkIterator.from_iterable(iter(seqit),
                                             maxshape=(None, ),
                                             buffer_size=seqit_bsize,
                                             dtype=np.dtype('uint8'))
    seqindex = DataChunkIterator.from_iterable(seqit.index_iter,
                                               maxshape=(None, ),
                                               buffer_size=2**0,
                                               dtype=np.dtype('int'))
    names = DataChunkIterator.from_iterable(seqit.names_iter,
                                            maxshape=(None, ),
                                            buffer_size=2**0,
                                            dtype=np.dtype('U'))
    ids = DataChunkIterator.from_iterable(seqit.id_iter,
                                          maxshape=(None, ),
                                          buffer_size=2**0,
                                          dtype=np.dtype('int'))
    taxa = DataChunkIterator.from_iterable(seqit.taxon_iter,
                                           maxshape=(None, ),
                                           buffer_size=2**0,
                                           dtype=np.dtype('uint16'))
    seqlens = DataChunkIterator.from_iterable(seqit.seqlens_iter,
                                              maxshape=(None, ),
                                              buffer_size=2**0,
                                              dtype=np.dtype('uint32'))

    io = get_hdf5io(h5path, 'w')

    tt_args = ['taxa_table', 'a table for storing taxa data', taxa_ids]
    tt_kwargs = dict()
    for t in taxlevels[1:]:
        tt_args.append(taxdf[t].values)
    if emb is not None:
        tt_kwargs['embedding'] = emb
    tt_kwargs['rep_taxon_id'] = rep_ids

    taxa_table = TaxaTable(*tt_args, **tt_kwargs)

    seq_table = SeqTable(
        'seq_table',
        'a table storing sequences for computing sequence embedding',
        io.set_dataio(names, compression='gzip', chunks=(2**15, )),
        io.set_dataio(packed,
                      compression='gzip',
                      maxshape=(None, ),
                      chunks=(2**15, )),
        io.set_dataio(seqindex,
                      compression='gzip',
                      maxshape=(None, ),
                      chunks=(2**15, )),
        io.set_dataio(seqlens,
                      compression='gzip',
                      maxshape=(None, ),
                      chunks=(2**15, )),
        io.set_dataio(taxa,
                      compression='gzip',
                      maxshape=(None, ),
                      chunks=(2**15, )),
        taxon_table=taxa_table,
        id=io.set_dataio(ids,
                         compression='gzip',
                         maxshape=(None, ),
                         chunks=(2**15, )))

    difile = DeepIndexFile(seq_table, taxa_table, tree, **di_kwargs)

    io.write(difile, exhaust_dci=False)
    io.close()

    logger.info("reading %s" % (h5path))
    h5size = os.path.getsize(h5path)
    logger.info("HDF5 size: %d", h5size)
Beispiel #14
0
parser.add_argument('-y', '--heigth', type=int, default=1080, help='heigth')
parser.add_argument('-f', '--fps', type=int, default=20, help='fps')
parser.add_argument('-b', '--bps', type=int, default=1200000, help='bitrate')
parser.add_argument('-o',
                    '--output_file',
                    default='video.h264',
                    help='video.h264')
parser.add_argument(
    '-l',
    '--log_level',
    metavar='log_level',
    default='WARN',
    choices=['DEBUG', 'INFO', 'WARN', 'ERROR'],
    help='file containing the configuration for autobot istance')

args = parser.parse_args()

camera = ActionPiCamera(args.width, args.heigth, args.fps, args.output_file)
io = ActionPiIO(camera, args.gpio)
api = ActionPiAPI(camera, args.host, args.port, True)
watchdog = ActionPiWhatchdog()

watchdog.watch()
io.start_monitoring()
api.serve()

#Stopping all services

api.close()
io.close()
watchdog.stop()