def _getplotdata(format='png'):
import io
io = io.StringIO()
plt.savefig(io, format=format)
val = io.getvalue()
io.close()
return val
def test_close(self):
io = Interface(lambda input: None, "textbox", "label")
_, local_url, _ = io.launch(prevent_thread_lock=True)
response = requests.get(local_url)
self.assertEqual(response.status_code, 200)
io.close()
with self.assertRaises(Exception):
response = requests.get(local_url)
def _do_close(self, io, closefd):
try:
io.close()
# self.fileio already knows whether or not to close the
# file descriptor
self.fileio.close()
finally:
self._fobj = None
self.fileio = None
def _do_close(self, io, closefd):
try:
io.close()
# self.fileio already knows whether or not to close the
# file descriptor
self.fileio.close()
finally:
self._fobj = None
self.fileio = None
def test_stream_io(self):
import io
f = gzip.open("tests/saint_barthelemy.osm.gz")
io = io.BytesIO(f.read())
i1 = OsmSaxReader(io, "tests/saint_barthelemy.state.txt")
o1 = TestCountObjects()
i1.CopyTo(o1)
self.assertEquals(o1.num_nodes, 8076)
self.assertEquals(o1.num_ways, 625)
self.assertEquals(o1.num_rels, 16)
io.close()
def test_response_is_iterable(self):
r = requests.Response()
io = StringIO.StringIO('abc')
read_ = io.read
def read_mock(amt, decode_content=None):
return read_(amt)
setattr(io, 'read', read_mock)
r.raw = io
assert next(iter(r))
io.close()
def test_response_is_iterable(self):
r = requests.Response()
io = StringIO.StringIO('abc')
read_ = io.read
def read_mock(amt, decode_content=None):
return read_(amt)
setattr(io, 'read', read_mock)
r.raw = io
assert next(iter(r))
io.close()
def test_read(self):
io = StringIO()
writer = excelcsv.DictWriter(io, [u'1', u'2', u'3'])
writer.writeheader()
writer.writerow({u'1': u'a', u'2': u'b', u'3': u'c'})
writer.writerow({u'1': u'd', u'2': u'e', u'3': u'f'})
value = io.getvalue()
self.assertIsInstance(value, text_type)
self.assertEqual(value, u'1,2,3\r\na,b,c\r\nd,e,f\r\n')
io.close()
def test_create_tunnel(self):
response = requests.get(networking.GRADIO_API_SERVER)
payload = response.json()[0]
io = Interface(lambda x: x, "text", "text")
_, path_to_local_server, _ = io.launch(prevent_thread_lock=True,
share=False)
_, localhost, port = path_to_local_server.split(":")
threading.Thread.start = mock.MagicMock(return_value=None)
paramiko.SSHClient.connect = mock.MagicMock(return_value=None)
tunneling.create_tunnel(payload, localhost, port)
threading.Thread.start.assert_called_once()
paramiko.SSHClient.connect.assert_called_once()
io.close()
def test_limited_stream_disconnection():
io = BytesIO(b'A bit of content')
# disconnect detection on out of bytes
stream = wsgi.LimitedStream(io, 255)
with pytest.raises(ClientDisconnected):
stream.read()
# disconnect detection because file close
io = BytesIO(b'x' * 255)
io.close()
stream = wsgi.LimitedStream(io, 255)
with pytest.raises(ClientDisconnected):
stream.read()
def test_limited_stream_disconnection():
io = BytesIO(b'A bit of content')
# disconnect detection on out of bytes
stream = wsgi.LimitedStream(io, 255)
with pytest.raises(ClientDisconnected):
stream.read()
# disconnect detection because file close
io = BytesIO(b'x' * 255)
io.close()
stream = wsgi.LimitedStream(io, 255)
with pytest.raises(ClientDisconnected):
stream.read()
def gather_archives(mailbox_url, search_terms, destio):
"""Download the mailbox at mailbox_url, and write every message
containing one of the terms in search_terms, or a message
(transitively) in reply to one of those messages, to destio,
the output mailbox stream."""
io = cStringIO.StringIO()
fetch_https_securely(mailbox_url, io,
username=passwords.get_w3c_username(),
password=passwords.get_w3c_password())
month_message_str = io.getvalue()
io.close()
message_ids_included = set()
for message in generate_messages(month_message_str):
include = False
message_id = None
for [header_name, header_value] in generate_headers(message):
header_name = header_name.lower()
if header_name == "message-id":
message_id = header_value
elif header_name == "in-reply-to":
if not include:
if header_value in message_ids_included:
# This is a reply to a message in our set.
include = True
elif header_name == "references":
if not include:
for reference in ws_re.split(header_value):
if reference in message_ids_included:
# This is a reply to a message in our set.
include = True
break
if not include:
for term in search_terms:
if message.find(term) != -1:
include = True
break
if include:
message_ids_included.add(message_id)
destio.write(message)
def prepare_data(argv=None):
'''Aggregate sequence data GTDB using a file-of-files'''
import argparse
import io
import sys
import logging
import h5py
import pandas as pd
from skbio import TreeNode
from hdmf.common import get_hdf5io
from hdmf.data_utils import DataChunkIterator
from ..utils import get_faa_path, get_fna_path, get_genomic_path
from exabiome.sequence.convert import AASeqIterator, DNASeqIterator, DNAVocabIterator, DNAVocabGeneIterator
from exabiome.sequence.dna_table import AATable, DNATable, SequenceTable, TaxaTable, DeepIndexFile, NewickString, CondensedDistanceMatrix
parser = argparse.ArgumentParser()
parser.add_argument(
'accessions',
type=str,
help='file of the NCBI accessions of the genomes to convert')
parser.add_argument('fadir',
type=str,
help='directory with NCBI sequence files')
parser.add_argument('metadata', type=str, help='metadata file from GTDB')
parser.add_argument('tree', type=str, help='the distances file')
parser.add_argument('out', type=str, help='output HDF5')
grp = parser.add_mutually_exclusive_group()
parser.add_argument('-e',
'--emb',
type=str,
help='embedding file',
default=None)
grp.add_argument('-p',
'--protein',
action='store_true',
default=False,
help='get paths for protein files')
grp.add_argument('-c',
'--cds',
action='store_true',
default=False,
help='get paths for CDS files')
grp.add_argument('-g',
'--genomic',
action='store_true',
default=False,
help='get paths for genomic files (default)')
parser.add_argument('-D',
'--dist_h5',
type=str,
help='the distances file',
default=None)
parser.add_argument(
'-d',
'--max_deg',
type=float,
default=None,
help='max number of degenerate characters in protein sequences')
parser.add_argument('-l',
'--min_len',
type=float,
default=None,
help='min length of sequences')
parser.add_argument('-V',
'--vocab',
action='store_true',
default=False,
help='store sequences as vocabulary data')
if len(sys.argv) == 1:
parser.print_help()
sys.exit(1)
args = parser.parse_args(args=argv)
if not any([args.protein, args.cds, args.genomic]):
args.genomic = True
logging.basicConfig(stream=sys.stdout,
level=logging.INFO,
format='%(asctime)s - %(message)s')
logger = logging.getLogger()
# read accessions
logger.info('reading accessions %s' % args.accessions)
with open(args.accessions, 'r') as f:
taxa_ids = [l[:-1] for l in f.readlines()]
# get paths to Fasta Files
fa_path_func = get_genomic_path
if args.cds:
fa_path_func = get_fna_path
elif args.protein:
fa_path_func = get_faa_path
fapaths = [fa_path_func(acc, args.fadir) for acc in taxa_ids]
di_kwargs = dict()
# if a distance matrix file has been given, read and select relevant distances
if args.dist_h5:
#############################
# read and filter distances
#############################
logger.info('reading distances from %s' % args.dist_h5)
with h5py.File(args.dist_h5, 'r') as f:
dist = f['distances'][:]
dist_taxa = f['leaf_names'][:].astype('U')
logger.info('selecting distances for taxa found in %s' %
args.accessions)
dist = select_distances(taxa_ids, dist_taxa, dist)
dist = CondensedDistanceMatrix('distances', data=dist)
di_kwargs['distances'] = dist
#############################
# read and filter taxonomies
#############################
logger.info('reading taxonomies from %s' % args.metadata)
taxlevels = [
'domain', 'phylum', 'class', 'order', 'family', 'genus', 'species'
]
def func(row):
dat = dict(zip(taxlevels, row['gtdb_taxonomy'].split(';')))
dat['species'] = dat['species'].split(' ')[1]
dat['gtdb_genome_representative'] = row['gtdb_genome_representative'][
3:]
dat['accession'] = row['accession'][3:]
return pd.Series(data=dat)
logger.info('selecting GTDB taxonomy for taxa found in %s' %
args.accessions)
taxdf = pd.read_csv(args.metadata, header=0, sep='\t')[['accession', 'gtdb_taxonomy', 'gtdb_genome_representative']]\
.apply(func, axis=1)\
.set_index('accession')\
.filter(items=taxa_ids, axis=0)
#############################
# read and filter embeddings
#############################
emb = None
if args.emb is not None:
logger.info('reading embeddings from %s' % args.emb)
with h5py.File(args.emb, 'r') as f:
emb = f['embedding'][:]
emb_taxa = f['leaf_names'][:]
logger.info('selecting embeddings for taxa found in %s' %
args.accessions)
emb = select_embeddings(taxa_ids, emb_taxa, emb)
#############################
# read and trim tree
#############################
logger.info('reading tree from %s' % args.tree)
root = TreeNode.read(args.tree, format='newick')
logger.info('transforming leaf names for shearing')
for tip in root.tips():
tip.name = tip.name[3:].replace(' ', '_')
logger.info('shearing taxa not found in %s' % args.accessions)
rep_ids = taxdf['gtdb_genome_representative'].values
root = root.shear(rep_ids)
logger.info('converting tree to Newick string')
bytes_io = io.BytesIO()
root.write(bytes_io, format='newick')
tree_str = bytes_io.getvalue()
tree = NewickString('tree', data=tree_str)
if di_kwargs.get('distances') is None:
from scipy.spatial.distance import squareform
tt_dmat = root.tip_tip_distances()
if (rep_ids != taxa_ids).any():
tt_dmat = get_nonrep_matrix(taxa_ids, rep_ids, tt_dmat)
dmat = tt_dmat.data
di_kwargs['distances'] = CondensedDistanceMatrix('distances',
data=dmat)
h5path = args.out
logger.info("reading %d Fasta files" % len(fapaths))
logger.info("Total size: %d", sum(os.path.getsize(f) for f in fapaths))
if args.vocab:
if args.protein:
SeqTable = SequenceTable
seqit = AAVocabIterator(fapaths,
logger=logger,
min_seq_len=args.min_len)
else:
SeqTable = DNATable
if args.cds:
logger.info("reading and writing CDS sequences")
seqit = DNAVocabGeneIterator(fapaths,
logger=logger,
min_seq_len=args.min_len)
else:
seqit = DNAVocabIterator(fapaths,
logger=logger,
min_seq_len=args.min_len)
else:
if args.protein:
logger.info("reading and writing protein sequences")
seqit = AASeqIterator(fapaths,
logger=logger,
max_degenerate=args.max_deg,
min_seq_len=args.min_len)
SeqTable = AATable
else:
logger.info("reading and writing DNA sequences")
seqit = DNASeqIterator(fapaths,
logger=logger,
min_seq_len=args.min_len)
SeqTable = DNATable
seqit_bsize = 2**25
if args.protein:
seqit_bsize = 2**15
elif args.cds:
seqit_bsize = 2**18
# set up DataChunkIterators
packed = DataChunkIterator.from_iterable(iter(seqit),
maxshape=(None, ),
buffer_size=seqit_bsize,
dtype=np.dtype('uint8'))
seqindex = DataChunkIterator.from_iterable(seqit.index_iter,
maxshape=(None, ),
buffer_size=2**0,
dtype=np.dtype('int'))
names = DataChunkIterator.from_iterable(seqit.names_iter,
maxshape=(None, ),
buffer_size=2**0,
dtype=np.dtype('U'))
ids = DataChunkIterator.from_iterable(seqit.id_iter,
maxshape=(None, ),
buffer_size=2**0,
dtype=np.dtype('int'))
taxa = DataChunkIterator.from_iterable(seqit.taxon_iter,
maxshape=(None, ),
buffer_size=2**0,
dtype=np.dtype('uint16'))
seqlens = DataChunkIterator.from_iterable(seqit.seqlens_iter,
maxshape=(None, ),
buffer_size=2**0,
dtype=np.dtype('uint32'))
io = get_hdf5io(h5path, 'w')
tt_args = ['taxa_table', 'a table for storing taxa data', taxa_ids]
tt_kwargs = dict()
for t in taxlevels[1:]:
tt_args.append(taxdf[t].values)
if emb is not None:
tt_kwargs['embedding'] = emb
tt_kwargs['rep_taxon_id'] = rep_ids
taxa_table = TaxaTable(*tt_args, **tt_kwargs)
seq_table = SeqTable(
'seq_table',
'a table storing sequences for computing sequence embedding',
io.set_dataio(names, compression='gzip', chunks=(2**15, )),
io.set_dataio(packed,
compression='gzip',
maxshape=(None, ),
chunks=(2**15, )),
io.set_dataio(seqindex,
compression='gzip',
maxshape=(None, ),
chunks=(2**15, )),
io.set_dataio(seqlens,
compression='gzip',
maxshape=(None, ),
chunks=(2**15, )),
io.set_dataio(taxa,
compression='gzip',
maxshape=(None, ),
chunks=(2**15, )),
taxon_table=taxa_table,
id=io.set_dataio(ids,
compression='gzip',
maxshape=(None, ),
chunks=(2**15, )))
difile = DeepIndexFile(seq_table, taxa_table, tree, **di_kwargs)
io.write(difile, exhaust_dci=False)
io.close()
logger.info("reading %s" % (h5path))
h5size = os.path.getsize(h5path)
logger.info("HDF5 size: %d", h5size)
parser.add_argument('-y', '--heigth', type=int, default=1080, help='heigth')
parser.add_argument('-f', '--fps', type=int, default=20, help='fps')
parser.add_argument('-b', '--bps', type=int, default=1200000, help='bitrate')
parser.add_argument('-o',
'--output_file',
default='video.h264',
help='video.h264')
parser.add_argument(
'-l',
'--log_level',
metavar='log_level',
default='WARN',
choices=['DEBUG', 'INFO', 'WARN', 'ERROR'],
help='file containing the configuration for autobot istance')
args = parser.parse_args()
camera = ActionPiCamera(args.width, args.heigth, args.fps, args.output_file)
io = ActionPiIO(camera, args.gpio)
api = ActionPiAPI(camera, args.host, args.port, True)
watchdog = ActionPiWhatchdog()
watchdog.watch()
io.start_monitoring()
api.serve()
#Stopping all services
api.close()
io.close()
watchdog.stop()