def _buildMappings(self):
"""Builds two dictionaries:
self._gene2species - keys are ID numbers, values are species
self._xrefs - keys are tuples
(idnum, idtype ['geneId','protId']),
values are gene names
Also builds the set:
self._species - All species names in the xml tree"""
mapping = dict()
xref = dict()
for species in self._findSubNodes("species"):
genes = self._findSubNodes("gene", root=species)
for gene in genes:
id_ = gene.get('id')
mapping[id_] = species
for tag in gene.keys():
if tag != "id":
xref[(id_, tag)] = gene.get(tag)
self._gene2species = mapping
self._xrefs = xref
self._species = frozenset({z.get('name') for z in mapping.values()})
self._levels = frozenset({
n.get('value')
for n in self._findSubNodes("property")
if n.get('name') == "TaxRange"
})
def safeurlencode(data):
if isiterable(data):
res = urlencode(
dict((to_bytes(x), to_bytes(y)) for x, y in dict(data).items()))
else:
res = urlencode(to_bytes(data))
return res
def map(self):
"""
Maps values in the params dict to the values in the resources dict
using jinja2 variable referencing syntax
"""
logger.debug('Mapping Parameters into and simplifying resources list')
if 'resources' not in self._template:
raise ManifestError(
'Resources not in template dict',
logger=logger
)
jinja_template = Template(json.dumps(dict(self._template['resources'])))
jinja_template.environment.undefined = StrictUndefined()
resources_dict = json.loads(jinja_template.render(dict(self._vars)))
# Turn resources into a list where the key ==> local_name
for key, value in resources_dict.items():
resource = value
resource['local_name'] = key
self._resources.append(resource)
with open(os.path.join(self._working_dir, "final.json"), "w+") as fobj:
fobj.write(json.dumps(self._resources, indent=1, sort_keys=True))
def _construct_register(reg, default_reg):
"""Constructs a register dict."""
if reg:
x = dict((k, reg.get(k, d)) for k, d in default_reg.items())
else:
x = dict(default_reg)
return x
def _construct_register(reg, default_reg):
"""Constructs a register dict."""
if reg:
x = dict((k, reg.get(k, d)) for k, d in default_reg.items())
else:
x = dict(default_reg)
return x
def safeurlencode(data):
if isiterable(data):
res = urlencode(
dict((to_bytes(x), to_bytes(y)) for x, y in dict(data).items()))
else:
res = urlencode(to_bytes(data))
return res
def _buildMappings(self):
"""
Builds two dictionaries:
self._gene2species - keys are ID numbers, values are species
self._xrefs - keys are tuples (idnum, idtype ['geneId','protId']), values are gene names
Also builds the set:
self._species - All species names in the xml tree
"""
mapping = dict()
xref = dict()
for species in self.root.findall(
".//{{{ns0}}}species".format(**self.ns)):
genes = species.findall(".//{{{ns0}}}gene".format(**self.ns))
for gene in genes:
id = gene.get('id')
mapping[gene.get('id')] = species
for tag in gene.keys():
if tag != "id":
xref[(id, tag)] = gene.get(tag)
self._gene2species = mapping
self._xrefs = xref
self._species = frozenset({z.get('name') for z in mapping.values()})
self._levels = set(
n.get('value') for n in self.root.findall(
".//{{{0}}}property".format(self.ns['ns0'])))
def __init__(self, parser, species, level):
self.parser = parser
self.species = set(species)
self.level = level
self._gene2copies = dict()
for g in self.species:
self._gene2copies[g] = collections.defaultdict(list)
self._gene2fam = dict()
self._famWhereLost = collections.defaultdict(list)
def __init__(self, parser, species, level):
self.parser = parser
self.species = set(species)
self.level = level
self._gene2copies = dict()
for g in self.species:
self._gene2copies[g] = collections.defaultdict(list)
self._gene2fam = dict()
self._famWhereLost = collections.defaultdict(list)
def do_status(options, do_http=None):
if do_http is None:
from allmydata.scripts.common_http import do_http
nodedir = options["node-directory"]
with open(os.path.join(nodedir, u'private', u'api_auth_token'), 'r') as f:
token = f.read().strip()
with open(os.path.join(nodedir, u'node.url'), 'r') as f:
options['node-url'] = f.read().strip()
# do *all* our data-retrievals first in case there's an error
try:
status_data = _handle_response_for_fragment(
do_http(**_get_request_parameters_for_fragment(
options,
'status?t=json',
method='POST',
post_args=dict(
t='json',
token=token,
),
)),
options['node-url'],
)
statistics_data = _handle_response_for_fragment(
do_http(**_get_request_parameters_for_fragment(
options,
'statistics?t=json',
method='POST',
post_args=dict(
t='json',
token=token,
),
)),
options['node-url'],
)
except Exception as e:
print(u"failed to retrieve data: %s" % str(e), file=options.stderr)
return 2
downloaded_bytes = statistics_data['counters'].get('downloader.bytes_downloaded', 0)
downloaded_files = statistics_data['counters'].get('downloader.files_downloaded', 0)
uploaded_bytes = statistics_data['counters'].get('uploader.bytes_uploaded', 0)
uploaded_files = statistics_data['counters'].get('uploader.files_uploaded', 0)
print(u"Statistics (for last {}):".format(abbreviate_time(statistics_data['stats']['node.uptime'])), file=options.stdout)
print(u" uploaded {} in {} files".format(abbreviate_space(uploaded_bytes), uploaded_files), file=options.stdout)
print(u" downloaded {} in {} files".format(abbreviate_space(downloaded_bytes), downloaded_files), file=options.stdout)
print(u"", file=options.stdout)
render_active(options.stdout, status_data)
render_recent(options['verbose'], options.stdout, status_data)
# open question: should we return non-zero if there were no
# operations at all to display?
return 0
def __init__(self):
"""Create an empty calibration instance.
Normally you should use from_points or from_coeffs classmethods.
Args:
none
"""
self._calpoints = dict()
self._coeffs = dict()
def from_unittest_case(cls, unittest_class, module_suites=None):
""""Constructs a new testify.TestCase from a unittest.TestCase class.
This operates recursively on the TestCase's class hierarchy by
converting each parent unittest.TestCase into a TestifiedTestCase.
If 'suites' are provided, they are treated as module-level suites to be
applied in addition to class- and test-level suites.
"""
# our base case: once we get to the parent TestCase, replace it with our
# own parent class that will just handle inheritance for super()
if unittest_class == unittest.TestCase:
return TestifiedUnitTest
# we're going to update our class dict with some testify defaults to
# make things Just Work
unittest_dict = dict(unittest_class.__dict__)
default_test_case_dict = dict(TestCase.__dict__)
# testify.TestCase defines its own deprecated fixtures; don't let them
# overwrite unittest's fixtures
for deprecated_fixture_name in DEPRECATED_FIXTURE_TYPE_MAP:
del default_test_case_dict[deprecated_fixture_name]
# set testify defaults on the unittest class
for member_name, member in default_test_case_dict.items():
unittest_dict.setdefault(member_name, member)
# use an __init__ smart enough to figure out our inheritance
unittest_dict['__init__'] = cls.__init__
# add module-level suites in addition to any suites already on the class
class_suites = set(getattr(unittest_class, '_suites', []))
unittest_dict['_suites'] = class_suites | set(module_suites or [])
# traverse our class hierarchy and 'testify' parent unittest.TestCases
bases = []
for base_class in unittest_class.__bases__:
if issubclass(base_class, unittest.TestCase):
base_class = cls.from_unittest_case(base_class, module_suites=module_suites)
bases.append(base_class)
# include our original unittest class so existing super() calls still
# work; this is our last base class to prevent infinite recursion in
# those super calls
bases.insert(1, unittest_class)
new_name = 'Testified' + unittest_class.__name__
return MetaTestCase(new_name, tuple(bases), unittest_dict)
def list_suites(self):
"""List the suites represented by this TestRunner's tests."""
suites = defaultdict(list)
for test_instance in self.discover():
for test_method in test_instance.runnable_test_methods():
for suite_name in test_instance.suites(test_method):
suites[suite_name].append(test_method)
suite_counts = dict((suite_name, "%d tests" % len(suite_members)) for suite_name, suite_members in suites.items())
pp = pprint.PrettyPrinter(indent=2)
print(pp.pformat(dict(suite_counts)))
return suite_counts
def test_multiple_versions(self):
# if we see a mix of versions in the grid, download_best_version
# should get the latest one
self._set_versions(dict([(i,2) for i in (0,2,4,6,8)]))
d = self._fn.download_best_version()
d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[4]))
# and the checker should report problems
d.addCallback(lambda res: self._fn.check(Monitor()))
d.addCallback(self.check_bad, "test_multiple_versions")
# but if everything is at version 2, that's what we should download
d.addCallback(lambda res:
self._set_versions(dict([(i,2) for i in range(10)])))
d.addCallback(lambda res: self._fn.download_best_version())
d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2]))
# if exactly one share is at version 3, we should still get v2
d.addCallback(lambda res:
self._set_versions({0:3}))
d.addCallback(lambda res: self._fn.download_best_version())
d.addCallback(lambda res: self.failUnlessEqual(res, self.CONTENTS[2]))
# but the servermap should see the unrecoverable version. This
# depends upon the single newer share being queried early.
d.addCallback(lambda res: self._fn.get_servermap(MODE_READ))
def _check_smap(smap):
self.failUnlessEqual(len(smap.unrecoverable_versions()), 1)
newer = smap.unrecoverable_newer_versions()
self.failUnlessEqual(len(newer), 1)
verinfo, health = list(newer.items())[0]
self.failUnlessEqual(verinfo[0], 4)
self.failUnlessEqual(health, (1,3))
self.failIf(smap.needs_merge())
d.addCallback(_check_smap)
# if we have a mix of two parallel versions (s4a and s4b), we could
# recover either
d.addCallback(lambda res:
self._set_versions({0:3,2:3,4:3,6:3,8:3,
1:4,3:4,5:4,7:4,9:4}))
d.addCallback(lambda res: self._fn.get_servermap(MODE_READ))
def _check_smap_mixed(smap):
self.failUnlessEqual(len(smap.unrecoverable_versions()), 0)
newer = smap.unrecoverable_newer_versions()
self.failUnlessEqual(len(newer), 0)
self.failUnless(smap.needs_merge())
d.addCallback(_check_smap_mixed)
d.addCallback(lambda res: self._fn.download_best_version())
d.addCallback(lambda res: self.failUnless(res == self.CONTENTS[3] or
res == self.CONTENTS[4]))
return d
def analyzeGeneFam(self, fam, level):
"""analyzes a single gene family and returns a summary dict.
This method classifies all genes in the family depending on
the number of copies per genome into MULTICOPY or SINGLECOPY
genes."""
spec2genes = collections.defaultdict(set)
for geneId in fam.getMemberGenes():
spec = self.parser.mapGeneToSpecies(geneId)
spec2genes[spec].add(geneId)
summary = dict()
famId = fam.getFamId()
for spec, set_ in spec2genes.items():
if len(set_) > 1:
gclass = self.GeneClasses.MULTICOPY
else:
if famId in self.parser.singletons:
gclass = self.GeneClasses.SINGLETON
else:
gclass = self.GeneClasses.SINGLECOPY
summary[spec] = SummaryOfSpecies(self.GeneClasses.reverse[gclass],
set_)
self.addLosses(fam, summary, level)
return summary
def write(self):
print('\nFamily Analysis:')
for species in self.species:
gids = filter(
lambda gid: self.parser.mapGeneToSpecies(gid) == species,
self.parser.getGeneIds())
gids.sort(cmp=lambda x, y: len(self._gene2copies[species][x]) -
len(self._gene2copies[species][y]))
coveredFams = set(map(lambda x: self._gene2fam.get(x, None), gids))
print("{} - {} of {} sub-families covered".format(
species, len(coveredFams),
len(coveredFams) + len(self._famWhereLost[species])))
for gid in gids:
if len(self._gene2copies[species][gid]) <= 0:
print(" {}: n/a (singleton not in any family)".format(
self.parser.mapGeneToXRef(gid, self.XRefTag)))
else:
args = dict(gXref=self.parser.mapGeneToXRef(
gid, self.XRefTag),
famId=self._gene2fam[gid],
cnt=len(self._gene2copies[species][gid]),
sib=";".join([
self.parser.mapGeneToXRef(z, self.XRefTag)
for z in self._gene2copies[species][gid]
]))
print(" {gXref}: {famId} ({cnt}): {sib}".format(**args))
for fam in self._famWhereLost[species]:
print(" n/a: {} (0) no member in subfamily".format(fam))
def single_request(self, host, handler, request_body, verbose=False):
# issue XML-RPC request
try:
http_conn = self.send_request(host, handler, request_body, verbose)
resp = http_conn.getresponse()
if resp.status == 200:
self.verbose = verbose
return self.parse_response(resp)
except Fault:
raise
except Exception:
#All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
#We got an error response.
#Discard any response data and raise exception
if resp.getheader("content-length", ""):
resp.read()
raise ProtocolError(
host + handler,
resp.status, resp.reason,
dict(resp.getheaders())
)
def __init__(self, manager, aid, loginname, owner,
activated, shared, password, options):
super(Account, self).__init__(manager.pyload, owner)
self.aid = aid
self.loginname = loginname
self.owner = owner
self.activated = activated
self.shared = shared
self.password = password
self.options = options
self.manager = manager
self.lock = RLock()
self.timestamp = 0
self.login_ts = 0 # timestamp for login
self.cj = CookieJar()
self.error = None
try:
self.config_data = dict(to_configdata(x) for x in self.__config__)
except Exception as exc:
self.log_error(self._('Invalid config'))
self.pyload.log.error(exc, exc_info=self.pyload.debug)
self.config_data = {}
self.init()
def open_file(self,filename):
self.statusBar().showMessage('Loading {}'.format(os.path.basename(filename)))
t=time.time()
Tiff=tifffile.TiffFile(filename)
try:
metadata=Tiff[0].image_description
metadata = self.txt2dict(metadata)
except AttributeError:
metadata=dict()
tif=Tiff.asarray().astype(np.float64)
Tiff.close()
#tif=imread(filename,plugin='tifffile').astype(g.m.settings['internal_data_type'])
if len(tif.shape)>3: # WARNING THIS TURNS COLOR movies TO BLACK AND WHITE BY AVERAGING ACROSS THE THREE CHANNELS
tif=np.mean(tif,3)
tif=np.squeeze(tif) #this gets rid of the meaningless 4th dimention in .stk files
if len(tif.shape)==3: #this could either be a movie or a colored still frame
if tif.shape[2]==3: #this is probably a colored still frame
tif=np.mean(tif,2)
tif=np.transpose(tif,(1,0)) # This keeps the x and y the same as in FIJI.
else:
tif=np.transpose(tif,(0,2,1)) # This keeps the x and y the same as in FIJI.
elif len(tif.shape)==2: # I haven't tested whether this preserved the x y and keeps it the same as in FIJI. TEST THIS!!
tif=np.transpose(tif,(0,1))
self.statusBar().showMessage('{} successfully loaded ({} s)'.format(os.path.basename(filename), time.time()-t))
return tif
def json_check_counts(r):
d = {
"count-happiness":
r.get_happiness(),
"count-shares-good":
r.get_share_counter_good(),
"count-shares-needed":
r.get_encoding_needed(),
"count-shares-expected":
r.get_encoding_expected(),
"count-good-share-hosts":
r.get_host_counter_good_shares(),
"count-corrupt-shares":
len(r.get_corrupt_shares()),
"list-corrupt-shares": [(s.get_longname(), base32.b2a(si), shnum)
for (s, si, shnum) in r.get_corrupt_shares()],
"servers-responding":
[s.get_longname() for s in r.get_servers_responding()],
"sharemap":
dict([(shareid, sorted([s.get_longname() for s in servers]))
for (shareid, servers) in r.get_sharemap().items()]),
"count-wrong-shares":
r.get_share_counter_wrong(),
"count-recoverable-versions":
r.get_version_counter_recoverable(),
"count-unrecoverable-versions":
r.get_version_counter_unrecoverable(),
}
return d
def get_jobs(self, occ):
# load jobs with file info
if occ not in self.job_cache:
self.job_cache[occ] = dict(
(k, self.get_file_info(fid))
for k, fid in self.db.get_jobs(occ).items())
return self.job_cache[occ]
def _get_request_parameters_for_fragment(options, fragment, method, post_args):
"""
Get parameters for ``do_http`` for requesting the given fragment.
:return dict: A dictionary suitable for use as keyword arguments to
``do_http``.
"""
nodeurl = options['node-url']
if nodeurl.endswith('/'):
nodeurl = nodeurl[:-1]
url = u'%s/%s' % (nodeurl, fragment)
if method == 'POST':
if post_args is None:
raise ValueError("Must pass post_args= for POST method")
body = urlencode(post_args)
else:
body = ''
if post_args is not None:
raise ValueError("post_args= only valid for POST method")
return dict(
method=method,
url=url,
body=body.encode("utf-8"),
)
def test_replace(self):
# if we see a mix of versions in the grid, we should be able to
# replace them all with a newer version
# if exactly one share is at version 3, we should download (and
# replace) v2, and the result should be v4. Note that the index we
# give to _set_versions is different than the sequence number.
target = dict([(i,2) for i in range(10)]) # seqnum3
target[0] = 3 # seqnum4
self._set_versions(target)
def _modify(oldversion, servermap, first_time):
return oldversion + b" modified"
d = self._fn.modify(_modify)
d.addCallback(lambda res: self._fn.download_best_version())
expected = self.CONTENTS[2] + b" modified"
d.addCallback(lambda res: self.failUnlessEqual(res, expected))
# and the servermap should indicate that the outlier was replaced too
d.addCallback(lambda res: self._fn.get_servermap(MODE_CHECK))
def _check_smap(smap):
self.failUnlessEqual(smap.highest_seqnum(), 5)
self.failUnlessEqual(len(smap.unrecoverable_versions()), 0)
self.failUnlessEqual(len(smap.recoverable_versions()), 1)
d.addCallback(_check_smap)
return d
def test_happy_path_post(self, http):
http.return_value = StringIO('{"some": "json"}')
resp = _get_json_for_fragment({'node-url': 'http://localhost:1234/'},
'/fragment/',
method='POST',
post_args={'foo': 'bar'})
self.assertEqual(resp, dict(some='json'))
def from_node_config(cls, config):
"""
Create a ``StorageClientConfig`` from a complete Tahoe-LAFS node
configuration.
:param _Config config: The loaded Tahoe-LAFS node configuration.
"""
ps = config.get_config("client", "peers.preferred", "").split(",")
preferred_peers = tuple([p.strip() for p in ps if p != ""])
enabled_storage_plugins = (name.strip() for name in config.get_config(
"client",
"storage.plugins",
"",
).split(u",") if name.strip())
storage_plugins = {}
for plugin_name in enabled_storage_plugins:
try:
plugin_config = config.items("storageclient.plugins." +
plugin_name)
except NoSectionError:
plugin_config = []
storage_plugins[plugin_name] = dict(plugin_config)
return cls(
preferred_peers,
storage_plugins,
)
def __init__(self,
prefix="fluctmatch",
tbltype="Kb",
ressep=3,
datadir=path.curdir):
"""
Parameters
----------
prefix : str, optional
Filename prefix for files
tbltype : {"Kb", "b0"}, optional
Table to create (coupling strength or bond distance)
ressep : int, optional
Number of residues to exclude from interactions.
datadir : str, optional
Directory with data subdirectories
"""
self._prefix = prefix
self._tbltype = tbltype
self._ressep = ressep
self._datadir = datadir
self.table = []
self._filenames = dict(
intcor="fluct.ic",
param=".".join((self._prefix, "dist", "prm")),
)
def single_request(self, host, handler, request_body, verbose=False):
# issue XML-RPC request
try:
http_conn = self.send_request(host, handler, request_body, verbose)
resp = http_conn.getresponse()
if resp.status == 200:
self.verbose = verbose
return self.parse_response(resp)
except Fault:
raise
except Exception:
# All unexpected errors leave connection in
# a strange state, so we clear it.
self.close()
raise
# We got an error response.
# Discard any response data and raise exception
if resp.getheader("content-length", ""):
resp.read()
raise ProtocolError(
host + handler,
resp.status, resp.reason,
dict(resp.getheaders())
)
def import_foam_mesh(path, exclude=None, times_slice=None):
""" returns a Dataframe containing the raw mesh data """
mesh_loc = "constant/polyMesh"
if mesh_loc not in path:
path = os.path.join(path, mesh_loc)
fileList = find_datafiles(
path,
search="[.\/A-Za-z]*",
files=['faces', 'points', 'owner', 'neighbour'],
exclude=exclude,
times_slice=times_slice,
)
if not fileList:
print("no mesh files found")
return
p_bar = ProgressBar(n_tot=sum([len(l) for l in fileList.values()]))
df = DataFrame()
from collections import defaultdict
origins = Origins()
els = list(fileList.items())
time, files = els[0]
df_tmp = dict()
for fn in files:
ret = read_data_file(fn, skiplines=1, maxlines=False)
p_bar.next()
field_names, x, hashes = ret
df_tmp[fn] = x
return df_tmp
def collate_values(messages, timestamp_to_time=None):
"""Collate messages values using message id.
If value is None then it is skipped. Message ids are mapped to
list of tuples of the form ``(timestamp, value)``.
:param messages: an iterable that produce messages
:type messages: :class:`~vartools.common.TraceMessage` list
:return: two maps of message ids to lists of tuples and to type id
:rtype: (dict, dict)
"""
timestamp_to_time = timestamp_to_time if timestamp_to_time else lambda x: x
collated_values = defaultdict(list)
message_type_dict = dict()
for message in messages:
if message.value:
collated_values[message.message_id].append(
(timestamp_to_time(message.timestamp), message.value))
if message.message_id in message_type_dict:
if message.type_id != message_type_dict[message.message_id]:
_logger.error(
'Different type ids correspond to the one message '
'id'.format(message.type_id,
message_type_dict[message.message_id]))
else:
message_type_dict[message.message_id] = message.type_id
return collated_values, message_type_dict
def import_foam_mesh(path, exclude=None, times_slice=None):
""" returns a Dataframe containing the raw mesh data """
mesh_loc = "constant/polyMesh"
if mesh_loc not in path:
path = os.path.join(path, mesh_loc)
fileList = find_datafiles(
path,
search="[.\/A-Za-z]*",
files=['faces', 'points', 'owner', 'neighbour'],
exclude=exclude,
times_slice=times_slice,
)
if not fileList:
print("no mesh files found")
return
p_bar = ProgressBar(n_tot=sum([len(l) for l in fileList.values()]))
df = DataFrame()
from collections import defaultdict
origins = Origins()
els = list(fileList.items())
time, files = els[0]
df_tmp = dict()
for fn in files:
ret = read_data_file(fn, skiplines=1, maxlines=False)
p_bar.next()
field_names, x, hashes = ret
df_tmp[fn] = x
return df_tmp
def fine_tune_all_layers(data_dir, train_dir, train_args=None):
args = dict(
dataset_name='deepdrive',
checkpoint_path=train_dir,
dataset_split_name='train',
dataset_dir=data_dir,
model_name=MOBILENET_V2_SLIM_NAME,
train_image_size=IMG_SIZE,
max_number_of_steps=9707,
# Performance degrades by 20e3 despite eval still dropping.
# TODO: Find out why
# (perhaps https://github.com/felipecode/coiltraine will say more) -
# basically we must do early stopping before loss indicates it to get
# optimum driving performance
batch_size=16,
learning_rate=0.00004,
learning_rate_decay_type='fixed',
save_interval_secs=180,
save_summaries_secs=60,
log_every_n_steps=20,
optimizer='rmsprop',
weight_decay=0.00004)
if train_args is not None:
args.update(train_args)
slim_train_image_nn(**args)
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self._mapping["N"] = "protein and name N"
self._mapping["CB"] = dict(
ALA="name CB",
ARG="name NH*",
ASN="name OD1 ND2",
ASP="name OD*",
CYS="name SG",
GLN="name OE1 NE2",
GLU="name OE*",
HIS="name CG ND1 CD2 CE1 NE2",
HSD="name CG ND1 CD2 CE1 NE2",
HSE="name CG ND1 CD2 CE1 NE2",
HSP="name CG ND1 CD2 CE1 NE2",
ILE="name CG1 CG2 CD CD1",
LEU="name CD1 CD2",
LYS="name NZ",
MET="name SD",
PHE="name CG CD* CE* CZ",
PRO="name CG",
SER="name OG",
THR="name OG1",
TRP="name CG CD* NE CE* CZ* CH",
TYR="name CG CD* CE* CZ OH",
VAL="name CG1 CG2",
)
self._mapping["O"] = "protein and name O OT1 OT2 OXT"
self._mapping["ions"] = "bioion"
kwargs["mapping"] = self._mapping
self._initialize(*args, **kwargs)
self._set_masses()
self._set_charges()
def analyzeGeneFam(self, fam, level):
"""analyzes a single gene family and returns a summary dict.
This method classifies all genes in the family depending on
the number of copies per genome into MULTICOPY or SINGLECOPY
genes."""
spec2genes = collections.defaultdict(set)
for geneId in fam.getMemberGenes():
spec = self.parser.mapGeneToSpecies(geneId)
spec2genes[spec].add(geneId)
summary = dict()
famId = fam.getFamId()
for spec, set_ in spec2genes.items():
if len(set_) > 1:
gclass = self.GeneClasses.MULTICOPY
else:
if famId in self.parser.singletons:
gclass = self.GeneClasses.SINGLETON
else:
gclass = self.GeneClasses.SINGLECOPY
summary[spec] = SummaryOfSpecies(self.GeneClasses.reverse[gclass],
set_)
self.addLosses(fam, summary, level)
return summary
def p_enum_member(self, p):
"""enum_member : ID
| ID ASSIGN INTEGER"""
if len(p) > 2:
self.current_enum_index = p[3]
p[0] = dict({self.current_enum_index: p[1]})
self.current_enum_index += 1
def _microcanonical_average_max_cluster_size(max_cluster_size, alpha):
"""
Compute the average size of the largest cluster
Helper function for :func:`microcanonical_averages`
Parameters
----------
max_cluster_size : 1-D :py:class:`numpy.ndarray` of int
Each entry is the ``max_cluster_size`` field of the output of
:func:`sample_states`:
The size of the largest cluster (absolute number of sites).
alpha: float
Significance level.
Returns
-------
ret : dict
Largest cluster statistics
ret['max_cluster_size'] : float
Average size of the largest cluster (absolute number of sites)
ret['max_cluster_size_ci'] : 1-D :py:class:`numpy.ndarray` of float, size 2
Lower and upper bounds of the normal confidence interval of the average
size of the largest cluster (absolute number of sites)
See Also
--------
sample_states : largest cluster detection
microcanonical_averages : largest cluster statistics
"""
ret = dict()
runs = max_cluster_size.size
sqrt_n = np.sqrt(runs)
max_cluster_size_sample_mean = max_cluster_size.mean()
ret['max_cluster_size'] = max_cluster_size_sample_mean
max_cluster_size_sample_std = max_cluster_size.std(ddof=1)
if max_cluster_size_sample_std:
old_settings = np.seterr(all='raise')
ret['max_cluster_size_ci'] = scipy.stats.t.interval(
1 - alpha,
df=runs - 1,
loc=max_cluster_size_sample_mean,
scale=max_cluster_size_sample_std / sqrt_n)
np.seterr(**old_settings)
else:
ret['max_cluster_size_ci'] = (max_cluster_size_sample_mean *
np.ones(2))
return ret
def test_numberOfGenesPerSpecies_specFilter(self):
expectedCnts = dict(HUMAN=4, PANTR=4, MOUSE=4, RATNO=2,
CANFA=3, XENTR=2)
param_list = [{'HUMAN'}, {'XENTR'}, {'PANTR', 'MOUSE'}, {}]
for param in param_list:
expected = sum([expectedCnts[z] for z in param])
returned = len(self._op.getGeneIds(speciesFilter=param))
self.assertEqual(expected, returned, 'failed with {}'.format(param))
def __parse_text_fields(self, text_fields):
for herd in self.top_level.iter('herd'):
self.population.append( dict() ) #empty
for t in text_fields:
if isinstance(t, tuple):
self.__populate_text_field(herd, *t)
else:
self.__populate_text_field(herd, t)
def _got_share_hashes(sh):
sharehashes = dict(sh)
try:
self.share_hash_tree.set_hashes(sharehashes)
except IndexError as le:
raise BadOrMissingHash(le)
except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le:
raise BadOrMissingHash(le)
def init_from_parsed(self, parsed):
nodetype, d = parsed
self.writecap = to_bytes(d.get("rw_uri"))
self.readcap = to_bytes(d.get("ro_uri"))
self.mutable = d.get("mutable", False) # older nodes don't provide it
self.children_d = dict([(str(name), value)
for (name, value) in d["children"].items()])
self.children = None
def __init__(self):
"""Create an empty calibration instance.
Normally you should use from_points or from_coeffs classmethods.
Args:
none
"""
self._calpoints = dict()
self._coeffs = dict()
# initialize fit constraints?
warnings.warn(
"The use of bq.EnergyCalBase classes is deprecated "
"and will be removed in a future release; "
"use bq.Calibration instead",
DeprecationWarning,
)
def __init__(self, filename):
self.filename = util.filename(filename, ext="prm")
self._prmbuffers = dict(
ATOMS=StringIO(),
BONDS=StringIO(),
ANGLES=StringIO(),
DIHEDRALS=StringIO(),
IMPROPER=StringIO(),
)
def _get_arg(self, arg_name, default, arg_list):
return dict(
arg.split(u"=", 1)
for arg
in arg_list
).get(
arg_name,
default,
)
def _got_crypttext_hashes(hashes):
if len(hashes) < len(crypttext_hash_tree):
raise BadOrMissingHash()
ct_hashes = dict(enumerate(hashes))
try:
crypttext_hash_tree.set_hashes(ct_hashes)
except IndexError as le:
raise BadOrMissingHash(le)
except (hashtree.BadHashError, hashtree.NotEnoughHashesError) as le:
raise BadOrMissingHash(le)
def _check1(args):
(rc, out, err) = args
self.assertEqual(len(err), 0, err)
self.failUnlessReallyEqual(rc, 0)
lines = out.split("\n")
children = dict([line.split() for line in lines if line])
latest_uri = children["Latest"]
self.failUnless(latest_uri.startswith("URI:DIR2-CHK:"), latest_uri)
childnames = list(children.keys())
self.failUnlessReallyEqual(sorted(childnames), ["Archives", "Latest"])
def test_numberOfGenesPerSpecies(self):
expectedCnts = dict(HUMAN=4, PANTR=4, MOUSE=4, RATNO=2,
CANFA=3, XENTR=2)
allGenes = self._op.getGeneIds()
for species in expectedCnts.keys():
self.assertEqual(
len([gid for gid in allGenes
if self._op.mapGeneToSpecies(gid) == species]),
expectedCnts[species],
"number of genes not correct for "+species)
def copy(self):
clone = mcts.TreeNode.copy(self)
# global data (shallow-copied)
clone.model = self.model
# local data (which must be copied)
clone.domains = dict(self.domains)
clone.relaxed = list(self.relaxed)
clone.upper_bound = self.upper_bound
clone.lower_bound = self.lower_bound
return clone
def _get_moved_resources(self, changes, undo=False):
result = {}
if isinstance(changes, rope.base.change.ChangeSet):
for change in changes.changes:
result.update(self._get_moved_resources(change))
if isinstance(changes, rope.base.change.MoveResource):
result[changes.resource] = changes.new_resource
if undo:
return dict([(value, key) for key, value in result.items()])
return result
def create_type_id_dict(enums, event_format=None):
"""Create dictionary with type id descriptions.
It is assumed that some types contain event codes. This function
looks for enums that correspond to event codes and create
dictionary that maps ``type_id`` to these event codes. It assumes
that if there is a type with the name ``SomethingEvents`` then
enum with name ``Something`` contains event codes.
:param enums: list of enums processed by :func:`clean_enums`
:type enums: list of :class:`EnumList`
:param str event_format: ``struct`` format string to parse event codes
"""
# construct type dict
type_dict = dict()
for e in enums:
if e.category != TYPE_CATEGORY_ID:
continue
overlapping_keys = e.members.keys() & type_dict.keys()
if overlapping_keys:
_logger.error('Overlapping keys: {0}'.format(
', '.join(overlapping_keys)))
for type_id, desc in e.members.items():
struct_object = None
if event_format and desc.name.endswith(
CATEGORY_SUFFIX_DICT[EVENT_CATEGORY_ID]):
struct_object = struct.Struct(event_format)
type_desc = vtc.TypeDescription(
name=desc.name, comment=desc.comment,
codes=dict(), struct_object=struct_object)
type_dict[type_id] = type_desc
# fill codes
for e in enums:
if e.category != EVENT_CATEGORY_ID:
continue
type_name = e.name + CATEGORY_SUFFIX_DICT[EVENT_CATEGORY_ID]
for type_id, type_desc in type_dict.items():
if type_name != type_desc.name:
continue
type_desc.codes.update(e.members)
return type_dict
def test_xrefMapping(self):
xreftags = dict(protId='', geneId='g')
allGenes = self._op.getGeneIds()
for gid in allGenes:
species = self._op.mapGeneToSpecies(gid)
for xreftag, prefix in xreftags.items():
expectedId = "{}{}{}".format(species, prefix, int(gid) % 10)
xref = self._op.mapGeneToXRef(gid, xreftag)
self.assertEqual(xref, expectedId,
"xrefmapping failed for {}: {} vs {}"
.format(gid, xref, expectedId))
def jump_to_global(self):
if not self._check_autoimport():
return
all_names = list(self.autoimport.get_all_names())
name = self.env.ask_values('Global name: ', all_names)
result = dict(self.autoimport.get_name_locations(name))
if len(result) == 1:
resource = list(result.keys())[0]
else:
resource = self._ask_file(result.keys())
if resource:
self._goto_location(resource, result[resource])
def assert_dicts_equal(left, right, ignore_keys=None, message="expected %(left)r == %(right)r [left has:%(extra_left)r, right has:%(extra_right)r]", msg=None):
"""Assert that two dictionarys are equal (optionally ignoring certain keys)."""
if msg:
warnings.warn("msg is deprecated", DeprecationWarning)
message = msg
if ignore_keys is not None:
left = dict((k, left[k]) for k in left if k not in ignore_keys)
right = dict((k, right[k]) for k in right if k not in ignore_keys)
if left == right:
return
extra_left = _dict_subtract(left, right)
extra_right = _dict_subtract(right, left)
raise AssertionError(message % {
'left': left,
'right': right,
'extra_left': extra_left,
'extra_right': extra_right,
})
def test_filtered_grouped_genes(self):
filters = ['HUMAN', 'PANTR', 'CANFA', 'MOUSE', 'RATNO', 'XENTR']
expected_ids = dict(HUMAN={'1', '2', '3'},
PANTR={'11', '12', '13', '14'},
CANFA={'21', '22', '23'},
MOUSE={'31', '32', '33', '34'},
RATNO={'41'},
XENTR={'51', '53'})
for filter_ in filters:
result_nodes = self._grouped_query(self._op.root, filter_)
result_ids = {n.get('id') for n in result_nodes}
self.assertEqual(result_ids, expected_ids[filter_],
'failed with {}'.format(filter_))
def txt2dict(metadata):
meta=dict()
try:
metadata=json.loads(metadata.decode('utf-8'))
return metadata
except ValueError: #if the metadata isn't in JSON
pass
for line in metadata.splitlines():
line=re.split('[:=]',line)
if len(line)==1:
meta[line[0]]=''
else:
meta[line[0].lstrip().rstrip()]=line[1].lstrip().rstrip()
return meta
def __init__(self, taxonomy, histories, comparisons=None):
# member variable error-checking and setup
self.check_histories(taxonomy, histories)
if comparisons is None:
comparisons = self.generate_comparisons(histories)
self.check_comparisons(comparisons)
self.check_comparisons_and_histories(histories, comparisons)
# build taxonomy
self.hierarchy = dict()
self.histories = dict()
nodes = [TaxNode(h.analyzedLevel) for h in histories]
self.root = nodes[0].name
for i in range(len(histories) - 1):
nodes[i].add_child(nodes[i+1])
nodes[i+1].add_parent(nodes[i])
self.hierarchy[nodes[i].name] = nodes[i]
self.histories[nodes[i].name] = histories[i]
nodes[i].attach_fam_history(histories[i])
nodes[i+1].attach_level_comparison_result(comparisons[i])
nodes[-1].attach_fam_history(histories[-1])
self.hierarchy[nodes[-1].name] = nodes[-1]
self.histories[nodes[-1].name] = histories[-1]
def _buildMappings(self):
"""
Builds two dictionaries:
self._gene2species - keys are ID numbers, values are species
self._xrefs - keys are tuples (idnum, idtype ['geneId','protId']), values are gene names
Also builds the set:
self._species - All species names in the xml tree
"""
mapping=dict()
xref = dict()
for species in self.root.findall(".//{{{ns0}}}species".format(**self.ns)):
genes = species.findall(".//{{{ns0}}}gene".format(**self.ns))
for gene in genes:
id=gene.get('id')
mapping[gene.get('id')] = species
for tag in gene.keys():
if tag!="id":
xref[(id,tag)]=gene.get(tag)
self._gene2species=mapping
self._xrefs = xref
self._species = frozenset({z.get('name') for z in mapping.values()})
self._levels = set(n.get('value') for n in self.root.findall(
".//{{{0}}}property".format(self.ns['ns0'])))
def parse(self):
"""
Loads the settings(s) into the template and params dicts,
which can then be validated.
"""
logger.debug('Parsing Parameters dict.')
# Simplify the Parameters dict to just key : values
if 'vars' not in self._template:
raise ManifestError(
'Parameters not in template dict',
logger=logger
)
self._vars = self._template['vars']
if 'vars' in self._settings:
self._vars.update(dict(self._settings.vars))
# Run any of the parser plugins on the Parameters dict
parsers = self._config.get_plugins(category_name='Parser')
for parser in parsers:
self._vars = parser.run(self._vars)
outfile = os.path.join(self._working_dir, "parsed.json")
logger.debug('Writing parsed vars to %s', outfile)
with open(outfile, 'w+') as fobj:
fobj.write(json.dumps(dict(self._vars), indent=1, sort_keys=True))
# Validate that we don't have any None values in the Parameters dict.
for val in self._vars.values():
if val is None:
raise ManifestError(
'Some parameters still equal None after parsing.',
logger=logger
)
def _buildMappings(self):
"""Builds two dictionaries:
self._gene2species - keys are ID numbers, values are species
self._xrefs - keys are tuples
(idnum, idtype ['geneId','protId']),
values are gene names
Also builds the set:
self._species - All species names in the xml tree"""
mapping = dict()
xref = dict()
for species in self._findSubNodes("species"):
genes = self._findSubNodes("gene", root=species)
for gene in genes:
id_ = gene.get('id')
mapping[id_] = species
for tag in gene.keys():
if tag != "id":
xref[(id_, tag)] = gene.get(tag)
self._gene2species = mapping
self._xrefs = xref
self._species = frozenset({z.get('name') for z in mapping.values()})
self._levels = frozenset({n.get('value')
for n in self._findSubNodes("property")
if n.get('name') == "TaxRange"})
def extractHierarchy(self):
self.hierarchy = dict(zip( self.nodes, map( TaxNode, self.nodes)))
for pair in itertools.product(self.nodes, repeat = 2):
if pair in self.adj:
if self.good(pair):
first, second = pair
#print "%s,%s is good" % pair
self.hierarchy[first].addChild(self.hierarchy[second])
self.hierarchy[second].addParent(self.hierarchy[first])
noParentNodes = [z for z in self.nodes if self.hierarchy[z].up is None]
if len(noParentNodes)!=1:
raise TaxonomyInconsistencyError(
"Warning: several/none TaxonomyNodes are roots: {}"
.format(noParentNodes))
self.root = noParentNodes[0]
def create_message_id_dict(enums):
"""Extract message ids from enums and store in a dictionary.
:param enums: list of enums processed by :func:`clean_enums`
:type enums: list of :class:`EnumList`
"""
id_dict = dict()
for enum in enums:
if enum.category != MESSAGE_CATEGORY_ID:
continue
overlapping_keys = enum.members.keys() & id_dict.keys()
if overlapping_keys:
_logger.error('Overlapping keys: {0}'.format(
', '.join(overlapping_keys)))
id_dict.update(enum.members)
return id_dict
