def test_namedbidict_raises_on_invalid_name(names):
""":func:`bidict.namedbidict` should raise if given invalid names."""
typename, keyname, valname = names
try:
namedbidict(typename, keyname, valname)
except ValueError:
# Either one of the names was invalid, or the keyname and valname were not distinct.
assert not all(map(st.NAMEDBIDICT_VALID_NAME_PAT.match, names)) or keyname == valname
def test_namedbidict_raises_on_invalid_name(names):
""":func:`bidict.namedbidict` should raise if given invalid names."""
typename, keyname, valname = names
try:
namedbidict(typename, keyname, valname)
except ValueError:
# Either one of the names was invalid, or the keyname and valname were not distinct.
assert not all(map(st.NAMEDBIDICT_VALID_NAME_PAT.match,
names)) or keyname == valname
def _namedbidict_types(draw,
names=NAMEDBIDICT_3_NAMES,
base_types=BIDICT_TYPES):
typename, keyname, valname = draw(names)
assume(keyname != valname)
base_type = draw(base_types)
return namedbidict(typename, keyname, valname, base_type=base_type)
def test_namedbidict(base_type, init_items, data):
"""Test the :func:`bidict.namedbidict` factory and custom accessors."""
names = typename, keyname, valname = [data.draw(H_NAMES) for _ in range(3)]
try:
nbcls = namedbidict(typename, keyname, valname, base_type=base_type)
except ValueError:
# Either one of the names was invalid, or the keyname and valname were not distinct.
assert not all(map(NAMEDBIDICT_VALID_NAME.match,
names)) or keyname == valname
return
except TypeError:
# The base type must not have been a BidirectionalMapping.
assert not issubclass(base_type, BidirectionalMapping)
return
assume(init_items)
instance = nbcls(init_items)
valfor = getattr(instance, valname + '_for')
keyfor = getattr(instance, keyname + '_for')
assert all(valfor[key] == val for (key, val) in iteritems(instance))
assert all(keyfor[val] == key for (key, val) in iteritems(instance))
# The same custom accessors should work on the inverse.
inv = instance.inv
valfor = getattr(inv, valname + '_for')
keyfor = getattr(inv, keyname + '_for')
assert all(valfor[key] == val for (key, val) in iteritems(instance))
assert all(keyfor[val] == key for (key, val) in iteritems(instance))
def reset(self):
"""Helper method for emptying all data contained in the RequestObject.
This RequestObject is now equivalent to a newly constructed one.
"""
for x in self.__dict__.keys():
setattr(self, x, None)
self.Paths = []
self.Keys = []
self.jobCount = 1
self.JobBounds = [None, None]
self.NamesOrder = []
self.ColumnMap = bidict.namedbidict('biMap', 'columns', 'names')({})
def reset(self):
"""Helper method for emptying all data contained in the RequestObject.
This RequestObject is now equivalent to a newly constructed one.
"""
for x in self.__dict__.keys():
setattr(self, x, None)
self.Paths = []
self.Keys = []
self.jobCount = 1;
self.JobBounds = [None,None]
self.NamesOrder = []
self.ColumnMap = bidict.namedbidict('biMap', 'columns', 'names')({})
def __init__(self):
"""
Basic constructor for initializing the RequestObject.
"""
self.Simple = None
self.URL = None
self.Paths = []
self.Keys = []
self.uniqueID = None
self.sortedBy = None
self.sortedByType = None
self.sortedByOBIEE = None
self.jobCount = 1
self.JobBounds = [None, None]
self.NamesOrder = []
self.ColumnMap = bidict.namedbidict('biMap', 'columns', 'names')({})
def __init__(self):
"""
Basic constructor for initializing the RequestObject.
"""
self.Simple = None
self.URL = None
self.Paths = []
self.Keys = []
self.uniqueID = None
self.sortedBy = None
self.sortedByType = None
self.sortedByOBIEE = None
self.jobCount = 1
self.JobBounds = [None, None]
self.NamesOrder = []
self.ColumnMap = bidict.namedbidict('biMap', 'columns', 'names')({})
def callLoadAllData(self, only_player: bool = False):
"""
从文件中读取所有信息。
"""
d = self.__storage.load(f'{self.__groupid}_alldata')
for k, v in d['playerdict'].items():
self.__playerdict[int(k)] = player()
self.__playerdict[int(k)].alldata = v
IDAndAlias = namedbidict('IDAndAlias', 'ID', 'Alias')
self.__IDAndAliasDict = IDAndAlias()
for k, v in self.__playerdict.items():
self.__IDAndAliasDict[k] = v.alias
if not only_player:
self.__boss.alldata = d['boss']
scorebuff = literal_eval(d['scorebuff'])
for k, v in scorebuff.items():
player.set_scorebuff(k, v)
self.__is_battle_active = d['is_battle_active']
self.__is_current_boss = d['is_current_boss']
self.__totaldays = d['totaldays']
self.__daypassed = d['daypassed']
self.__session = d['session']
self.__lastattack = d['lastattack']
self.__solutions = literal_eval(d['solutions'])
from pymongo.mongo_client import MongoClient
def logmethod(func):
name = func.__name__
def decorator(self, *args, **kwargs):
debug('>> %s %s %s', name, args, kwargs)
result = func(self, *args, **kwargs)
debug('<< %s %s', name, result)
return result
return decorator
OIDCache = namedbidict('OIDCache', 'oids', 'ints')
oid_cache = OIDCache()
grid_cache = {}
def int2oid(num):
return oid_cache.ints[num]
def oid2int(oid):
if not oid in oid_cache.oids:
oid_cache.oids[oid] = len(oid_cache) + 2
return oid_cache.oids[oid]
# coding=utf-8
# Conversions to/from HTML and LaTeX for Latin1 and Latin2 text entities.
# You need to run // sudo pip install bidict // to get this facility.
from bidict import namedbidict
import re
import sys
HTMLEntities = namedbidict('HTMLEntities', 'html_lookup', 'latex_lookup')
entities = HTMLEntities({
"amp": "\\&",
"uogon": "\\k{u}",
"Uogon": "\\k{U}",
"uring": "\\r{u}",
"Uring": "\\r{U}",
"utilde": "\\~{u}",
"Utilde": "\\~{U}",
"wcirc": "\\^{w}",
"Wcirc": "\\^{W}",
"ycirc": "\\^{y}",
"Ycirc": "\\^{Y}",
"Yuml": "\\\"{Y}",
"zacute": "\\'{z}",
"Zacute": "\\'{Z}",
"zcaron": "\\v{z}",
"Zcaron": "\\v{Z}",
"zdot": "\\.{z}",
"Zdot": "\\.{Z}",
"ncaron": "\\v{n}",
"Ncaron": "\\v{N}",
# -*- coding: utf-8 -*-
# Copyright 2009-2019 Joshua Bronson. All Rights Reserved.
#
# This Source Code Form is subject to the terms of the Mozilla Public
# License, v. 2.0. If a copy of the MPL was not distributed with this
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""Types for Hypothoses tests."""
from collections import OrderedDict
from bidict import bidict, OrderedBidict, frozenbidict, FrozenOrderedBidict, namedbidict
from bidict.compat import ItemsView, KeysView, Mapping
MyNamedBidict = namedbidict('MyNamedBidict', 'key', 'val')
MyNamedFrozenBidict = namedbidict('MyNamedFrozenBidict', 'key', 'val', base_type=frozenbidict)
MyNamedOrderedBidict = namedbidict('MyNamedOrderedBidict', 'key', 'val', base_type=OrderedBidict)
MUTABLE_BIDICT_TYPES = (bidict, OrderedBidict, MyNamedBidict)
FROZEN_BIDICT_TYPES = (frozenbidict, FrozenOrderedBidict, MyNamedFrozenBidict)
ORDERED_BIDICT_TYPES = (OrderedBidict, FrozenOrderedBidict, MyNamedOrderedBidict)
BIDICT_TYPES = tuple(set(MUTABLE_BIDICT_TYPES + FROZEN_BIDICT_TYPES + ORDERED_BIDICT_TYPES))
class _FrozenDict(KeysView, Mapping):
def __init__(self, *args, **kw): # pylint: disable=super-init-not-called
self._mapping = dict(*args, **kw)
def __getitem__(self, key):
return self._mapping[key]
Base classes for modular pipeline interface, with a simple graph engine.
Classes 'Unit' and 'Pipeline' implement the generic backend API, specified in
'base.py' and equip the backend with a simple graph engine. These classes
should be used with backends, which don't have their own dependency resolution
and graph management systems. Otherwise, one should implement 'GenericUnit' and
'GenericPipeline' directly.
"""
from earlpipeline.backends.base import GenericUnit, GenericPipeline, Edge, Parameter
from abc import ABCMeta, abstractmethod
from bidict import namedbidict
import inspect
from earlpipeline import tools
UnitMap = namedbidict('UnitMap', 'by_name', 'by_instance')
# TODO: implement optional ports
class Port(object):
"""
A descriptor class, which is used to abstract away the communication-layer
machinery from the inner implementation of the 'Unit'. Ports are accessed
via this descriptor (by instance) from the inside of the unit's
implementation, and by name (using 'read_port(name)' method) from the
outside.
Parameters
----------
name : str
name of the port, to be used from the outside
"""
class _DictSubcls(dict):
pass
class _OrderedBidictSubcls(OrderedBidict):
pass
# pylint: disable=C0103
items = [('a', 1), ('b', 2)] # use int values so makes sense with Counter
itemsreversed = list(reversed(items))
bidict_of_items = bidict(items)
frozenbidict_of_items = frozenbidict(items)
namedbidict_of_items = namedbidict('named', 'keys', 'vals')(items)
orderedbidict_of_items = OrderedBidict(items)
orderedbidict_of_itemsreversed = OrderedBidict(itemsreversed)
orderedbidictsubcls_of_items = _OrderedBidictSubcls(items)
orderedbidictsubcls_of_itemsreversed = _OrderedBidictSubcls(itemsreversed)
frozenorderedbidict_of_items = FrozenOrderedBidict(items)
frozenorderedbidict_of_itemsreversed = FrozenOrderedBidict(itemsreversed)
bidicts = (
bidict_of_items,
frozenbidict_of_items,
namedbidict_of_items,
orderedbidict_of_items,
orderedbidict_of_itemsreversed,
orderedbidictsubcls_of_items,
orderedbidictsubcls_of_itemsreversed,
frozenorderedbidict_of_items,
import pytest
from collections import Counter, OrderedDict, defaultdict
from bidict import bidict, frozenbidict, namedbidict
from itertools import product
d = dict(H='hydrogen', He='helium')
c = Counter(d)
o = OrderedDict(d)
dd = defaultdict(int, d)
class dictsubclass(dict): pass
s = dictsubclass(d)
b = bidict(d)
f = frozenbidict(d)
n = namedbidict('named', 'keys', 'vals')(d)
dicts = (d, c, o, dd, s)
bidicts = (b, f, n)
@pytest.mark.parametrize('d, b', product(dicts, bidicts))
def test_eq(d, b):
assert d == b
import collections
import logging
from bidict import namedbidict
import braintree
from .compat import getcallargs
logger = logging.getLogger(__name__)
IDMap = namedbidict('IDMap', 'fake_id', 'real_id')
def ensure_state_is_init(f):
def wrapper(*args, **kwargs):
# There's not currently a place to provide global init for the state dict,
# each action needs to ensure subitems are initialized.
named_args = getcallargs(f, *args, **kwargs)
state = named_args['state']
if 'id_maps' not in state:
state['id_maps'] = collections.defaultdict(IDMap)
if 'last_fake_ids' not in state:
state['last_fake_ids'] = {}
return f(*args, **kwargs)
return wrapper
import collections
import logging
from bidict import namedbidict
import braintree
from .compat import getcallargs
logger = logging.getLogger(__name__)
IDMap = namedbidict('IDMap', 'fake_id', 'real_id')
def ensure_state_is_init(f):
def wrapper(*args, **kwargs):
# There's not currently a place to provide global init for the state dict,
# each action needs to ensure subitems are initialized.
named_args = getcallargs(f, *args, **kwargs)
state = named_args['state']
if 'id_maps' not in state:
state['id_maps'] = collections.defaultdict(IDMap)
if 'last_fake_ids' not in state:
state['last_fake_ids'] = {}
return f(*args, **kwargs)
return wrapper
def clear_old_creation_ids(state, call_params, options):
def _namedbidict_types(draw, names=NAMEDBIDICT_3_NAMES, base_types=BIDICT_TYPES):
typename, keyname, valname = draw(names)
assume(keyname != valname)
base_type = draw(base_types)
return namedbidict(typename, keyname, valname, base_type=base_type)
def test_namedbidict_raises_on_invalid_base_type(names, invalid_base_type):
""":func:`bidict.namedbidict` should raise if given a non-bidict base_type."""
with pytest.raises(TypeError):
namedbidict(*names, base_type=invalid_base_type)
def test_namedbidict_raises_on_invalid_name(names):
""":func:`bidict.namedbidict` should raise if given invalid names."""
typename, keyname, valname = names
with pytest.raises(ValueError):
namedbidict(typename, keyname, valname)
class UserBidictNotOwnInverse(bidict):
"""Custom bidict subclass that is not its own inverse."""
_fwdm_cls = dict
_invm_cls = UserDict
UserBidictNotOwnInverseInv = UserBidictNotOwnInverse._inv_cls
assert UserBidictNotOwnInverseInv is not UserBidictNotOwnInverse
class UserBidictNotOwnInverse2(UserBidictNotOwnInverse):
"""Another custom bidict subclass that is not its own inverse."""
NamedBidict = namedbidict('NamedBidict', 'key', 'val', base_type=bidict)
NamedFrozenBidict = namedbidict('NamedFrozenBidict',
'key',
'val',
base_type=frozenbidict)
NamedOrderedBidict = namedbidict('NamedOrderedBidict',
'key',
'val',
base_type=OrderedBidict)
NamedUserBidict = namedbidict('NamedUserBidict',
'key',
'val',
base_type=UserBidict)
NAMED_BIDICT_TYPES = (NamedBidict, NamedFrozenBidict, NamedOrderedBidict,
NamedUserBidict)
# pylint: disable=C0111
def to_inv_odict(items):
"""Return an OrderedDict with the inverse of each item in ``items``."""
return OrderedDict((v, k) for (k, v) in items)
def dedup(items):
"""Given some generated items, prune any with duplicated keys or values."""
pruned = list(iteritems(to_inv_odict(iteritems(to_inv_odict(items)))))
assume(len(pruned) >= len(items) // 2)
return pruned
# pylint: disable=C0103
MyNamedBidict = namedbidict('MyNamedBidict', 'key', 'val')
ondupbehaviors = (IGNORE, OVERWRITE, RAISE)
mutable_bidict_types = (bidict, OrderedBidict, MyNamedBidict)
immutable_bidict_types = (frozenbidict, FrozenOrderedBidict)
bidict_types = mutable_bidict_types + immutable_bidict_types
mutating_methods_by_arity = {
0: ('clear', 'popitem'),
1: ('__delitem__', 'pop', 'setdefault', 'move_to_end'),
2: ('__setitem__', 'pop', 'put', 'forceput', 'setdefault'),
-1: ('update', 'forceupdate'),
}
immutable = integers()
itemlists = lists(tuples(immutable, immutable))
inititems = itemlists.map(dedup)
MUTABLE_BIDICTS = _bidict_strat(MUTABLE_BIDICT_TYPES)
ORDERED_BIDICTS = _bidict_strat(ORDERED_BIDICT_TYPES)
_ALPHABET = [chr(i) for i in range(0x10ffff) if chr(i).isidentifier()]
_NAMEDBI_VALID_NAMES = st.text(_ALPHABET, min_size=1)
IS_VALID_NAME = str.isidentifier
NAMEDBIDICT_NAMES_ALL_VALID = st.lists(_NAMEDBI_VALID_NAMES,
min_size=3,
max_size=3,
unique=True)
NAMEDBIDICT_NAMES_SOME_INVALID = st.lists(
st.text(min_size=1), min_size=3,
max_size=3).filter(lambda i: not all(IS_VALID_NAME(name) for name in i))
NAMEDBIDICT_TYPES = st.tuples(
NAMEDBIDICT_NAMES_ALL_VALID,
BIDICT_TYPES).map(lambda i: namedbidict(*i[0], base_type=i[1]))
NAMEDBIDICTS = _bidict_strat(NAMEDBIDICT_TYPES)
def _bi_and_map(bi_types, map_types, init_items=L_PAIRS_NODUP):
return st.tuples(bi_types, map_types,
init_items).map(lambda i: (i[0](i[2]), i[1](i[2])))
BI_AND_MAP_FROM_SAME_ITEMS = _bi_and_map(BIDICT_TYPES, MAPPING_TYPES)
OBI_AND_OD_FROM_SAME_ITEMS = _bi_and_map(ORDERED_BIDICT_TYPES,
st.just(OrderedDict))
OBI_AND_OMAP_FROM_SAME_ITEMS = _bi_and_map(ORDERED_BIDICT_TYPES,
ORDERED_MAPPING_TYPES)
HBI_AND_HMAP_FROM_SAME_ITEMS = _bi_and_map(FROZEN_BIDICT_TYPES,
HASHABLE_MAPPING_TYPES)
def test_namedbidict_raises_on_invalid_base_type(names, invalid_base_type):
""":func:`bidict.namedbidict` should raise if given a non-bidict base_type."""
with pytest.raises(TypeError):
namedbidict(*names, base_type=invalid_base_type)
def __init__(self, botid: int, groupid: int, type: str = 'tw'):
self.__botid = botid
self.__groupid = groupid
# 处理boss信息
self.__boss = boss()
# 处理所有玩家信息
self.__playerdict = {}
# 玩家名称与ID对应字典
IDAndAlias = namedbidict('IDAndAlias', 'ID', 'Alias')
self.__IDAndAliasDict = IDAndAlias()
self.reservedname = 'UNDEFINED'
self.__storage = storage.getStorageObj('pcr')
self.__is_battle_active = False
self.__is_current_boss = False
self.__totaldays = 0
self.__daypassed = 0
self.__session = ''
self.__solutions = {
1: {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
},
2: {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
},
3: {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
},
4: {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
},
5: {
0: [],
1: [],
2: [],
3: [],
4: [],
5: []
},
}
# 已出刀的信息
self.__lastattack = []
# 已出刀数据格式
# {
# 'name': self.reservedname,
# 'boss': 0,
# 'stage': 0,
# 'dmg': 0,
# # 记录上一刀状态，实时刀或是漏报刀，0为实时1为漏报
# 'type': 0,
# 'playerstate': 0,
# 'time': 2020-01-01 00:00:00
# }
# 锁
self.__lock = {
'status': False,
'player': self.reservedname,
}
# 计时器
self.__timer = Timer()
self.__timestamp = []
# 初始化数据
if type == 'cn':
maxhp = {
1: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
2: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
3: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
4: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
stagechange = {1: 1, 2: 2, 3: 100, 4: 1000, 5: 10000}
scorebuff = {
1: {
1: 1,
2: 1,
3: 1.1,
4: 1.1,
5: 1.2,
},
2: {
1: 1.2,
2: 1.2,
3: 1.5,
4: 1.7,
5: 2,
},
3: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
4: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
elif type == 'tw':
maxhp = {
1: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
2: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
3: {
1: 7000000,
2: 9000000,
3: 13000000,
4: 15000000,
5: 20000000
},
4: {
1: 15000000,
2: 16000000,
3: 18000000,
4: 19000000,
5: 20000000
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
stagechange = {1: 1, 2: 4, 3: 11, 4: 35, 5: 10000}
scorebuff = {
1: {
1: 1.2,
2: 1.2,
3: 1.3,
4: 1.4,
5: 1.5
},
2: {
1: 1.6,
2: 1.6,
3: 1.8,
4: 1.9,
5: 2
},
3: {
1: 2,
2: 2,
3: 2.4,
4: 2.4,
5: 2.6
},
4: {
1: 3.5,
2: 3.5,
3: 3.7,
4: 3.8,
5: 4
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
elif type == 'jp':
maxhp = {
1: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
2: {
1: 6000000,
2: 8000000,
3: 10000000,
4: 12000000,
5: 15000000
},
3: {
1: 7000000,
2: 9000000,
3: 13000000,
4: 15000000,
5: 20000000
},
4: {
1: 15000000,
2: 16000000,
3: 18000000,
4: 19000000,
5: 20000000
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
stagechange = {1: 1, 2: 4, 3: 11, 4: 35, 5: 10000}
scorebuff = {
1: {
1: 1.2,
2: 1.2,
3: 1.3,
4: 1.4,
5: 1.5
},
2: {
1: 1.6,
2: 1.6,
3: 1.8,
4: 1.9,
5: 2
},
3: {
1: 2,
2: 2,
3: 2.4,
4: 2.4,
5: 2.6
},
4: {
1: 3.5,
2: 3.5,
3: 3.7,
4: 3.8,
5: 4
},
5: {
1: 0,
2: 0,
3: 0,
4: 0,
5: 0
},
}
else:
raise ValueError('wrong clan type')
for k, v in maxhp.items():
self.__boss.setMAXHP(k, v)
self.__boss.setStageChange(stagechange)
for k, v in scorebuff.items():
player.set_scorebuff(k, v)
# coding=utf-8
# Conversions to/from HTML and LaTeX for Latin1 and Latin2 text entities.
# You need to run // sudo pip install bidict // to get this facility.
from bidict import namedbidict
import re
import sys
HTMLEntities = namedbidict('HTMLEntities', 'html_lookup', 'latex_lookup')
entities = HTMLEntities({
"amp": "\\&",
"uogon": "\\k{u}",
"Uogon": "\\k{U}",
"uring": "\\r{u}",
"Uring": "\\r{U}",
"utilde": "\\~{u}",
"Utilde": "\\~{U}",
"wcirc": "\\^{w}",
"Wcirc": "\\^{W}",
"ycirc": "\\^{y}",
"Ycirc": "\\^{Y}",
"Yuml": "\\\"{Y}",
"zacute": "\\'{z}",
"Zacute": "\\'{Z}",
"zcaron": "\\v{z}",
"Zcaron": "\\v{Z}",
"zdot": "\\.{z}",
"Zdot": "\\.{Z}",
"ncaron": "\\v{n}",
"Ncaron": "\\v{N}",
def test_namedbidict_raises_on_same_keyname_as_valname(names):
""":func:`bidict.namedbidict` should raise if given same keyname as valname."""
typename, keyname, _ = names
with pytest.raises(ValueError):
namedbidict(typename, keyname, keyname)
import pytest
from collections import Counter, OrderedDict, defaultdict
from bidict import bidict, frozenbidict, namedbidict
from itertools import product
class dictsubclass(dict):
pass
d = dict(H='hydrogen', He='helium')
c = Counter(d)
o = OrderedDict(d)
dd = defaultdict(int, d)
s = dictsubclass(d)
b = bidict(d)
f = frozenbidict(d)
n = namedbidict('named', 'keys', 'vals')(d)
dicts = (d, c, o, dd, s)
bidicts = (b, f, n)
@pytest.mark.parametrize('d, b', product(dicts, bidicts))
def test_eq(d, b):
assert d == b
BIDICTS = _bidict_strat(BIDICT_TYPES)
FROZEN_BIDICTS = _bidict_strat(FROZEN_BIDICT_TYPES)
MUTABLE_BIDICTS = _bidict_strat(MUTABLE_BIDICT_TYPES)
ORDERED_BIDICTS = _bidict_strat(ORDERED_BIDICT_TYPES)
_ALPHABET = [chr(i) for i in range(0x10ffff) if chr(i).isidentifier()]
_NAMEDBI_VALID_NAMES = st.text(_ALPHABET, min_size=1)
IS_VALID_NAME = str.isidentifier
NAMEDBIDICT_NAMES_ALL_VALID = st.lists(_NAMEDBI_VALID_NAMES, min_size=3, max_size=3, unique=True)
NAMEDBIDICT_NAMES_SOME_INVALID = st.lists(st.text(min_size=1), min_size=3, max_size=3).filter(
lambda i: not all(IS_VALID_NAME(name) for name in i)
)
NAMEDBIDICT_TYPES = st.tuples(NAMEDBIDICT_NAMES_ALL_VALID, BIDICT_TYPES).map(
lambda i: namedbidict(*i[0], base_type=i[1])
)
NAMEDBIDICTS = _bidict_strat(NAMEDBIDICT_TYPES)
def _bi_and_map(bi_types, map_types, init_items=L_PAIRS_NODUP):
return st.tuples(bi_types, map_types, init_items).map(
lambda i: (i[0](i[2]), i[1](i[2]))
)
BI_AND_MAP_FROM_SAME_ITEMS = _bi_and_map(BIDICT_TYPES, MAPPING_TYPES)
OBI_AND_OD_FROM_SAME_ITEMS = _bi_and_map(ORDERED_BIDICT_TYPES, st.just(OrderedDict))
OBI_AND_OMAP_FROM_SAME_ITEMS = _bi_and_map(ORDERED_BIDICT_TYPES, ORDERED_MAPPING_TYPES)
HBI_AND_HMAP_FROM_SAME_ITEMS = _bi_and_map(FROZEN_BIDICT_TYPES, HASHABLE_MAPPING_TYPES)
def load_data():
'''
This function loads the data as (X, y), and also creates a graph
using the PPI network data.
the global Globals object has everything this function needs to
read the data.
The output is (X, y, graph, sample_annotation, feature_annotation)
'''
print('reading PPI network...')
table = read_csv(Globals.ppi_file, True);
refseq_ids = get_column(table, 0)
refseq_ids.extend(get_column(table, 3));
refseq_ids = list(set(refseq_ids));
interactions = np.array(table)[:,[0,3]]
#dump_list(refseq_ids, 'refseq_ids.txt');
genename2entrez_raw = read_csv(Globals.genename2entrez_file, True);
genename2entrez_array = np.array(genename2entrez_raw)[:,[0,2]];
genename2entrez = {}
entrez2genename = {}
for row in genename2entrez_raw:
genename2entrez[row[0]] = row[2];
#entrez2genename[row[2]] = row[0];
expressions = read_csv(Globals.expressions_file, False);
probe2gene_raw = read_csv(Globals.probe2gene_file, False);
probe2gene_array = np.array(probe2gene_raw);
probe2gene = {}
gene2probe = {}
for row in probe2gene_raw:
probe2gene[row[0]] = row[1];
#gene2probe[row[1]] = row[0];
tmp = np.asarray(expressions);
expressions_array = tmp[1:,1:].T;
expressions_colnames = tmp[1:,0];
expressions_rownames = tmp[0,1:];
print("converting expression values to floats...");
tmp = np.empty(expressions_array.shape);
for i in range(tmp.shape[0]):
tmp[i,:] = [(np.nan if len(x.strip()) == 0 else float(x))
for x in expressions_array[i,:]]
print("replacing nans with medians...");
tmp_masked = np.ma.masked_array(tmp, [np.isnan(x) for x in tmp]);
medians = np.ma.median(tmp_masked, axis=0);
expressions_array = tmp_masked.filled(medians);
print("refactoring interactions into entrezid format...")
new_interactions = np.empty([0,2])
for i in range(interactions.shape[0]):
gene_1_list = get_genename_entrezids(interactions[i,0],
genename2entrez_array)
gene_2_list = get_genename_entrezids(interactions[i,1],
genename2entrez_array)
if gene_1_list.shape[0] > 0 and gene_2_list.shape[0] > 0:
for g1 in gene_1_list:
for g2 in gene_2_list:
new_interactions = np.append(new_interactions, [[g1, g2]], 0);
print("refactoring expressions into entrezid format...")
print("\tAlso calculating expression median for multiple mapped probes")
genes = set(genename2entrez_array[:,1])
expressions_colgenes = list()
X = np.empty([expressions_array.shape[0],0])
for entrezid in genes:
indices = get_gene_expression_indices(entrezid,
expressions_colnames,
probe2gene_array)
if (len(indices) == 0):
continue;
expressions_colgenes.append(entrezid)
new_col = np.median(expressions_array[:,indices], axis=1)
X = np.append(X, new_col.reshape([-1,1]), 1)
print("extracting common genes between expressions and network...");
usable_interaction_indices = [i for i in range(new_interactions.shape[0])
if new_interactions[i,0] in expressions_colgenes
and new_interactions[i,1] in expressions_colgenes]
common_genes = set(new_interactions[usable_interaction_indices,:].
reshape(-1)).intersection(set(expressions_colgenes))
interactions = new_interactions[usable_interaction_indices,:]
print("rearrange expressions array into X...")
common_genes_list = list(common_genes)
rearrange_indices = [expressions_colgenes.index(gene)
for gene in common_genes_list]
X = X[:,rearrange_indices]
print("creating graph from network data...");
node_indices_t = bidict.namedbidict('node_indices_t', 'entrezids', 'indices')
node_indices = node_indices_t({common_genes_list[x]:x
for x in range(len(common_genes_list))})
g = gt.Graph(directed=False);
vlist = g.add_vertex(len(node_indices))
for i in range(interactions.shape[0]):
tmp_e = g.add_edge(node_indices.entrezids[interactions[i,0]],
node_indices.entrezids[interactions[i,1]])
print("reading sample descriptions and setting Y...")
descriptions_raw = read_csv(Globals.description_file, True)
descriptions_array = np.array(descriptions_raw)
Y = np.empty([descriptions_array.shape[0]], dtype=np.int32)
Y[:] = 0
Y[[i for i in range(Y.shape[0]) if descriptions_array[i,15] == 'Good']] = 1
Y[[i for i in range(Y.shape[0]) if descriptions_array[i,15] == 'Poor']] = -1
samples = [i for i in range(Y.shape[0]) if Y[i] != 0]
Y = Y[samples]
sample_names = descriptions_array[samples,0]
expression_sample_indices = [list(expressions_rownames).index(sample_names[i])
for i in range(sample_names.shape[0])]
X = X[expression_sample_indices,:]
return (X, Y, g, descriptions_array[samples,].view(np.ndarray), np.array(common_genes_list))