def unmap(self, uids, idtype):
idtype = ascii(idtype)
def lookup(id):
key = self.to_backward_key(idtype, id)
return self._db.get(key, None)
return list(map(lookup, uids))
def load(self, idtype, mapping):
"""
resets and loads the given mapping
:param idtype:
:param mapping: array of tuples (id, uid)
:return:
"""
idtype = ascii(idtype)
# delete cache
del self._cache[idtype]
self._db.execute('DELETE FROM mapping WHERE idtype=?', (idtype,))
self._db.executemany('insert or ignore into mapping values ("' + idtype + '",?,?)', mapping)
self._db.commit()
self._cache[idtype] = {id: uid for id, uid in mapping}
def convert_field(self, value, conversion):
"""
do any conversion on the resulting object
"""
if conversion is None: # pylint: disable=no-else-return
return value
elif conversion == 's':
return str(value)
elif conversion == 'r':
return repr(value)
elif conversion == 'a':
return ascii(value)
elif conversion == 'A':
return _ascii2(value)
raise ValueError(
"Unknown conversion specifier {0!s}".format(conversion))
def __call__(self, ids, idtype):
"""
return the integer index ids for the given ids in the given idtype
"""
idtype = ascii(idtype)
max_old = -1 if idtype not in self._db else int(self._db[idtype])
r = []
for id in ids:
key = self.to_forward_key(idtype, id)
if key in self._db:
r.append(int(self._db[key]))
else:
i = max_old + 1
_log.debug('create %s %d', key, i)
max_old += 1
self._db[key] = str(i)
self._db[self.to_backward_key(idtype, i)] = str(id).encode('ascii', 'ignore')
r.append(i)
self._db[idtype] = str(max_old)
return r
def load(self, idtype, mapping):
"""
resets and loads the given mapping
:param idtype:
:param mapping: array of tuples (id, uid)
:return:
"""
idtype = ascii(idtype)
# assuming incremental ids
if idtype in self._db:
# clear old data
for key in self._db.keys():
if key.startswith(idtype + '2id.') or key.startswith('id2' + idtype + '.'):
del self._db[key]
max_uid = None
for id, uid in mapping:
key = self.to_forward_key(idtype, id)
max_uid = uid if max_uid is None else max(uid, max_uid)
self._db[key] = str(uid)
self._db[self.to_backward_key(idtype, uid)] = str(id).encode('ascii', 'ignore')
def write_to_read_and_location_dbs_txn(report_db, fin, read_group_id,
curr_read_group, records_per_txn,
fp_put_read_group,
fp_append_location_db):
"""
Write a 'records_per_txn' number of reads and optionally locations to DB.
Args:
read_group_id (int): The current (or starting) read_group_id.
curr_read_group (ReadGroup): Any leftover read_groups from previous txn are
passed along here.
records_per_txn (int): The number of records
fp_put_read_group (function): Function to add a read_group to the
read_group database.
fp_append_location_db (function): Function to append a read_group_id to the
genomic location database.
Returns:
(bool, int, [str]): 3-tuple with elements a) done or not, b) current
read_group_id, c) current unwritten/incomplete read_group.
"""
record_count = 0
total_count = 0
read = None
remaining_lines = True
# Used to keep keys ordered.
read_group_id_str = ascii(read_group_id).zfill(READ_GROUP_ID_DIGITS)
if len(curr_read_group) == 0:
# First txn
read_group = ReadGroup()
prev_read = None
else:
# Leftover read_group not written in last txn.
read_group = curr_read_group
prev_read = curr_read_group[-1]
while remaining_lines and record_count < records_per_txn:
while True: # Assumption: average len(read_group) << records_per_txn
# Collect all reads with same RNAME
total_count += 1
line = fin.readline().rstrip()
if not line:
read = None
remaining_lines = False
break
if line[0] == '@':
# skip header lines
continue
read = Read(line)
if prev_read and prev_read.qname != read.qname:
# New read name found, write to store current read_group.
break
else:
read_group.append(read)
prev_read = read
if prev_read:
# Writing to store
read_name = read_group[0].qname.split(DELIM_ANNO)[0]
fp_put_read_group(read_group_id_str, read_group)
fp_append_location_db(read_group_id_str, read_group)
report_db[LOG_NUM_READ_GROUPS] += 1
# Prep for next round
read_group = ReadGroup([read]) if read else ReadGroup()
prev_read = read
record_count += 1
read_group_id += 1
read_group_id_str = ascii(read_group_id).zfill(READ_GROUP_ID_DIGITS)
if not remaining_lines:
break
if not remaining_lines and len(read_group) > 0:
# Get the very last read_group of the file
fp_put_read_group(read_group_id_str, read_group)
fp_append_location_db(read_group_id_str, read_group)
report_db[LOG_NUM_READ_GROUPS] += 1
done = not remaining_lines
return (done, read_group_id, read_group)
import unicodedata
from collections import defaultdict, Counter
__all__ = [
"pprint", "pformat", "isreadable", "isrecursive", "saferepr",
"PrettyPrinter",
]
PY3 = sys.version_info >= (3, 0, 0)
BytesType = bytes
TextType = str if PY3 else unicode
u_prefix = '' if PY3 else 'u'
if PY3:
# Import builins explicitly to keep Py2 static analyzers happy
import builtins
chr_to_ascii = lambda x: builtins.ascii(x)[1:-1]
unichr = chr
else:
chr_to_ascii = lambda x: repr(x)[2:-1]
class TextIO(io.TextIOWrapper):
def __init__(self, encoding=None):
io.TextIOWrapper.__init__(self, io.BytesIO(), encoding=encoding)
def getvalue(self):
self.flush()
return self.buffer.getvalue().decode(self.encoding)
# pprintpp will make an attempt to print as many Unicode characters as is
# safely possible. It will use the character category along with this table to
def _ascii2(value):
"""
A variant of the `ascii()` built-in function known from Python 3 that:
(1) ensures ASCII-only output, and
(2) produces a nicer formatting for use in exception and warning messages
and other human consumption.
This function calls `ascii()` and post-processes its output as follows:
* For unicode strings, a leading 'u' is stripped (u'xxx' becomes 'xxx'),
if present.
* For byte strings, a leading 'b' is stripped (b'xxx' becomes 'xxx'),
if present.
* For unicode strings, non-ASCII Unicode characters in the range U+0000 to
U+00FF are represented as '/u00hh' instead of the confusing '/xhh'
('/' being a backslash, 'hh' being a 2-digit hex number).
This function correctly handles values of collection types such as list,
tuple, dict, and set, by producing the usual Python representation string
for them. If the type is not the standard Python type (i.e. OrderedDict
instead of dict), the type name is also shown in the result.
Returns:
str: ASCII string
"""
if isinstance(value, Mapping):
# NocaseDict in current impl. is not a Mapping; it uses
# its own repr() implementation (via ascii(), called further down)
items = [
_ascii2(k) + ": " + _ascii2(v) for k, v in six.iteritems(value)
]
item_str = "{" + ", ".join(items) + "}"
if value.__class__.__name__ == 'dict':
return item_str
return "{0}({1})".format(value.__class__.__name__, item_str)
if isinstance(value, Set):
items = [_ascii2(v) for v in value]
item_str = "{" + ", ".join(items) + "}"
if value.__class__.__name__ == 'set':
return item_str
return "{0}({1})".format(value.__class__.__name__, item_str)
if isinstance(value, MutableSequence):
items = [_ascii2(v) for v in value]
item_str = "[" + ", ".join(items) + "]"
if value.__class__.__name__ == 'list':
return item_str
return "{0}({1})".format(value.__class__.__name__, item_str)
if isinstance(value, Sequence) and \
not isinstance(value, (six.text_type, six.binary_type)):
items = [_ascii2(v) for v in value]
if len(items) == 1:
item_str = "(" + ", ".join(items) + ",)"
else:
item_str = "(" + ", ".join(items) + ")"
if value.__class__.__name__ == 'tuple':
return item_str
return "{0}({1})".format(value.__class__.__name__, item_str)
if isinstance(value, six.text_type):
ret = ascii(value) # returns type str in py2 and py3
if ret.startswith('u'):
ret = ret[1:]
# Convert /xhh into /u00hh.
# The two look-behind patterns address at least some of the cases that
# should not be converted: Up to 5 backslashes in repr() result are
# handled correctly. The failure that happens starting with 6
# backslashes and even numbers of backslashes above that is not
# dramatic: The /xhh is converted to /u00hh even though it shouldn't.
ret = re.sub(r'(?<![^\\]\\)(?<![^\\]\\\\\\)\\x([0-9a-fA-F]{2})',
r'\\u00\1', ret)
elif isinstance(value, six.binary_type):
ret = ascii(value) # returns type str in py2 and py3
if ret.startswith('b'):
ret = ret[1:]
elif isinstance(value, (six.integer_types, float)):
# str() on Python containers calls repr() on the items. PEP 3140
# that attempted to fix that, has been rejected. See
# https://www.python.org/dev/peps/pep-3140/.
# We don't want to make that same mistake, and because ascii() calls
# repr(), we call str() on the items explicitly. This makes a
# difference for example for all pywbem.CIMInt values.
ret = str(value)
else:
ret = ascii(value) # returns type str in py2 and py3
return ret
from implicit_lambda import args_resolver
abs = functools.update_wrapper(
lambda *args, **kwargs: builtins.abs(*args, **kwargs), builtins.abs)
abs._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.abs)(*args, **kwargs), builtins.abs)
all = functools.update_wrapper(
lambda *args, **kwargs: builtins.all(*args, **kwargs), builtins.all)
all._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.all)(*args, **kwargs), builtins.all)
any = functools.update_wrapper(
lambda *args, **kwargs: builtins.any(*args, **kwargs), builtins.any)
any._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.any)(*args, **kwargs), builtins.any)
ascii = functools.update_wrapper(
lambda *args, **kwargs: builtins.ascii(*args, **kwargs), builtins.ascii)
ascii._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.ascii)(*args, **kwargs),
builtins.ascii)
bin = functools.update_wrapper(
lambda *args, **kwargs: builtins.bin(*args, **kwargs), builtins.bin)
bin._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.bin)(*args, **kwargs), builtins.bin)
bool = functools.update_wrapper(
lambda *args, **kwargs: builtins.bool(*args, **kwargs), builtins.bool)
bool._ = functools.update_wrapper(
lambda *args, **kwargs: wrap(builtins.bool)(*args, **kwargs),
builtins.bool)
breakpoint = functools.update_wrapper(
lambda *args, **kwargs: builtins.breakpoint(*args, **kwargs),
builtins.breakpoint)
def test_Entity():
# Test the basics of creating and accessing properties on an entity
for i in range(2):
e = Entity(name='Test object', description='I hope this works',
annotations = dict(foo=123, nerds=['chris','jen','janey'], annotations='How confusing!'),
properties = dict(annotations='/repo/v1/entity/syn1234/annotations',
md5='cdef636522577fc8fb2de4d95875b27c',
parentId='syn1234'),
concreteType='org.sagebionetworks.repo.model.Data')
# Should be able to create an Entity from an Entity
if i == 1:
e = Entity.create(e)
assert e.parentId == 'syn1234'
assert e['parentId'] == 'syn1234'
assert e.properties['parentId'] == 'syn1234'
assert e.properties.parentId =='syn1234'
assert e.foo == 123
assert e['foo'] == 123
assert e.annotations['foo'] == 123
assert e.annotations.foo == 123
assert hasattr(e, 'parentId')
assert hasattr(e, 'foo')
assert not hasattr(e, 'qwerqwer')
# Annotations is a bit funny, because there is a property call
# 'annotations', which will be masked by a member of the object
# called 'annotations'. Because annotations are open-ended, we
# might even have an annotations called 'annotations', which gets
# really confusing.
assert isinstance(e.annotations, collections.Mapping)
assert isinstance(e['annotations'], collections.Mapping)
assert e.properties['annotations'] == '/repo/v1/entity/syn1234/annotations'
assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations'
assert e.annotations.annotations == 'How confusing!'
assert e.annotations['annotations'] == 'How confusing!'
assert e.nerds == ['chris','jen','janey']
assert all([ k in e for k in ['name', 'description', 'foo', 'nerds', 'annotations', 'md5', 'parentId']])
# Test modifying properties
e.description = 'Working, so far'
assert e['description'] == 'Working, so far'
e['description'] = 'Wiz-bang flapdoodle'
assert e.description == 'Wiz-bang flapdoodle'
# Test modifying annotations
e.foo = 999
assert e.annotations['foo'] == 999
e['foo'] = 12345
assert e.annotations.foo == 12345
# Test creating a new annotation
e['bar'] = 888
assert e.annotations['bar'] == 888
e['bat'] = 7788
assert e.annotations['bat'] == 7788
# Test replacing annotations object
e.annotations = {'splat':'a totally new set of annotations', 'foo':456}
assert e.foo == 456
assert e['foo'] == 456
assert isinstance(e.annotations, collections.Mapping)
assert isinstance(e['annotations'], collections.Mapping)
assert e.annotations.foo == 456
assert e.properties['annotations'] == '/repo/v1/entity/syn1234/annotations'
assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations'
## test unicode properties
e.train = '時刻表には記載されない 月への列車が来ると聞いて'
e.band = "Motörhead"
e.lunch = "すし"
if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding and sys.stdout.encoding.lower() == 'utf-8':
print(e)
else:
print(ascii(e))
def to_backward_key(idtype, id):
return ascii('id2' + idtype + '.' + str(id))
def to_forward_key(idtype, identifier): return ascii(idtype + '2id.' + str(identifier))
class dummy_class(object):
__repr__ = object()
OrderedDict = defaultdict = Counter = dummy_class
_test_has_collections = False
PY3 = sys.version_info >= (3, 0, 0)
BytesType = bytes
TextType = str if PY3 else unicode
u_prefix = '' if PY3 else 'u'
if PY3:
# Import builins explicitly to keep Py2 static analyzers happy
import builtins
chr_to_ascii = lambda x: builtins.ascii(x)[1:-1]
unichr = chr
from .safesort import safesort
_iteritems = lambda x: x.items()
else:
chr_to_ascii = lambda x: repr(x)[2:-1]
safesort = sorted
_iteritems = lambda x: x.iteritems()
def _sorted_py2(iterable):
with warnings.catch_warnings():
if getattr(sys, "py3kwarning", False):
warnings.filterwarnings("ignore", "comparing unequal types "
"not supported", DeprecationWarning)
return sorted(iterable)
def test_Entity():
# Test the basics of creating and accessing properties on an entity
for i in range(2):
e = Entity(name='Test object',
description='I hope this works',
annotations=dict(foo=123,
nerds=['chris', 'jen', 'janey'],
annotations='How confusing!'),
properties=dict(
annotations='/repo/v1/entity/syn1234/annotations',
md5='cdef636522577fc8fb2de4d95875b27c',
parentId='syn1234'),
concreteType='org.sagebionetworks.repo.model.Data')
# Should be able to create an Entity from an Entity
if i == 1:
e = Entity.create(e)
assert e.parentId == 'syn1234'
assert e['parentId'] == 'syn1234'
assert e.properties['parentId'] == 'syn1234'
assert e.properties.parentId == 'syn1234'
assert e.foo == 123
assert e['foo'] == 123
assert e.annotations['foo'] == 123
assert e.annotations.foo == 123
assert hasattr(e, 'parentId')
assert hasattr(e, 'foo')
assert not hasattr(e, 'qwerqwer')
# Annotations is a bit funny, because there is a property call
# 'annotations', which will be masked by a member of the object
# called 'annotations'. Because annotations are open-ended, we
# might even have an annotations called 'annotations', which gets
# really confusing.
assert isinstance(e.annotations, collections.Mapping)
assert isinstance(e['annotations'], collections.Mapping)
assert e.properties[
'annotations'] == '/repo/v1/entity/syn1234/annotations'
assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations'
assert e.annotations.annotations == 'How confusing!'
assert e.annotations['annotations'] == 'How confusing!'
assert e.nerds == ['chris', 'jen', 'janey']
assert all([
k in e for k in [
'name', 'description', 'foo', 'nerds', 'annotations', 'md5',
'parentId'
]
])
# Test modifying properties
e.description = 'Working, so far'
assert e['description'] == 'Working, so far'
e['description'] = 'Wiz-bang flapdoodle'
assert e.description == 'Wiz-bang flapdoodle'
# Test modifying annotations
e.foo = 999
assert e.annotations['foo'] == 999
e['foo'] = 12345
assert e.annotations.foo == 12345
# Test creating a new annotation
e['bar'] = 888
assert e.annotations['bar'] == 888
e['bat'] = 7788
assert e.annotations['bat'] == 7788
# Test replacing annotations object
e.annotations = {
'splat': 'a totally new set of annotations',
'foo': 456
}
assert e.foo == 456
assert e['foo'] == 456
assert isinstance(e.annotations, collections.Mapping)
assert isinstance(e['annotations'], collections.Mapping)
assert e.annotations.foo == 456
assert e.properties[
'annotations'] == '/repo/v1/entity/syn1234/annotations'
assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations'
## test unicode properties
e.train = '時刻表には記載されない 月への列車が来ると聞いて'
e.band = "Motörhead"
e.lunch = "すし"
if hasattr(
sys.stdout, 'encoding'
) and sys.stdout.encoding and sys.stdout.encoding.lower() == 'utf-8':
print(e)
else:
print(ascii(e))
def ascii(x=None):
if x is None:
return bpipe(ascii)
else:
return builtins.ascii(x)
