def unmap(self, uids, idtype): idtype = ascii(idtype) def lookup(id): key = self.to_backward_key(idtype, id) return self._db.get(key, None) return list(map(lookup, uids))
def load(self, idtype, mapping): """ resets and loads the given mapping :param idtype: :param mapping: array of tuples (id, uid) :return: """ idtype = ascii(idtype) # delete cache del self._cache[idtype] self._db.execute('DELETE FROM mapping WHERE idtype=?', (idtype,)) self._db.executemany('insert or ignore into mapping values ("' + idtype + '",?,?)', mapping) self._db.commit() self._cache[idtype] = {id: uid for id, uid in mapping}
def convert_field(self, value, conversion): """ do any conversion on the resulting object """ if conversion is None: # pylint: disable=no-else-return return value elif conversion == 's': return str(value) elif conversion == 'r': return repr(value) elif conversion == 'a': return ascii(value) elif conversion == 'A': return _ascii2(value) raise ValueError( "Unknown conversion specifier {0!s}".format(conversion))
def __call__(self, ids, idtype): """ return the integer index ids for the given ids in the given idtype """ idtype = ascii(idtype) max_old = -1 if idtype not in self._db else int(self._db[idtype]) r = [] for id in ids: key = self.to_forward_key(idtype, id) if key in self._db: r.append(int(self._db[key])) else: i = max_old + 1 _log.debug('create %s %d', key, i) max_old += 1 self._db[key] = str(i) self._db[self.to_backward_key(idtype, i)] = str(id).encode('ascii', 'ignore') r.append(i) self._db[idtype] = str(max_old) return r
def load(self, idtype, mapping): """ resets and loads the given mapping :param idtype: :param mapping: array of tuples (id, uid) :return: """ idtype = ascii(idtype) # assuming incremental ids if idtype in self._db: # clear old data for key in self._db.keys(): if key.startswith(idtype + '2id.') or key.startswith('id2' + idtype + '.'): del self._db[key] max_uid = None for id, uid in mapping: key = self.to_forward_key(idtype, id) max_uid = uid if max_uid is None else max(uid, max_uid) self._db[key] = str(uid) self._db[self.to_backward_key(idtype, uid)] = str(id).encode('ascii', 'ignore')
def write_to_read_and_location_dbs_txn(report_db, fin, read_group_id, curr_read_group, records_per_txn, fp_put_read_group, fp_append_location_db): """ Write a 'records_per_txn' number of reads and optionally locations to DB. Args: read_group_id (int): The current (or starting) read_group_id. curr_read_group (ReadGroup): Any leftover read_groups from previous txn are passed along here. records_per_txn (int): The number of records fp_put_read_group (function): Function to add a read_group to the read_group database. fp_append_location_db (function): Function to append a read_group_id to the genomic location database. Returns: (bool, int, [str]): 3-tuple with elements a) done or not, b) current read_group_id, c) current unwritten/incomplete read_group. """ record_count = 0 total_count = 0 read = None remaining_lines = True # Used to keep keys ordered. read_group_id_str = ascii(read_group_id).zfill(READ_GROUP_ID_DIGITS) if len(curr_read_group) == 0: # First txn read_group = ReadGroup() prev_read = None else: # Leftover read_group not written in last txn. read_group = curr_read_group prev_read = curr_read_group[-1] while remaining_lines and record_count < records_per_txn: while True: # Assumption: average len(read_group) << records_per_txn # Collect all reads with same RNAME total_count += 1 line = fin.readline().rstrip() if not line: read = None remaining_lines = False break if line[0] == '@': # skip header lines continue read = Read(line) if prev_read and prev_read.qname != read.qname: # New read name found, write to store current read_group. break else: read_group.append(read) prev_read = read if prev_read: # Writing to store read_name = read_group[0].qname.split(DELIM_ANNO)[0] fp_put_read_group(read_group_id_str, read_group) fp_append_location_db(read_group_id_str, read_group) report_db[LOG_NUM_READ_GROUPS] += 1 # Prep for next round read_group = ReadGroup([read]) if read else ReadGroup() prev_read = read record_count += 1 read_group_id += 1 read_group_id_str = ascii(read_group_id).zfill(READ_GROUP_ID_DIGITS) if not remaining_lines: break if not remaining_lines and len(read_group) > 0: # Get the very last read_group of the file fp_put_read_group(read_group_id_str, read_group) fp_append_location_db(read_group_id_str, read_group) report_db[LOG_NUM_READ_GROUPS] += 1 done = not remaining_lines return (done, read_group_id, read_group)
import unicodedata from collections import defaultdict, Counter __all__ = [ "pprint", "pformat", "isreadable", "isrecursive", "saferepr", "PrettyPrinter", ] PY3 = sys.version_info >= (3, 0, 0) BytesType = bytes TextType = str if PY3 else unicode u_prefix = '' if PY3 else 'u' if PY3: # Import builins explicitly to keep Py2 static analyzers happy import builtins chr_to_ascii = lambda x: builtins.ascii(x)[1:-1] unichr = chr else: chr_to_ascii = lambda x: repr(x)[2:-1] class TextIO(io.TextIOWrapper): def __init__(self, encoding=None): io.TextIOWrapper.__init__(self, io.BytesIO(), encoding=encoding) def getvalue(self): self.flush() return self.buffer.getvalue().decode(self.encoding) # pprintpp will make an attempt to print as many Unicode characters as is # safely possible. It will use the character category along with this table to
def _ascii2(value): """ A variant of the `ascii()` built-in function known from Python 3 that: (1) ensures ASCII-only output, and (2) produces a nicer formatting for use in exception and warning messages and other human consumption. This function calls `ascii()` and post-processes its output as follows: * For unicode strings, a leading 'u' is stripped (u'xxx' becomes 'xxx'), if present. * For byte strings, a leading 'b' is stripped (b'xxx' becomes 'xxx'), if present. * For unicode strings, non-ASCII Unicode characters in the range U+0000 to U+00FF are represented as '/u00hh' instead of the confusing '/xhh' ('/' being a backslash, 'hh' being a 2-digit hex number). This function correctly handles values of collection types such as list, tuple, dict, and set, by producing the usual Python representation string for them. If the type is not the standard Python type (i.e. OrderedDict instead of dict), the type name is also shown in the result. Returns: str: ASCII string """ if isinstance(value, Mapping): # NocaseDict in current impl. is not a Mapping; it uses # its own repr() implementation (via ascii(), called further down) items = [ _ascii2(k) + ": " + _ascii2(v) for k, v in six.iteritems(value) ] item_str = "{" + ", ".join(items) + "}" if value.__class__.__name__ == 'dict': return item_str return "{0}({1})".format(value.__class__.__name__, item_str) if isinstance(value, Set): items = [_ascii2(v) for v in value] item_str = "{" + ", ".join(items) + "}" if value.__class__.__name__ == 'set': return item_str return "{0}({1})".format(value.__class__.__name__, item_str) if isinstance(value, MutableSequence): items = [_ascii2(v) for v in value] item_str = "[" + ", ".join(items) + "]" if value.__class__.__name__ == 'list': return item_str return "{0}({1})".format(value.__class__.__name__, item_str) if isinstance(value, Sequence) and \ not isinstance(value, (six.text_type, six.binary_type)): items = [_ascii2(v) for v in value] if len(items) == 1: item_str = "(" + ", ".join(items) + ",)" else: item_str = "(" + ", ".join(items) + ")" if value.__class__.__name__ == 'tuple': return item_str return "{0}({1})".format(value.__class__.__name__, item_str) if isinstance(value, six.text_type): ret = ascii(value) # returns type str in py2 and py3 if ret.startswith('u'): ret = ret[1:] # Convert /xhh into /u00hh. # The two look-behind patterns address at least some of the cases that # should not be converted: Up to 5 backslashes in repr() result are # handled correctly. The failure that happens starting with 6 # backslashes and even numbers of backslashes above that is not # dramatic: The /xhh is converted to /u00hh even though it shouldn't. ret = re.sub(r'(?<![^\\]\\)(?<![^\\]\\\\\\)\\x([0-9a-fA-F]{2})', r'\\u00\1', ret) elif isinstance(value, six.binary_type): ret = ascii(value) # returns type str in py2 and py3 if ret.startswith('b'): ret = ret[1:] elif isinstance(value, (six.integer_types, float)): # str() on Python containers calls repr() on the items. PEP 3140 # that attempted to fix that, has been rejected. See # https://www.python.org/dev/peps/pep-3140/. # We don't want to make that same mistake, and because ascii() calls # repr(), we call str() on the items explicitly. This makes a # difference for example for all pywbem.CIMInt values. ret = str(value) else: ret = ascii(value) # returns type str in py2 and py3 return ret
from implicit_lambda import args_resolver abs = functools.update_wrapper( lambda *args, **kwargs: builtins.abs(*args, **kwargs), builtins.abs) abs._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.abs)(*args, **kwargs), builtins.abs) all = functools.update_wrapper( lambda *args, **kwargs: builtins.all(*args, **kwargs), builtins.all) all._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.all)(*args, **kwargs), builtins.all) any = functools.update_wrapper( lambda *args, **kwargs: builtins.any(*args, **kwargs), builtins.any) any._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.any)(*args, **kwargs), builtins.any) ascii = functools.update_wrapper( lambda *args, **kwargs: builtins.ascii(*args, **kwargs), builtins.ascii) ascii._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.ascii)(*args, **kwargs), builtins.ascii) bin = functools.update_wrapper( lambda *args, **kwargs: builtins.bin(*args, **kwargs), builtins.bin) bin._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.bin)(*args, **kwargs), builtins.bin) bool = functools.update_wrapper( lambda *args, **kwargs: builtins.bool(*args, **kwargs), builtins.bool) bool._ = functools.update_wrapper( lambda *args, **kwargs: wrap(builtins.bool)(*args, **kwargs), builtins.bool) breakpoint = functools.update_wrapper( lambda *args, **kwargs: builtins.breakpoint(*args, **kwargs), builtins.breakpoint)
def test_Entity(): # Test the basics of creating and accessing properties on an entity for i in range(2): e = Entity(name='Test object', description='I hope this works', annotations = dict(foo=123, nerds=['chris','jen','janey'], annotations='How confusing!'), properties = dict(annotations='/repo/v1/entity/syn1234/annotations', md5='cdef636522577fc8fb2de4d95875b27c', parentId='syn1234'), concreteType='org.sagebionetworks.repo.model.Data') # Should be able to create an Entity from an Entity if i == 1: e = Entity.create(e) assert e.parentId == 'syn1234' assert e['parentId'] == 'syn1234' assert e.properties['parentId'] == 'syn1234' assert e.properties.parentId =='syn1234' assert e.foo == 123 assert e['foo'] == 123 assert e.annotations['foo'] == 123 assert e.annotations.foo == 123 assert hasattr(e, 'parentId') assert hasattr(e, 'foo') assert not hasattr(e, 'qwerqwer') # Annotations is a bit funny, because there is a property call # 'annotations', which will be masked by a member of the object # called 'annotations'. Because annotations are open-ended, we # might even have an annotations called 'annotations', which gets # really confusing. assert isinstance(e.annotations, collections.Mapping) assert isinstance(e['annotations'], collections.Mapping) assert e.properties['annotations'] == '/repo/v1/entity/syn1234/annotations' assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations' assert e.annotations.annotations == 'How confusing!' assert e.annotations['annotations'] == 'How confusing!' assert e.nerds == ['chris','jen','janey'] assert all([ k in e for k in ['name', 'description', 'foo', 'nerds', 'annotations', 'md5', 'parentId']]) # Test modifying properties e.description = 'Working, so far' assert e['description'] == 'Working, so far' e['description'] = 'Wiz-bang flapdoodle' assert e.description == 'Wiz-bang flapdoodle' # Test modifying annotations e.foo = 999 assert e.annotations['foo'] == 999 e['foo'] = 12345 assert e.annotations.foo == 12345 # Test creating a new annotation e['bar'] = 888 assert e.annotations['bar'] == 888 e['bat'] = 7788 assert e.annotations['bat'] == 7788 # Test replacing annotations object e.annotations = {'splat':'a totally new set of annotations', 'foo':456} assert e.foo == 456 assert e['foo'] == 456 assert isinstance(e.annotations, collections.Mapping) assert isinstance(e['annotations'], collections.Mapping) assert e.annotations.foo == 456 assert e.properties['annotations'] == '/repo/v1/entity/syn1234/annotations' assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations' ## test unicode properties e.train = '時刻表には記載されない 月への列車が来ると聞いて' e.band = "Motörhead" e.lunch = "すし" if hasattr(sys.stdout, 'encoding') and sys.stdout.encoding and sys.stdout.encoding.lower() == 'utf-8': print(e) else: print(ascii(e))
def to_backward_key(idtype, id): return ascii('id2' + idtype + '.' + str(id))
def to_forward_key(idtype, identifier): return ascii(idtype + '2id.' + str(identifier))
class dummy_class(object): __repr__ = object() OrderedDict = defaultdict = Counter = dummy_class _test_has_collections = False PY3 = sys.version_info >= (3, 0, 0) BytesType = bytes TextType = str if PY3 else unicode u_prefix = '' if PY3 else 'u' if PY3: # Import builins explicitly to keep Py2 static analyzers happy import builtins chr_to_ascii = lambda x: builtins.ascii(x)[1:-1] unichr = chr from .safesort import safesort _iteritems = lambda x: x.items() else: chr_to_ascii = lambda x: repr(x)[2:-1] safesort = sorted _iteritems = lambda x: x.iteritems() def _sorted_py2(iterable): with warnings.catch_warnings(): if getattr(sys, "py3kwarning", False): warnings.filterwarnings("ignore", "comparing unequal types " "not supported", DeprecationWarning) return sorted(iterable)
def test_Entity(): # Test the basics of creating and accessing properties on an entity for i in range(2): e = Entity(name='Test object', description='I hope this works', annotations=dict(foo=123, nerds=['chris', 'jen', 'janey'], annotations='How confusing!'), properties=dict( annotations='/repo/v1/entity/syn1234/annotations', md5='cdef636522577fc8fb2de4d95875b27c', parentId='syn1234'), concreteType='org.sagebionetworks.repo.model.Data') # Should be able to create an Entity from an Entity if i == 1: e = Entity.create(e) assert e.parentId == 'syn1234' assert e['parentId'] == 'syn1234' assert e.properties['parentId'] == 'syn1234' assert e.properties.parentId == 'syn1234' assert e.foo == 123 assert e['foo'] == 123 assert e.annotations['foo'] == 123 assert e.annotations.foo == 123 assert hasattr(e, 'parentId') assert hasattr(e, 'foo') assert not hasattr(e, 'qwerqwer') # Annotations is a bit funny, because there is a property call # 'annotations', which will be masked by a member of the object # called 'annotations'. Because annotations are open-ended, we # might even have an annotations called 'annotations', which gets # really confusing. assert isinstance(e.annotations, collections.Mapping) assert isinstance(e['annotations'], collections.Mapping) assert e.properties[ 'annotations'] == '/repo/v1/entity/syn1234/annotations' assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations' assert e.annotations.annotations == 'How confusing!' assert e.annotations['annotations'] == 'How confusing!' assert e.nerds == ['chris', 'jen', 'janey'] assert all([ k in e for k in [ 'name', 'description', 'foo', 'nerds', 'annotations', 'md5', 'parentId' ] ]) # Test modifying properties e.description = 'Working, so far' assert e['description'] == 'Working, so far' e['description'] = 'Wiz-bang flapdoodle' assert e.description == 'Wiz-bang flapdoodle' # Test modifying annotations e.foo = 999 assert e.annotations['foo'] == 999 e['foo'] = 12345 assert e.annotations.foo == 12345 # Test creating a new annotation e['bar'] = 888 assert e.annotations['bar'] == 888 e['bat'] = 7788 assert e.annotations['bat'] == 7788 # Test replacing annotations object e.annotations = { 'splat': 'a totally new set of annotations', 'foo': 456 } assert e.foo == 456 assert e['foo'] == 456 assert isinstance(e.annotations, collections.Mapping) assert isinstance(e['annotations'], collections.Mapping) assert e.annotations.foo == 456 assert e.properties[ 'annotations'] == '/repo/v1/entity/syn1234/annotations' assert e.properties.annotations == '/repo/v1/entity/syn1234/annotations' ## test unicode properties e.train = '時刻表には記載されない 月への列車が来ると聞いて' e.band = "Motörhead" e.lunch = "すし" if hasattr( sys.stdout, 'encoding' ) and sys.stdout.encoding and sys.stdout.encoding.lower() == 'utf-8': print(e) else: print(ascii(e))
def ascii(x=None): if x is None: return bpipe(ascii) else: return builtins.ascii(x)