def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
if isinstance(where, Lookup):
# If where.lhs don't refer to a field then don't bother
if not hasattr(where.lhs, 'target'):
return SOME_TREE
# Don't bother with complex right hand side either
if isinstance(where.rhs, (QuerySet, Query)):
return SOME_TREE
# Skip conditions on non-serialized fields
if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
return SOME_TREE
# 1.10: django.db.models.fields.related_lookups.RelatedExact
attname = where.lhs.target.attname
if isinstance(where, Exact):
return [[(where.lhs.alias, attname, where.rhs, True)]]
elif isinstance(where, IsNull):
return [[(where.lhs.alias, attname, None, where.rhs)]]
elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
return [[(where.lhs.alias, attname, v, True)]
for v in where.rhs]
else:
return SOME_TREE
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return SOME_TREE
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
if isinstance(where, Lookup):
# If where.lhs don't refer to a field then don't bother
if not hasattr(where.lhs, 'target'):
return SOME_TREE
# Don't bother with complex right hand side either
if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
return SOME_TREE
# Skip conditions on non-serialized fields
if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
return SOME_TREE
attname = where.lhs.target.attname
if isinstance(where, Exact):
return [[(where.lhs.alias, attname, where.rhs, True)]]
elif isinstance(where, IsNull):
return [[(where.lhs.alias, attname, None, where.rhs)]]
elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
else:
return SOME_TREE
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return SOME_TREE
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
if isinstance(where, tuple):
constraint, lookup, annotation, value = where
attname = attname_of(model, constraint.col)
if isinstance(value, (QuerySet, Query, SQLEvaluator)):
return [[SOME_COND]]
elif lookup == 'exact':
if isinstance(constraint.field, NON_SERIALIZABLE_FIELDS):
return [[SOME_COND]]
else:
return [[(constraint.alias, attname, value, True)]]
elif lookup == 'isnull':
return [[(constraint.alias, attname, None, value)]]
elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
return [[(constraint.alias, attname, v, True)] for v in value]
else:
return [[SOME_COND]]
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return [[SOME_COND]]
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
if isinstance(where, tuple):
constraint, lookup, annotation, value = where
attname = attname_of(model, constraint.col)
if isinstance(value, (QuerySet, Query, SQLEvaluator)):
return [[SOME_COND]]
elif lookup == 'exact':
if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
return [[SOME_COND]]
else:
return [[(constraint.alias, attname, value, True)]]
elif lookup == 'isnull':
return [[(constraint.alias, attname, None, value)]]
elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
return [[(constraint.alias, attname, v, True)] for v in value]
else:
return [[SOME_COND]]
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return [[SOME_COND]]
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def test_iffy():
assert map(iffy(_ % 2, _ * 2, _ / 2), [1, 2, 3, 4]) == [2, 1, 6, 2]
assert map(iffy(_ % 2, _ * 2), [1, 2, 3, 4]) == [2, 2, 6, 4]
assert map(iffy(_ * 2), [21, '', None]) == [42, '', None]
assert map(iffy(_ % 2, _ * 2, None), [1, 2, 3, 4]) == [2, None, 6, None]
assert map(iffy(_ + 1, default=1), [1, None, 2]) == [2, 1, 3]
assert map(iffy(set([1, 4, 5]), _ * 2), [1, 2, 3, 4]) == [2, 2, 3, 8]
assert map(iffy(r'\d+', str.upper), ['a2', 'c']) == ['A2', 'c']
assert map(iffy(set([1, 4, 5])), [False, 2, 4]) == [False, False, True]
assert map(iffy(None), [False, 2, 3, 4]) == [False, 2, 3, 4]
def get_clusters(refs, queries):
all_ref_seqs, all_ref_ids = extract_info(refs)
ref1, ref2 = all_ref_seqs[:2]
dists1, dists2 = partial(hamming, ref1), partial(hamming, ref2)
ref_seqs, ref_ids = all_ref_seqs[2:], all_ref_ids[2:]
if ref_seqs:
ref_dists1, ref_dists2 = map(dists1, ref_seqs), map(dists2, ref_seqs)
else:
ref_dists1 = ref_dists2 = [0,0]
ref_ids = ['','']
query_seqs, query_ids = extract_info(queries)
query_dists1, query_dists2 = map(dists1, query_seqs), map(dists2, query_seqs)
return ref_dists1, ref_dists2, ref_ids, query_dists1, query_dists2, query_ids, all_ref_ids[0], all_ref_ids[1]
def cached_as(*samples, **kwargs):
"""
Caches results of a function and invalidates them same way as given queryset(s).
NOTE: Ignores queryset cached ops settings, always caches.
"""
timeout = kwargs.pop('timeout', None)
extra = kwargs.pop('extra', None)
key_func = kwargs.pop('key_func', func_cache_key)
lock = kwargs.pop('lock', None)
if kwargs:
raise TypeError('Unexpected keyword arguments %s' % ', '.join(kwargs))
# If we unexpectedly get list instead of queryset return identity decorator.
# Paginator could do this when page.object_list is empty.
if len(samples) == 1 and isinstance(samples[0], list):
return lambda func: func
def _get_queryset(sample):
if isinstance(sample, Model):
queryset = sample.__class__.objects.filter(pk=sample.pk)
elif isinstance(sample, type) and issubclass(sample, Model):
queryset = sample.objects.all()
else:
queryset = sample
queryset._require_cacheprofile()
return queryset
querysets = map(_get_queryset, samples)
cond_dnfs = mapcat(dnfs, querysets)
key_extra = [qs._cache_key() for qs in querysets]
key_extra.append(extra)
if not timeout: # TODO: switch to is None on major release
timeout = min(qs._cacheprofile['timeout'] for qs in querysets)
if lock is None:
lock = any(qs._cacheprofile['lock'] for qs in querysets)
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if transaction_state.is_dirty() or not settings.CACHEOPS_ENABLED:
return func(*args, **kwargs)
cache_key = 'as:' + key_func(func, args, kwargs, key_extra)
with redis_client.getting(cache_key, lock=lock) as cache_data:
cache_read.send(sender=None,
func=func,
hit=cache_data is not None)
if cache_data is not None:
return pickle.loads(cache_data)
else:
result = func(*args, **kwargs)
cache_thing(cache_key, result, cond_dnfs, timeout)
return result
return wrapper
return decorator
def _cache_key(self, extra=''):
"""
Compute a cache key for this queryset
"""
md = md5()
md.update('%s.%s' % (self.__class__.__module__, self.__class__.__name__))
# Vary cache key for proxy models
md.update('%s.%s' % (self.model.__module__, self.model.__name__))
# Protect from field list changes in model
md.update(stamp_fields(self.model))
# Use query SQL as part of a key
try:
sql, params = self.query.get_compiler(self._db or DEFAULT_DB_ALIAS).as_sql()
try:
sql_str = sql % params
except UnicodeDecodeError:
sql_str = sql % map(smart_str, params)
md.update(smart_str(sql_str))
except EmptyResultSet:
pass
# If query results differ depending on database
if self._cacheprofile and not self._cacheprofile['db_agnostic']:
md.update(self.db)
if extra:
md.update(str(extra))
# Thing only appeared in Django 1.8 and was renamed in Django 1.9
it_class = getattr(self, '_iterator_class', None) or getattr(self, '_iterable_class', None)
if it_class:
md.update('%s.%s' % (it_class.__module__, it_class.__name__))
# 'flat' attribute changes results formatting for values_list() in Django 1.8 and earlier
if hasattr(self, 'flat'):
md.update(str(self.flat))
return 'q:%s' % md.hexdigest()
def _cache_key(self):
"""
Compute a cache key for this queryset
"""
md = md5()
md.update('%s.%s' %
(self.__class__.__module__, self.__class__.__name__))
# Vary cache key for proxy models
md.update('%s.%s' % (self.model.__module__, self.model.__name__))
# Protect from field list changes in model
md.update(stamp_fields(self.model))
# Use query SQL as part of a key
try:
sql, params = self.query.get_compiler(
self._db or DEFAULT_DB_ALIAS).as_sql()
try:
sql_str = sql % params
except UnicodeDecodeError:
sql_str = sql % map(smart_str, params)
md.update(smart_str(sql_str))
except EmptyResultSet:
pass
# If query results differ depending on database
if self._cacheprofile and not self._cacheprofile['db_agnostic']:
md.update(self.db)
# Thing only appeared in Django 1.9
it_class = getattr(self, '_iterable_class', None)
if it_class:
md.update('%s.%s' % (it_class.__module__, it_class.__name__))
# 'flat' attribute changes results formatting for values_list() in Django 1.8 and earlier
if hasattr(self, 'flat'):
md.update(str(self.flat))
return 'q:%s' % md.hexdigest()
def _cached_as(*samples, **kwargs):
"""
Caches results of a function and invalidates them same way as given queryset.
NOTE: Ignores queryset cached ops settings, just caches.
"""
timeout = kwargs.get('timeout')
extra = kwargs.get('extra')
_get_key = kwargs.get('_get_key')
# If we unexpectedly get list instead of queryset return identity decorator.
# Paginator could do this when page.object_list is empty.
# TODO: think of better way doing this.
if len(samples) == 1 and isinstance(samples[0], list):
return lambda func: func
def _get_queryset(sample):
if isinstance(sample, Model):
queryset = sample.__class__.objects.inplace().filter(pk=sample.pk)
elif isinstance(sample, type) and issubclass(sample, Model):
queryset = sample.objects.all()
else:
queryset = sample
queryset._require_cacheprofile()
return queryset
querysets = map(_get_queryset, samples)
cond_dnfs = mapcat(dnfs, querysets)
key_extra = [qs._cache_key() for qs in querysets]
key_extra.append(extra)
if not timeout:
timeout = min(qs._cacheconf['timeout'] for qs in querysets)
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
cache_key = 'as:' + _get_key(func, args, kwargs, key_extra)
cache_data = redis_client.get(cache_key)
if cache_data is not None:
return pickle.loads(cache_data)
if USE_LOCK:
with redis_lock.Lock(redis_client, cache_key):
cache_data = redis_client.get(cache_key)
if cache_data is not None:
return pickle.loads(cache_data)
result = func(*args)
cache_thing(cache_key, result, cond_dnfs, timeout)
else:
result = func(*args)
cache_thing(cache_key, result, cond_dnfs, timeout)
return result
return wrapper
return decorator
def cached_as(*samples, **kwargs):
"""
Caches results of a function and invalidates them same way as given queryset(s).
NOTE: Ignores queryset cached ops settings, always caches.
"""
timeout = kwargs.pop('timeout', None)
extra = kwargs.pop('extra', None)
key_func = kwargs.pop('key_func', func_cache_key)
lock = kwargs.pop('lock', None)
if not samples:
raise TypeError('Pass a queryset, a model or an object to cache like')
if kwargs:
raise TypeError('Unexpected keyword arguments %s' % ', '.join(kwargs))
# If we unexpectedly get list instead of queryset return identity decorator.
# Paginator could do this when page.object_list is empty.
if len(samples) == 1 and isinstance(samples[0], list):
return lambda func: func
def _get_queryset(sample):
if isinstance(sample, Model):
queryset = sample.__class__.objects.filter(pk=sample.pk)
elif isinstance(sample, type) and issubclass(sample, Model):
queryset = sample.objects.all()
else:
queryset = sample
queryset._require_cacheprofile()
return queryset
querysets = map(_get_queryset, samples)
cond_dnfs = mapcat(dnfs, querysets)
key_extra = [qs._cache_key() for qs in querysets]
key_extra.append(extra)
if not timeout: # TODO: switch to is None on major release
timeout = min(qs._cacheprofile['timeout'] for qs in querysets)
if lock is None:
lock = any(qs._cacheprofile['lock'] for qs in querysets)
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if transaction_state.is_dirty() or not settings.CACHEOPS_ENABLED:
return func(*args, **kwargs)
cache_key = 'as:' + key_func(func, args, kwargs, key_extra)
with redis_client.getting(cache_key, lock=lock) as cache_data:
cache_read.send(sender=None, func=func, hit=cache_data is not None)
if cache_data is not None:
return pickle.loads(cache_data)
else:
result = func(*args, **kwargs)
cache_thing(cache_key, result, cond_dnfs, timeout)
return result
return wrapper
return decorator
def clean_dnf(tree, for_alias):
cleaned = [clean_conj(conj, for_alias) for conj in tree]
# Any empty conjunction eats up the rest
# NOTE: a more elaborate DNF reduction is not really needed,
# just keep your querysets sane.
if not all(cleaned):
return [[]]
# To keep all schemes the same we sort conjunctions
return map(sorted, cleaned)
def clean_dnf(tree, for_alias):
cleaned = [clean_conj(conj, for_alias) for conj in tree]
# Any empty conjunction eats up the rest
# NOTE: a more elaborate DNF reduction is not really needed,
# just keep your querysets sane.
if not all(cleaned):
return [[]]
# To keep all schemes the same we sort conjunctions
return map(sorted, cleaned)
def test_log_durations(monkeypatch):
timestamps = iter([0, 0.01, 1, 1.000025])
monkeypatch.setattr('time.time', lambda: next(timestamps))
log = []
f = log_durations(log.append)(lambda: None)
f()
with log_durations(log.append, 'hello'):
pass
assert map(r'^\s*(\d+\.\d+ mk?s) in (?:<lambda>\(\)|hello)$', log) == ['10.00 ms', '25.00 mks']
def cached_as(*samples, **kwargs):
"""
Caches results of a function and invalidates them same way as given queryset.
NOTE: Ignores queryset cached ops settings, just caches.
"""
timeout = kwargs.get('timeout')
extra = kwargs.get('extra')
key_func = kwargs.get('key_func', func_cache_key)
# If we unexpectedly get list instead of queryset return identity decorator.
# Paginator could do this when page.object_list is empty.
if len(samples) == 1 and isinstance(samples[0], list):
return lambda func: func
def _get_queryset(sample):
if isinstance(sample, Model):
queryset = sample.__class__.objects.filter(pk=sample.pk)
elif isinstance(sample, type) and issubclass(sample, Model):
queryset = sample.objects.all()
else:
queryset = sample
queryset._require_cacheprofile()
return queryset
querysets = map(_get_queryset, samples)
cond_dnfs = mapcat(dnfs, querysets)
key_extra = [qs._cache_key() for qs in querysets]
key_extra.append(extra)
if not timeout:
timeout = min(qs._cacheconf['timeout'] for qs in querysets)
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
if in_transaction():
return func(*args, **kwargs)
default_key = 'as:' + key_func(func, args, kwargs, key_extra)
cache_key = get_user_defined_key(default_key=default_key) or default_key
cache_data = redis_client.get(cache_key)
cache_read.send(sender=None, func=func, hit=cache_data is not None)
if cache_data is not None:
return pickle.loads(cache_data)
result = func(*args, **kwargs)
cache_thing(cache_key, result, cond_dnfs, timeout)
return result
return wrapper
return decorator
def test_log_durations_ex(monkeypatch):
timestamps = [0, 0.01, 1, 1.001, 2, 2.02]
timestamps_iter = iter(timestamps)
monkeypatch.setattr('time.time', lambda: next(timestamps_iter))
log = []
f = log_durations(log.append, unit='ms', threshold=1.1e-3)(lambda: None)
f(); f(); f()
assert len(log) == 2
assert map(r'^\s*(\d+\.\d+) ms in', log) == ['10.00', '20.00']
def process(refs_fn, query_fn, save_path=None, html=True):
ref_seqs, ref_dates, ref_names = zip(*sorted(zip(*get_seqs_and_dates(refs_fn)), key=get(1)))
#assert len(ref_seqs) > 1, "Need more than 1 reference sequence"
ref_seqs = map(str.upper, ref_seqs)
super_ref_seq, super_ref_date, super_ref_name = ref_seqs[0], ref_dates[0], ref_names[0]
print(super_ref_name)
print(super_ref_date)
get_mutations = partial(hamming, super_ref_seq)
def get_relative_info(seqs, dates, names):
muts = map(get_mutations, seqs)
dists = [(yr - super_ref_date).days for yr in dates]
return muts, dists, names
ref_muts, ref_dists, ref_names = get_relative_info(ref_seqs, ref_dates, ref_names)
query_muts, query_dists, query_names = get_relative_info(*get_seqs_and_dates(query_fn))
do_plot(ref_dists, ref_muts, ref_names, query_dists, query_muts, query_names, save_path, html)
def test_iffy():
assert map(iffy(_ % 2, _ * 2, _ / 2), [1,2,3,4]) == [2,1,6,2]
assert map(iffy(_ % 2, _ * 2), [1,2,3,4]) == [2,2,6,4]
assert map(iffy(_ * 2), [21, '', None]) == [42, '', None]
assert map(iffy(_ % 2, _ * 2, None), [1,2,3,4]) == [2, None, 6, None]
def test_list_iter():
assert is_list(py2.map(None, []))
assert is_iter(py3.map(None, []))
assert is_list(funcy.map(None, [])) == PY2
assert is_iter(funcy.map(None, [])) == PY3
def _stringify_query():
"""
Serializes query object, so that it can be used to create cache key.
We can't just do pickle because order of keys in dicts is arbitrary,
we can use str(query) which compiles it to SQL, but it's too slow,
so we use json.dumps with sort_keys=True and object hooks.
NOTE: I like this function no more than you, it's messy
and pretty hard linked to django internals.
I just don't have nicer solution for now.
Probably the best way out of it is optimizing SQL generation,
which would be valuable by itself. The problem with it is that
any significant optimization will most likely require a major
refactor of sql.Query class, which is a substantial part of ORM.
"""
from datetime import datetime, date, time, timedelta
from decimal import Decimal
from django.db.models.expressions import ExpressionNode, F
from django.db.models.fields import Field
from django.db.models.fields.related import ManyToOneRel, OneToOneRel
from django.db.models.sql.where import Constraint, WhereNode, ExtraWhere, \
EverythingNode, NothingNode
from django.db.models.sql import Query
from django.db.models.sql.aggregates import Aggregate
from django.db.models.sql.datastructures import Date
from django.db.models.sql.expressions import SQLEvaluator
attrs = {}
# Try to not require geo libs
try:
from django.contrib.gis.db.models.sql.where import GeoWhereNode
except: # either ImportError or GEOSException
GeoWhereNode = WhereNode
# A new things in Django 1.6
try:
from django.db.models.sql.where import EmptyWhere, SubqueryConstraint
attrs[EmptyWhere] = ()
attrs[SubqueryConstraint] = ('alias', 'columns', 'targets', 'query_object')
except ImportError:
pass
# RawValue removed in Django 1.7
try:
from django.db.models.sql.datastructures import RawValue
attrs[RawValue] = ('value',)
except ImportError:
pass
# Moved in Django 1.7
try:
from django.contrib.contenttypes.fields import GenericRel
except ImportError:
from django.contrib.contenttypes.generic import GenericRel
# New things in Django 1.7
try:
from django.db.models.lookups import Lookup
from django.db.models.sql.datastructures import Col
attrs[Lookup] = ('lhs', 'rhs')
attrs[Col] = ('alias', 'target', 'source')
except ImportError:
class Lookup(object):
pass
attrs[WhereNode] = attrs[GeoWhereNode] = attrs[ExpressionNode] \
= ('connector', 'negated', 'children')
attrs[SQLEvaluator] = ('expression',)
attrs[ExtraWhere] = ('sqls', 'params')
attrs[Aggregate] = ('source', 'is_summary', 'col', 'extra')
attrs[Date] = ('col', 'lookup_type')
attrs[F] = ('name',)
attrs[ManyToOneRel] = attrs[OneToOneRel] = attrs[GenericRel] = ('field',)
attrs[EverythingNode] = attrs[NothingNode] = ()
q = Query(None)
q_keys = q.__dict__.keys()
q_ignored = ['join_map', 'dupe_avoidance', '_extra_select_cache', '_aggregate_select_cache',
'used_aliases']
attrs[Query] = tuple(sorted(set(q_keys) - set(q_ignored)))
try:
for k, v in attrs.items():
attrs[k] = map(intern, v)
except NameError:
# No intern() in Python 3
pass
def encode_attrs(obj, cls=None):
return (obj.__class__, [getattr(obj, attr) for attr in attrs[cls or obj.__class__]])
def encode_object(obj):
if isinstance(obj, set):
return sorted(obj)
elif isinstance(obj, type):
return '%s.%s' % (obj.__module__, obj.__name__)
elif hasattr(obj, '__uniq_key__'):
return (obj.__class__, obj.__uniq_key__())
elif isinstance(obj, (datetime, date, time, timedelta, Decimal)):
return str(obj)
elif isinstance(obj, Constraint):
return (obj.alias, obj.col)
elif isinstance(obj, Field):
return (obj.model, obj.name)
elif obj.__class__ in attrs:
return encode_attrs(obj)
elif isinstance(obj, QuerySet):
return (obj.__class__, obj.query)
elif isinstance(obj, Aggregate):
return encode_attrs(obj, Aggregate)
elif isinstance(obj, Query):
return encode_attrs(obj, Query) # for custom subclasses of Query
elif isinstance(obj, Lookup):
return encode_attrs(obj, Lookup)
# Fall back for unknown objects
elif not STRICT_STRINGIFY and hasattr(obj, '__dict__'):
return (obj.__class__, obj.__dict__)
else:
raise TypeError("Can't stringify %s" % repr(obj))
def stringify_query(query):
# HACK: Catch TypeError and reraise it as ValueError
# since django hides it and behave weird when gets a TypeError in Queryset.iterator()
try:
return json.dumps(query, default=encode_object, skipkeys=True,
sort_keys=True, separators=(',', ':'))
except TypeError as e:
raise ValueError(*e.args)
return stringify_query
def test_list_iter():
assert is_list(py2.map(None, []))
assert is_iter(py3.map(None, []))
assert is_list(funcy.map(None, [])) == PY2
assert is_iter(funcy.map(None, [])) == PY3
def do_plot(x1, y1, ref_names, x2, y2, query_names, save_path=None, html=True, \
title='Mutations over time (days)', x_axis_label='days', y_axis_label='bases'):
'''
:param iterable x1: reference dates distances
:param iterable y1: reference p-distances
:param iterable x2: query dates diferences
:param iterable y2: query p-distances
:param str save_path: path to save image or None to open GTK if installed
'''
assert len(x1) > 0, "No reference dates to use"
assert len(y2) > 0, "No reference p-distances to use"
assert len(x2) > 0, "No query dates to use"
assert len(y2) > 0, "No query p-distances to use"
assert len(ref_names) == len(x1) and len(query_names) == len(x2)
fig = plt.figure()
ax = plt.subplot(111)
# from matplotlib.dates import YearLocator, MonthLocator, DateFormatter
# years = YearLocator() # every year
# months = MonthLocator() # every month
# yearsFmt = DateFormatter('%Y')
# ax.xaxis.set_major_locator(years)
# ax.xaxis.set_major_formatter(yearsFmt)
# ax.xaxis.set_minor_locator(months)
max_x = max(max(x1), max(x2))
#legend_info = [mpatches.Patch(label=n, color=c) for n, c in legend.items()]
""" http://stackoverflow.com/questions/4700614/how-to-put-the-legend-out-of-the-plot"""
ref_info = zip(ref_names, x1, y1)
query_info = zip(query_names, x2, y2)
all_info = sorted(ref_info + query_info, key=lambda x: x[2], reverse=True)
if save_path:
fh = open(save_path+'.csv', 'wt')
else:
fh = sys.stdout
fh.write('name,dates,p-dist\n')
outcsv = csv.writer(fh)
map(outcsv.writerow, all_info)
if html:
assert sys.version[:3] != '2.6', "Requires python 2.7 or higher to run bokeh."
import bokeh.models as bkm
import bokeh.plotting as bkp
bokeh_tools = [bkm.WheelZoomTool(), bkm.PanTool(), bkm.BoxZoomTool(),
bkm.PreviewSaveTool(), bkm.ResetTool(), bkm.BoxSelectTool(),
bkm.ResizeTool()]
ref_names = map('R: {0}'.format, ref_names)
query_names = map('Q: {0}'.format, query_names)
hover = bkm.HoverTool(tooltips=[("id", "@ids"),]) # ("(days,muts)", "($x, $y)"),
source1 = bkm.ColumnDataSource(data=dict(x=x1, y=y1, ids=ref_names))
source2 = bkm.ColumnDataSource(data=dict(x2=x2, y2=y2, ids=query_names))
p = bkp.figure(plot_width=400, plot_height=400, tools=[hover]+bokeh_tools, title=title)
p.circle('x', 'y', source=source1, line_color='gray', legend='reference')
p.square('x2', 'y2', source=source2, fill_color='red', legend='query')
if save_path:
bkp.output_file(save_path + '.html')
bkp.save(p)
else: bkp.show(p)
else:
plot_muts(ax, x1, y1, plotkwargs=dict(label='references (blue)', color=legend['references'], marker='s'), polyfit=True, max_x=max_x, dist=None)
plot_muts(ax, x2, y2, plotkwargs=dict(label='queries (red)', color=legend['queries']), dist=None)
box = ax.get_position()
ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
#ax.legend(handles=legend_info, loc='center left', bbox_to_anchor=(1, 0.5))
#ax.legend(loc='center left', bbox_to_anchor=(1, 0.5), framealpha=0)
ax.legend(framealpha=0)
plt.xlabel("days since Base reference")
plt.ylabel("p-distance")
if save_path:
plt.savefig(save_path)
else:
plt.show()
def get_relative_info(seqs, dates, names):
muts = map(get_mutations, seqs)
dists = [(yr - super_ref_date).days for yr in dates]
return muts, dists, names
def get_seqs_and_dates(fn):
fasta = SeqIO.parse(fn, format="fasta")
info = [(str(seq.seq), seq.id, seq.description) for seq in fasta]
seqs, ids, descriptions = zip(*info)
dates = map(extract_date, ids)
return seqs, dates, ids
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
# Lookups appeared in Django 1.7
if isinstance(where, Lookup):
# If where.lhs don't refer to a field then don't bother
if not hasattr(where.lhs, 'target'):
return [[SOME_COND]]
# TODO: check if all of this are possible
if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
return [[SOME_COND]]
attname = where.lhs.target.attname
if isinstance(where, Exact):
if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
return [[SOME_COND]]
else:
return [[(where.lhs.alias, attname, where.rhs, True)]]
elif isinstance(where, IsNull):
return [[(where.lhs.alias, attname, None, where.rhs)]]
elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
else:
return [[SOME_COND]]
# Django 1.6 and earlier used tuples to encode conditions
elif isinstance(where, tuple):
constraint, lookup, annotation, value = where
attname = attname_of(model, constraint.col)
if isinstance(value, (QuerySet, Query, SQLEvaluator)):
return [[SOME_COND]]
elif lookup == 'exact':
# TODO: check for non-serialized for both exact and in
if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
return [[SOME_COND]]
else:
return [[(constraint.alias, attname, value, True)]]
elif lookup == 'isnull':
return [[(constraint.alias, attname, None, value)]]
elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
return [[(constraint.alias, attname, v, True)] for v in value]
else:
return [[SOME_COND]]
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return [[SOME_COND]]
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def _stringify_query():
"""
Serializes query object, so that it can be used to create cache key.
We can't just do pickle because order of keys in dicts is arbitrary,
we can use str(query) which compiles it to SQL, but it's too slow,
so we use json.dumps with sort_keys=True and object hooks.
NOTE: I like this function no more than you, it's messy
and pretty hard linked to django internals.
I just don't have nicer solution for now.
Probably the best way out of it is optimizing SQL generation,
which would be valuable by itself. The problem with it is that
any significant optimization will most likely require a major
refactor of sql.Query class, which is a substantial part of ORM.
"""
from datetime import datetime, date, time, timedelta
from decimal import Decimal
from django.db.models.expressions import ExpressionNode, F
from django.db.models.fields import Field
from django.db.models.fields.related import ManyToOneRel, OneToOneRel
from django.db.models.sql.where import Constraint, WhereNode, ExtraWhere, \
EverythingNode, NothingNode
from django.db.models.sql import Query
from django.db.models.sql.aggregates import Aggregate
from django.db.models.sql.datastructures import Date
from django.db.models.sql.expressions import SQLEvaluator
attrs = {}
# Try to not require geo libs
try:
from django.contrib.gis.db.models.sql.where import GeoWhereNode
except: # either ImportError or GEOSException
GeoWhereNode = WhereNode
# A new things in Django 1.6
try:
from django.db.models.sql.where import EmptyWhere, SubqueryConstraint
attrs[EmptyWhere] = ()
attrs[SubqueryConstraint] = ('alias', 'columns', 'targets', 'query_object')
except ImportError:
pass
# RawValue removed in Django 1.7
try:
from django.db.models.sql.datastructures import RawValue
attrs[RawValue] = ('value',)
except ImportError:
pass
# Moved in Django 1.7
try:
from django.contrib.contenttypes.fields import GenericRel
except ImportError:
from django.contrib.contenttypes.generic import GenericRel
# New things in Django 1.7
try:
from django.db.models.lookups import Lookup
from django.db.models.sql.datastructures import Col
attrs[Lookup] = ('lhs', 'rhs')
attrs[Col] = ('alias', 'target', 'source')
except ImportError:
class Lookup(object):
pass
attrs[WhereNode] = attrs[GeoWhereNode] = attrs[ExpressionNode] \
= ('connector', 'negated', 'children')
attrs[SQLEvaluator] = ('expression',)
attrs[ExtraWhere] = ('sqls', 'params')
attrs[Aggregate] = ('source', 'is_summary', 'col', 'extra')
attrs[Date] = ('col', 'lookup_type')
attrs[F] = ('name',)
attrs[ManyToOneRel] = attrs[OneToOneRel] = attrs[GenericRel] = ('field',)
attrs[EverythingNode] = attrs[NothingNode] = ()
q = Query(None)
q_keys = q.__dict__.keys()
q_ignored = ['join_map', 'dupe_avoidance', '_extra_select_cache', '_aggregate_select_cache',
'used_aliases']
attrs[Query] = tuple(sorted(set(q_keys) - set(q_ignored)))
try:
for k, v in attrs.items():
attrs[k] = map(intern, v)
except NameError:
# No intern() in Python 3
pass
def encode_attrs(obj, cls=None):
return (obj.__class__, [getattr(obj, attr) for attr in attrs[cls or obj.__class__]])
def encode_object(obj):
if isinstance(obj, set):
return sorted(obj)
elif isinstance(obj, type):
return '%s.%s' % (obj.__module__, obj.__name__)
elif hasattr(obj, '__uniq_key__'):
return (obj.__class__, obj.__uniq_key__())
elif isinstance(obj, (datetime, date, time, timedelta, Decimal)):
return str(obj)
elif isinstance(obj, Constraint):
return (obj.alias, obj.col)
elif isinstance(obj, Field):
return (obj.model, obj.name)
elif obj.__class__ in attrs:
return encode_attrs(obj)
elif isinstance(obj, QuerySet):
return (obj.__class__, obj.query)
elif isinstance(obj, Aggregate):
return encode_attrs(obj, Aggregate)
elif isinstance(obj, Query):
return encode_attrs(obj, Query) # for custom subclasses of Query
elif isinstance(obj, Lookup):
return encode_attrs(obj, Lookup)
# Fall back for unknown objects
elif not STRICT_STRINGIFY and hasattr(obj, '__dict__'):
return (obj.__class__, obj.__dict__)
else:
raise TypeError("Can't stringify %s" % repr(obj))
def stringify_query(query):
# HACK: Catch TypeError and reraise it as ValueError
# since django hides it and behave weird when gets a TypeError in Queryset.iterator()
try:
return json.dumps(query, default=encode_object, skipkeys=True,
sort_keys=True, separators=(',', ':'))
except TypeError as e:
raise ValueError(*e.args)
return stringify_query
def required_star(f, *seqs):
return map(f, *seqs)
def _dnf(where):
"""
Constructs DNF of where tree consisting of terms in form:
(alias, attribute, value, negation)
meaning `alias.attribute = value`
or `not alias.attribute = value` if negation is False
Any conditions other then eq are dropped.
"""
# Lookups appeared in Django 1.7
if isinstance(where, Lookup):
# If where.lhs don't refer to a field then don't bother
if not hasattr(where.lhs, 'target'):
return SOME_TREE
# Don't bother with complex right hand side either
if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
return SOME_TREE
# Skip conditions on non-serialized fields
if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
return SOME_TREE
attname = where.lhs.target.attname
if isinstance(where, Exact):
return [[(where.lhs.alias, attname, where.rhs, True)]]
elif isinstance(where, IsNull):
return [[(where.lhs.alias, attname, None, where.rhs)]]
elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
else:
return SOME_TREE
# Django 1.6 and earlier used tuples to encode conditions
elif isinstance(where, tuple):
constraint, lookup, annotation, value = where
# Don't bother with complex right hand side
if isinstance(value, (QuerySet, Query, SQLEvaluator)):
return SOME_TREE
# Skip conditions on non-serialized fields
if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
return SOME_TREE
attname = attname_of(model, constraint.col)
if lookup == 'isnull':
return [[(constraint.alias, attname, None, value)]]
elif lookup == 'exact':
return [[(constraint.alias, attname, value, True)]]
elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
return [[(constraint.alias, attname, v, True)] for v in value]
else:
return SOME_TREE
elif isinstance(where, EverythingNode):
return [[]]
elif isinstance(where, NothingNode):
return []
elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
return SOME_TREE
elif len(where) == 0:
return [[]]
else:
chilren_dnfs = map(_dnf, where.children)
if len(chilren_dnfs) == 0:
return [[]]
elif len(chilren_dnfs) == 1:
result = chilren_dnfs[0]
else:
# Just unite children joined with OR
if where.connector == OR:
result = cat(chilren_dnfs)
# Use Cartesian product to AND children
else:
result = map(cat, product(*chilren_dnfs))
# Negating and expanding brackets
if where.negated:
result = [map(negate, p) for p in product(*result)]
return result
def test_juxt():
assert juxt(__add__, __sub__)(10, 2) == [12, 8]
assert map(juxt(_ + 1, _ - 1), [2, 3]) == [[3, 1], [4, 2]]