Пример #1
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        if isinstance(where, Lookup):
            # If where.lhs don't refer to a field then don't bother
            if not hasattr(where.lhs, 'target'):
                return SOME_TREE
            # Don't bother with complex right hand side either
            if isinstance(where.rhs, (QuerySet, Query)):
                return SOME_TREE
            # Skip conditions on non-serialized fields
            if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
                return SOME_TREE

            # 1.10: django.db.models.fields.related_lookups.RelatedExact

            attname = where.lhs.target.attname
            if isinstance(where, Exact):
                return [[(where.lhs.alias, attname, where.rhs, True)]]
            elif isinstance(where, IsNull):
                return [[(where.lhs.alias, attname, None, where.rhs)]]
            elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
                return [[(where.lhs.alias, attname, v, True)]
                        for v in where.rhs]
            else:
                return SOME_TREE
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return SOME_TREE
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #2
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        if isinstance(where, Lookup):
            # If where.lhs don't refer to a field then don't bother
            if not hasattr(where.lhs, 'target'):
                return SOME_TREE
            # Don't bother with complex right hand side either
            if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
                return SOME_TREE
            # Skip conditions on non-serialized fields
            if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
                return SOME_TREE

            attname = where.lhs.target.attname
            if isinstance(where, Exact):
                return [[(where.lhs.alias, attname, where.rhs, True)]]
            elif isinstance(where, IsNull):
                return [[(where.lhs.alias, attname, None, where.rhs)]]
            elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
                return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
            else:
                return SOME_TREE
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return SOME_TREE
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #3
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        if isinstance(where, tuple):
            constraint, lookup, annotation, value = where
            attname = attname_of(model, constraint.col)
            if isinstance(value, (QuerySet, Query, SQLEvaluator)):
                return [[SOME_COND]]
            elif lookup == 'exact':
                if isinstance(constraint.field, NON_SERIALIZABLE_FIELDS):
                    return [[SOME_COND]]
                else:
                    return [[(constraint.alias, attname, value, True)]]
            elif lookup == 'isnull':
                return [[(constraint.alias, attname, None, value)]]
            elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
                return [[(constraint.alias, attname, v, True)] for v in value]
            else:
                return [[SOME_COND]]
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return [[SOME_COND]]
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #4
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        if isinstance(where, tuple):
            constraint, lookup, annotation, value = where
            attname = attname_of(model, constraint.col)
            if isinstance(value, (QuerySet, Query, SQLEvaluator)):
                return [[SOME_COND]]
            elif lookup == 'exact':
                if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
                    return [[SOME_COND]]
                else:
                    return [[(constraint.alias, attname, value, True)]]
            elif lookup == 'isnull':
                return [[(constraint.alias, attname, None, value)]]
            elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
                return [[(constraint.alias, attname, v, True)] for v in value]
            else:
                return [[SOME_COND]]
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return [[SOME_COND]]
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #5
0
def test_iffy():
    assert map(iffy(_ % 2, _ * 2, _ / 2), [1, 2, 3, 4]) == [2, 1, 6, 2]
    assert map(iffy(_ % 2, _ * 2), [1, 2, 3, 4]) == [2, 2, 6, 4]
    assert map(iffy(_ * 2), [21, '', None]) == [42, '', None]
    assert map(iffy(_ % 2, _ * 2, None), [1, 2, 3, 4]) == [2, None, 6, None]
    assert map(iffy(_ + 1, default=1), [1, None, 2]) == [2, 1, 3]
    assert map(iffy(set([1, 4, 5]), _ * 2), [1, 2, 3, 4]) == [2, 2, 3, 8]
    assert map(iffy(r'\d+', str.upper), ['a2', 'c']) == ['A2', 'c']
    assert map(iffy(set([1, 4, 5])), [False, 2, 4]) == [False, False, True]
    assert map(iffy(None), [False, 2, 3, 4]) == [False, 2, 3, 4]
Пример #6
0
def get_clusters(refs, queries):
    all_ref_seqs, all_ref_ids = extract_info(refs)
    ref1, ref2 = all_ref_seqs[:2]
    dists1, dists2 = partial(hamming, ref1), partial(hamming, ref2)
    ref_seqs, ref_ids = all_ref_seqs[2:], all_ref_ids[2:]
    if ref_seqs:
        ref_dists1, ref_dists2 = map(dists1, ref_seqs), map(dists2, ref_seqs)
    else:
        ref_dists1 = ref_dists2 = [0,0]
        ref_ids = ['','']

    query_seqs, query_ids = extract_info(queries)
    query_dists1, query_dists2 = map(dists1, query_seqs), map(dists2, query_seqs)

    return ref_dists1, ref_dists2, ref_ids, query_dists1, query_dists2, query_ids, all_ref_ids[0], all_ref_ids[1]
Пример #7
0
def cached_as(*samples, **kwargs):
    """
    Caches results of a function and invalidates them same way as given queryset(s).
    NOTE: Ignores queryset cached ops settings, always caches.
    """
    timeout = kwargs.pop('timeout', None)
    extra = kwargs.pop('extra', None)
    key_func = kwargs.pop('key_func', func_cache_key)
    lock = kwargs.pop('lock', None)
    if kwargs:
        raise TypeError('Unexpected keyword arguments %s' % ', '.join(kwargs))

    # If we unexpectedly get list instead of queryset return identity decorator.
    # Paginator could do this when page.object_list is empty.
    if len(samples) == 1 and isinstance(samples[0], list):
        return lambda func: func

    def _get_queryset(sample):
        if isinstance(sample, Model):
            queryset = sample.__class__.objects.filter(pk=sample.pk)
        elif isinstance(sample, type) and issubclass(sample, Model):
            queryset = sample.objects.all()
        else:
            queryset = sample

        queryset._require_cacheprofile()

        return queryset

    querysets = map(_get_queryset, samples)
    cond_dnfs = mapcat(dnfs, querysets)
    key_extra = [qs._cache_key() for qs in querysets]
    key_extra.append(extra)
    if not timeout:  # TODO: switch to is None on major release
        timeout = min(qs._cacheprofile['timeout'] for qs in querysets)
    if lock is None:
        lock = any(qs._cacheprofile['lock'] for qs in querysets)

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            if transaction_state.is_dirty() or not settings.CACHEOPS_ENABLED:
                return func(*args, **kwargs)

            cache_key = 'as:' + key_func(func, args, kwargs, key_extra)

            with redis_client.getting(cache_key, lock=lock) as cache_data:
                cache_read.send(sender=None,
                                func=func,
                                hit=cache_data is not None)
                if cache_data is not None:
                    return pickle.loads(cache_data)
                else:
                    result = func(*args, **kwargs)
                    cache_thing(cache_key, result, cond_dnfs, timeout)
                    return result

        return wrapper

    return decorator
Пример #8
0
    def _cache_key(self, extra=''):
        """
        Compute a cache key for this queryset
        """
        md = md5()
        md.update('%s.%s' % (self.__class__.__module__, self.__class__.__name__))
        # Vary cache key for proxy models
        md.update('%s.%s' % (self.model.__module__, self.model.__name__))
        # Protect from field list changes in model
        md.update(stamp_fields(self.model))
        # Use query SQL as part of a key
        try:
            sql, params = self.query.get_compiler(self._db or DEFAULT_DB_ALIAS).as_sql()
            try:
                sql_str = sql % params
            except UnicodeDecodeError:
                sql_str = sql % map(smart_str, params)
            md.update(smart_str(sql_str))
        except EmptyResultSet:
            pass
        # If query results differ depending on database
        if self._cacheprofile and not self._cacheprofile['db_agnostic']:
            md.update(self.db)
        if extra:
            md.update(str(extra))
        # Thing only appeared in Django 1.8 and was renamed in Django 1.9
        it_class = getattr(self, '_iterator_class', None) or getattr(self, '_iterable_class', None)
        if it_class:
            md.update('%s.%s' % (it_class.__module__, it_class.__name__))
        # 'flat' attribute changes results formatting for values_list() in Django 1.8 and earlier
        if hasattr(self, 'flat'):
            md.update(str(self.flat))

        return 'q:%s' % md.hexdigest()
Пример #9
0
    def _cache_key(self):
        """
        Compute a cache key for this queryset
        """
        md = md5()
        md.update('%s.%s' %
                  (self.__class__.__module__, self.__class__.__name__))
        # Vary cache key for proxy models
        md.update('%s.%s' % (self.model.__module__, self.model.__name__))
        # Protect from field list changes in model
        md.update(stamp_fields(self.model))
        # Use query SQL as part of a key
        try:
            sql, params = self.query.get_compiler(
                self._db or DEFAULT_DB_ALIAS).as_sql()
            try:
                sql_str = sql % params
            except UnicodeDecodeError:
                sql_str = sql % map(smart_str, params)
            md.update(smart_str(sql_str))
        except EmptyResultSet:
            pass
        # If query results differ depending on database
        if self._cacheprofile and not self._cacheprofile['db_agnostic']:
            md.update(self.db)
        # Thing only appeared in Django 1.9
        it_class = getattr(self, '_iterable_class', None)
        if it_class:
            md.update('%s.%s' % (it_class.__module__, it_class.__name__))
        # 'flat' attribute changes results formatting for values_list() in Django 1.8 and earlier
        if hasattr(self, 'flat'):
            md.update(str(self.flat))

        return 'q:%s' % md.hexdigest()
Пример #10
0
def _cached_as(*samples, **kwargs):
    """
    Caches results of a function and invalidates them same way as given queryset.
    NOTE: Ignores queryset cached ops settings, just caches.
    """
    timeout = kwargs.get('timeout')
    extra = kwargs.get('extra')
    _get_key = kwargs.get('_get_key')

    # If we unexpectedly get list instead of queryset return identity decorator.
    # Paginator could do this when page.object_list is empty.
    # TODO: think of better way doing this.
    if len(samples) == 1 and isinstance(samples[0], list):
        return lambda func: func

    def _get_queryset(sample):
        if isinstance(sample, Model):
            queryset = sample.__class__.objects.inplace().filter(pk=sample.pk)
        elif isinstance(sample, type) and issubclass(sample, Model):
            queryset = sample.objects.all()
        else:
            queryset = sample

        queryset._require_cacheprofile()

        return queryset

    querysets = map(_get_queryset, samples)
    cond_dnfs = mapcat(dnfs, querysets)
    key_extra = [qs._cache_key() for qs in querysets]
    key_extra.append(extra)
    if not timeout:
        timeout = min(qs._cacheconf['timeout'] for qs in querysets)

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            cache_key = 'as:' + _get_key(func, args, kwargs, key_extra)

            cache_data = redis_client.get(cache_key)
            if cache_data is not None:
                return pickle.loads(cache_data)

            if USE_LOCK:
                with redis_lock.Lock(redis_client, cache_key):
                    cache_data = redis_client.get(cache_key)
                    if cache_data is not None:
                        return pickle.loads(cache_data)
                    result = func(*args)
                    cache_thing(cache_key, result, cond_dnfs, timeout)
            else:
                result = func(*args)
                cache_thing(cache_key, result, cond_dnfs, timeout)

            return result

        return wrapper

    return decorator
Пример #11
0
def cached_as(*samples, **kwargs):
    """
    Caches results of a function and invalidates them same way as given queryset(s).
    NOTE: Ignores queryset cached ops settings, always caches.
    """
    timeout = kwargs.pop('timeout', None)
    extra = kwargs.pop('extra', None)
    key_func = kwargs.pop('key_func', func_cache_key)
    lock = kwargs.pop('lock', None)
    if not samples:
        raise TypeError('Pass a queryset, a model or an object to cache like')
    if kwargs:
        raise TypeError('Unexpected keyword arguments %s' % ', '.join(kwargs))

    # If we unexpectedly get list instead of queryset return identity decorator.
    # Paginator could do this when page.object_list is empty.
    if len(samples) == 1 and isinstance(samples[0], list):
        return lambda func: func

    def _get_queryset(sample):
        if isinstance(sample, Model):
            queryset = sample.__class__.objects.filter(pk=sample.pk)
        elif isinstance(sample, type) and issubclass(sample, Model):
            queryset = sample.objects.all()
        else:
            queryset = sample

        queryset._require_cacheprofile()

        return queryset

    querysets = map(_get_queryset, samples)
    cond_dnfs = mapcat(dnfs, querysets)
    key_extra = [qs._cache_key() for qs in querysets]
    key_extra.append(extra)
    if not timeout:  # TODO: switch to is None on major release
        timeout = min(qs._cacheprofile['timeout'] for qs in querysets)
    if lock is None:
        lock = any(qs._cacheprofile['lock'] for qs in querysets)

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            if transaction_state.is_dirty() or not settings.CACHEOPS_ENABLED:
                return func(*args, **kwargs)

            cache_key = 'as:' + key_func(func, args, kwargs, key_extra)

            with redis_client.getting(cache_key, lock=lock) as cache_data:
                cache_read.send(sender=None, func=func, hit=cache_data is not None)
                if cache_data is not None:
                    return pickle.loads(cache_data)
                else:
                    result = func(*args, **kwargs)
                    cache_thing(cache_key, result, cond_dnfs, timeout)
                    return result

        return wrapper
    return decorator
Пример #12
0
 def clean_dnf(tree, for_alias):
     cleaned = [clean_conj(conj, for_alias) for conj in tree]
     # Any empty conjunction eats up the rest
     # NOTE: a more elaborate DNF reduction is not really needed,
     #       just keep your querysets sane.
     if not all(cleaned):
         return [[]]
     # To keep all schemes the same we sort conjunctions
     return map(sorted, cleaned)
Пример #13
0
 def clean_dnf(tree, for_alias):
     cleaned = [clean_conj(conj, for_alias) for conj in tree]
     # Any empty conjunction eats up the rest
     # NOTE: a more elaborate DNF reduction is not really needed,
     #       just keep your querysets sane.
     if not all(cleaned):
         return [[]]
     # To keep all schemes the same we sort conjunctions
     return map(sorted, cleaned)
Пример #14
0
def test_log_durations(monkeypatch):
    timestamps = iter([0, 0.01, 1, 1.000025])
    monkeypatch.setattr('time.time', lambda: next(timestamps))
    log = []

    f = log_durations(log.append)(lambda: None)
    f()
    with log_durations(log.append, 'hello'):
        pass

    assert map(r'^\s*(\d+\.\d+ mk?s) in (?:<lambda>\(\)|hello)$', log) == ['10.00 ms', '25.00 mks']
Пример #15
0
def cached_as(*samples, **kwargs):
    """
    Caches results of a function and invalidates them same way as given queryset.
    NOTE: Ignores queryset cached ops settings, just caches.
    """
    timeout = kwargs.get('timeout')
    extra = kwargs.get('extra')
    key_func = kwargs.get('key_func', func_cache_key)

    # If we unexpectedly get list instead of queryset return identity decorator.
    # Paginator could do this when page.object_list is empty.
    if len(samples) == 1 and isinstance(samples[0], list):
        return lambda func: func

    def _get_queryset(sample):
        if isinstance(sample, Model):
            queryset = sample.__class__.objects.filter(pk=sample.pk)
        elif isinstance(sample, type) and issubclass(sample, Model):
            queryset = sample.objects.all()
        else:
            queryset = sample

        queryset._require_cacheprofile()

        return queryset

    querysets = map(_get_queryset, samples)
    cond_dnfs = mapcat(dnfs, querysets)
    key_extra = [qs._cache_key() for qs in querysets]
    key_extra.append(extra)
    if not timeout:
        timeout = min(qs._cacheconf['timeout'] for qs in querysets)

    def decorator(func):
        @wraps(func)
        def wrapper(*args, **kwargs):
            if in_transaction():
                return func(*args, **kwargs)

            default_key = 'as:' + key_func(func, args, kwargs, key_extra)
            cache_key = get_user_defined_key(default_key=default_key) or default_key

            cache_data = redis_client.get(cache_key)
            cache_read.send(sender=None, func=func, hit=cache_data is not None)
            if cache_data is not None:
                return pickle.loads(cache_data)

            result = func(*args, **kwargs)
            cache_thing(cache_key, result, cond_dnfs, timeout)
            return result

        return wrapper

    return decorator
Пример #16
0
def test_log_durations_ex(monkeypatch):
    timestamps = [0, 0.01, 1, 1.001, 2, 2.02]
    timestamps_iter = iter(timestamps)
    monkeypatch.setattr('time.time', lambda: next(timestamps_iter))
    log = []

    f = log_durations(log.append, unit='ms', threshold=1.1e-3)(lambda: None)
    f(); f(); f()

    assert len(log) == 2
    assert map(r'^\s*(\d+\.\d+) ms in', log) == ['10.00', '20.00']
Пример #17
0
def process(refs_fn, query_fn, save_path=None, html=True):
    ref_seqs, ref_dates, ref_names = zip(*sorted(zip(*get_seqs_and_dates(refs_fn)), key=get(1)))
    #assert len(ref_seqs) > 1, "Need more than 1 reference sequence"
    ref_seqs = map(str.upper, ref_seqs)
    super_ref_seq, super_ref_date, super_ref_name = ref_seqs[0], ref_dates[0], ref_names[0]
    print(super_ref_name)
    print(super_ref_date)
    get_mutations = partial(hamming, super_ref_seq)
    def get_relative_info(seqs, dates, names):
        muts = map(get_mutations, seqs)
        dists = [(yr - super_ref_date).days for yr in dates]
        return muts, dists, names
    ref_muts, ref_dists, ref_names =  get_relative_info(ref_seqs, ref_dates, ref_names)
    query_muts, query_dists, query_names = get_relative_info(*get_seqs_and_dates(query_fn))
    do_plot(ref_dists, ref_muts, ref_names, query_dists, query_muts, query_names, save_path, html)
Пример #18
0
def test_iffy():
    assert map(iffy(_ % 2, _ * 2, _ / 2), [1,2,3,4]) == [2,1,6,2]
    assert map(iffy(_ % 2, _ * 2), [1,2,3,4]) == [2,2,6,4]
    assert map(iffy(_ * 2), [21, '', None]) == [42, '', None]
    assert map(iffy(_ % 2, _ * 2, None), [1,2,3,4]) == [2, None, 6, None]
Пример #19
0
def test_list_iter():
    assert is_list(py2.map(None, []))
    assert is_iter(py3.map(None, []))
    assert is_list(funcy.map(None, [])) == PY2
    assert is_iter(funcy.map(None, [])) == PY3
Пример #20
0
def _stringify_query():
    """
    Serializes query object, so that it can be used to create cache key.
    We can't just do pickle because order of keys in dicts is arbitrary,
    we can use str(query) which compiles it to SQL, but it's too slow,
    so we use json.dumps with sort_keys=True and object hooks.

    NOTE: I like this function no more than you, it's messy
          and pretty hard linked to django internals.
          I just don't have nicer solution for now.

          Probably the best way out of it is optimizing SQL generation,
          which would be valuable by itself. The problem with it is that
          any significant optimization will most likely require a major
          refactor of sql.Query class, which is a substantial part of ORM.
    """
    from datetime import datetime, date, time, timedelta
    from decimal import Decimal
    from django.db.models.expressions import ExpressionNode, F
    from django.db.models.fields import Field
    from django.db.models.fields.related import ManyToOneRel, OneToOneRel
    from django.db.models.sql.where import Constraint, WhereNode, ExtraWhere, \
                                           EverythingNode, NothingNode
    from django.db.models.sql import Query
    from django.db.models.sql.aggregates import Aggregate
    from django.db.models.sql.datastructures import Date
    from django.db.models.sql.expressions import SQLEvaluator

    attrs = {}

    # Try to not require geo libs
    try:
        from django.contrib.gis.db.models.sql.where import GeoWhereNode
    except: # either ImportError or GEOSException
        GeoWhereNode = WhereNode

    # A new things in Django 1.6
    try:
        from django.db.models.sql.where import EmptyWhere, SubqueryConstraint
        attrs[EmptyWhere] = ()
        attrs[SubqueryConstraint] = ('alias', 'columns', 'targets', 'query_object')
    except ImportError:
        pass

    # RawValue removed in Django 1.7
    try:
        from django.db.models.sql.datastructures import RawValue
        attrs[RawValue] = ('value',)
    except ImportError:
        pass

    # Moved in Django 1.7
    try:
        from django.contrib.contenttypes.fields import GenericRel
    except ImportError:
        from django.contrib.contenttypes.generic import GenericRel

    # New things in Django 1.7
    try:
        from django.db.models.lookups import Lookup
        from django.db.models.sql.datastructures import Col
        attrs[Lookup] = ('lhs', 'rhs')
        attrs[Col] = ('alias', 'target', 'source')
    except ImportError:
        class Lookup(object):
            pass

    attrs[WhereNode] = attrs[GeoWhereNode] = attrs[ExpressionNode] \
        = ('connector', 'negated', 'children')
    attrs[SQLEvaluator] = ('expression',)
    attrs[ExtraWhere] = ('sqls', 'params')
    attrs[Aggregate] = ('source', 'is_summary', 'col', 'extra')
    attrs[Date] = ('col', 'lookup_type')
    attrs[F] = ('name',)
    attrs[ManyToOneRel] = attrs[OneToOneRel] = attrs[GenericRel] = ('field',)
    attrs[EverythingNode] = attrs[NothingNode] = ()

    q = Query(None)
    q_keys = q.__dict__.keys()
    q_ignored = ['join_map', 'dupe_avoidance', '_extra_select_cache', '_aggregate_select_cache',
                 'used_aliases']
    attrs[Query] = tuple(sorted(set(q_keys) - set(q_ignored)))

    try:
        for k, v in attrs.items():
            attrs[k] = map(intern, v)
    except NameError:
        # No intern() in Python 3
        pass

    def encode_attrs(obj, cls=None):
        return (obj.__class__, [getattr(obj, attr) for attr in attrs[cls or obj.__class__]])

    def encode_object(obj):
        if isinstance(obj, set):
            return sorted(obj)
        elif isinstance(obj, type):
            return '%s.%s' % (obj.__module__, obj.__name__)
        elif hasattr(obj, '__uniq_key__'):
            return (obj.__class__, obj.__uniq_key__())
        elif isinstance(obj, (datetime, date, time, timedelta, Decimal)):
            return str(obj)
        elif isinstance(obj, Constraint):
            return (obj.alias, obj.col)
        elif isinstance(obj, Field):
            return (obj.model, obj.name)
        elif obj.__class__ in attrs:
            return encode_attrs(obj)
        elif isinstance(obj, QuerySet):
            return (obj.__class__, obj.query)
        elif isinstance(obj, Aggregate):
            return encode_attrs(obj, Aggregate)
        elif isinstance(obj, Query):
            return encode_attrs(obj, Query) # for custom subclasses of Query
        elif isinstance(obj, Lookup):
            return encode_attrs(obj, Lookup)
        # Fall back for unknown objects
        elif not STRICT_STRINGIFY and hasattr(obj, '__dict__'):
            return (obj.__class__, obj.__dict__)
        else:
            raise TypeError("Can't stringify %s" % repr(obj))

    def stringify_query(query):
        # HACK: Catch TypeError and reraise it as ValueError
        #       since django hides it and behave weird when gets a TypeError in Queryset.iterator()
        try:
            return json.dumps(query, default=encode_object, skipkeys=True,
                                     sort_keys=True, separators=(',', ':'))
        except TypeError as e:
            raise ValueError(*e.args)

    return stringify_query
Пример #21
0
def test_list_iter():
    assert is_list(py2.map(None, []))
    assert is_iter(py3.map(None, []))
    assert is_list(funcy.map(None, [])) == PY2
    assert is_iter(funcy.map(None, [])) == PY3
Пример #22
0
def do_plot(x1, y1, ref_names, x2, y2, query_names, save_path=None, html=True, \
            title='Mutations over time (days)', x_axis_label='days', y_axis_label='bases'):
    '''
    :param iterable x1: reference dates distances
    :param iterable y1: reference p-distances
    :param iterable x2: query dates diferences
    :param iterable y2: query p-distances
    :param str save_path: path to save image or None to open GTK if installed
    '''
    assert len(x1) > 0, "No reference dates to use"
    assert len(y2) > 0, "No reference p-distances to use"
    assert len(x2) > 0, "No query dates to use"
    assert len(y2) > 0, "No query p-distances to use"
    assert len(ref_names) == len(x1) and len(query_names) == len(x2)
    fig = plt.figure()
    ax = plt.subplot(111)
#    from matplotlib.dates import YearLocator, MonthLocator, DateFormatter
#    years = YearLocator()   # every year
#    months = MonthLocator()  # every month
#    yearsFmt = DateFormatter('%Y')
#    ax.xaxis.set_major_locator(years)
#    ax.xaxis.set_major_formatter(yearsFmt)
#    ax.xaxis.set_minor_locator(months)
    max_x = max(max(x1), max(x2))
    #legend_info = [mpatches.Patch(label=n, color=c) for n, c in legend.items()]
    """ http://stackoverflow.com/questions/4700614/how-to-put-the-legend-out-of-the-plot"""

    ref_info = zip(ref_names, x1, y1)
    query_info = zip(query_names, x2, y2)
    all_info = sorted(ref_info + query_info, key=lambda x: x[2], reverse=True)

    if save_path:
        fh = open(save_path+'.csv', 'wt')
    else:
        fh = sys.stdout
    fh.write('name,dates,p-dist\n')
    outcsv = csv.writer(fh)
    map(outcsv.writerow, all_info)
    if html:
        assert sys.version[:3] != '2.6', "Requires python 2.7 or higher to run bokeh."
        import bokeh.models as bkm
        import bokeh.plotting as bkp
        bokeh_tools = [bkm.WheelZoomTool(), bkm.PanTool(), bkm.BoxZoomTool(),
             bkm.PreviewSaveTool(), bkm.ResetTool(), bkm.BoxSelectTool(),
             bkm.ResizeTool()]
        ref_names = map('R: {0}'.format, ref_names)
        query_names = map('Q: {0}'.format, query_names)
        hover = bkm.HoverTool(tooltips=[("id", "@ids"),]) #   ("(days,muts)", "($x, $y)"),
        source1 = bkm.ColumnDataSource(data=dict(x=x1, y=y1,  ids=ref_names))
        source2 = bkm.ColumnDataSource(data=dict(x2=x2, y2=y2, ids=query_names))
        p = bkp.figure(plot_width=400, plot_height=400, tools=[hover]+bokeh_tools, title=title)
        p.circle('x', 'y', source=source1, line_color='gray', legend='reference')
        p.square('x2', 'y2', source=source2, fill_color='red', legend='query')
        if save_path:
            bkp.output_file(save_path + '.html')
            bkp.save(p)
        else: bkp.show(p)
    else:
        plot_muts(ax, x1, y1, plotkwargs=dict(label='references (blue)', color=legend['references'], marker='s'), polyfit=True, max_x=max_x, dist=None)
        plot_muts(ax, x2, y2, plotkwargs=dict(label='queries (red)', color=legend['queries']), dist=None)
        box = ax.get_position()
        ax.set_position([box.x0, box.y0, box.width * 0.8, box.height])
        #ax.legend(handles=legend_info, loc='center left', bbox_to_anchor=(1, 0.5))
        #ax.legend(loc='center left', bbox_to_anchor=(1, 0.5), framealpha=0)
        ax.legend(framealpha=0)
        plt.xlabel("days since Base reference")
        plt.ylabel("p-distance")
        if save_path:
            plt.savefig(save_path)
        else:
            plt.show()
Пример #23
0
 def get_relative_info(seqs, dates, names):
     muts = map(get_mutations, seqs)
     dists = [(yr - super_ref_date).days for yr in dates]
     return muts, dists, names
Пример #24
0
def get_seqs_and_dates(fn):
    fasta = SeqIO.parse(fn, format="fasta")
    info = [(str(seq.seq), seq.id, seq.description) for seq in fasta]
    seqs, ids, descriptions = zip(*info)
    dates = map(extract_date, ids)
    return seqs, dates, ids
Пример #25
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        # Lookups appeared in Django 1.7
        if isinstance(where, Lookup):
            # If where.lhs don't refer to a field then don't bother
            if not hasattr(where.lhs, 'target'):
                return [[SOME_COND]]
            # TODO: check if all of this are possible
            if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
                return [[SOME_COND]]

            attname = where.lhs.target.attname
            if isinstance(where, Exact):
                if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
                    return [[SOME_COND]]
                else:
                    return [[(where.lhs.alias, attname, where.rhs, True)]]
            elif isinstance(where, IsNull):
                return [[(where.lhs.alias, attname, None, where.rhs)]]
            elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
                return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
            else:
                return [[SOME_COND]]
        # Django 1.6 and earlier used tuples to encode conditions
        elif isinstance(where, tuple):
            constraint, lookup, annotation, value = where
            attname = attname_of(model, constraint.col)
            if isinstance(value, (QuerySet, Query, SQLEvaluator)):
                return [[SOME_COND]]
            elif lookup == 'exact':
                # TODO: check for non-serialized for both exact and in
                if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
                    return [[SOME_COND]]
                else:
                    return [[(constraint.alias, attname, value, True)]]
            elif lookup == 'isnull':
                return [[(constraint.alias, attname, None, value)]]
            elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
                return [[(constraint.alias, attname, v, True)] for v in value]
            else:
                return [[SOME_COND]]
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return [[SOME_COND]]
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #26
0
def _stringify_query():
    """
    Serializes query object, so that it can be used to create cache key.
    We can't just do pickle because order of keys in dicts is arbitrary,
    we can use str(query) which compiles it to SQL, but it's too slow,
    so we use json.dumps with sort_keys=True and object hooks.

    NOTE: I like this function no more than you, it's messy
          and pretty hard linked to django internals.
          I just don't have nicer solution for now.

          Probably the best way out of it is optimizing SQL generation,
          which would be valuable by itself. The problem with it is that
          any significant optimization will most likely require a major
          refactor of sql.Query class, which is a substantial part of ORM.
    """
    from datetime import datetime, date, time, timedelta
    from decimal import Decimal
    from django.db.models.expressions import ExpressionNode, F
    from django.db.models.fields import Field
    from django.db.models.fields.related import ManyToOneRel, OneToOneRel
    from django.db.models.sql.where import Constraint, WhereNode, ExtraWhere, \
                                           EverythingNode, NothingNode
    from django.db.models.sql import Query
    from django.db.models.sql.aggregates import Aggregate
    from django.db.models.sql.datastructures import Date
    from django.db.models.sql.expressions import SQLEvaluator

    attrs = {}

    # Try to not require geo libs
    try:
        from django.contrib.gis.db.models.sql.where import GeoWhereNode
    except: # either ImportError or GEOSException
        GeoWhereNode = WhereNode

    # A new things in Django 1.6
    try:
        from django.db.models.sql.where import EmptyWhere, SubqueryConstraint
        attrs[EmptyWhere] = ()
        attrs[SubqueryConstraint] = ('alias', 'columns', 'targets', 'query_object')
    except ImportError:
        pass

    # RawValue removed in Django 1.7
    try:
        from django.db.models.sql.datastructures import RawValue
        attrs[RawValue] = ('value',)
    except ImportError:
        pass

    # Moved in Django 1.7
    try:
        from django.contrib.contenttypes.fields import GenericRel
    except ImportError:
        from django.contrib.contenttypes.generic import GenericRel

    # New things in Django 1.7
    try:
        from django.db.models.lookups import Lookup
        from django.db.models.sql.datastructures import Col
        attrs[Lookup] = ('lhs', 'rhs')
        attrs[Col] = ('alias', 'target', 'source')
    except ImportError:
        class Lookup(object):
            pass

    attrs[WhereNode] = attrs[GeoWhereNode] = attrs[ExpressionNode] \
        = ('connector', 'negated', 'children')
    attrs[SQLEvaluator] = ('expression',)
    attrs[ExtraWhere] = ('sqls', 'params')
    attrs[Aggregate] = ('source', 'is_summary', 'col', 'extra')
    attrs[Date] = ('col', 'lookup_type')
    attrs[F] = ('name',)
    attrs[ManyToOneRel] = attrs[OneToOneRel] = attrs[GenericRel] = ('field',)
    attrs[EverythingNode] = attrs[NothingNode] = ()

    q = Query(None)
    q_keys = q.__dict__.keys()
    q_ignored = ['join_map', 'dupe_avoidance', '_extra_select_cache', '_aggregate_select_cache',
                 'used_aliases']
    attrs[Query] = tuple(sorted(set(q_keys) - set(q_ignored)))

    try:
        for k, v in attrs.items():
            attrs[k] = map(intern, v)
    except NameError:
        # No intern() in Python 3
        pass

    def encode_attrs(obj, cls=None):
        return (obj.__class__, [getattr(obj, attr) for attr in attrs[cls or obj.__class__]])

    def encode_object(obj):
        if isinstance(obj, set):
            return sorted(obj)
        elif isinstance(obj, type):
            return '%s.%s' % (obj.__module__, obj.__name__)
        elif hasattr(obj, '__uniq_key__'):
            return (obj.__class__, obj.__uniq_key__())
        elif isinstance(obj, (datetime, date, time, timedelta, Decimal)):
            return str(obj)
        elif isinstance(obj, Constraint):
            return (obj.alias, obj.col)
        elif isinstance(obj, Field):
            return (obj.model, obj.name)
        elif obj.__class__ in attrs:
            return encode_attrs(obj)
        elif isinstance(obj, QuerySet):
            return (obj.__class__, obj.query)
        elif isinstance(obj, Aggregate):
            return encode_attrs(obj, Aggregate)
        elif isinstance(obj, Query):
            return encode_attrs(obj, Query) # for custom subclasses of Query
        elif isinstance(obj, Lookup):
            return encode_attrs(obj, Lookup)
        # Fall back for unknown objects
        elif not STRICT_STRINGIFY and hasattr(obj, '__dict__'):
            return (obj.__class__, obj.__dict__)
        else:
            raise TypeError("Can't stringify %s" % repr(obj))

    def stringify_query(query):
        # HACK: Catch TypeError and reraise it as ValueError
        #       since django hides it and behave weird when gets a TypeError in Queryset.iterator()
        try:
            return json.dumps(query, default=encode_object, skipkeys=True,
                                     sort_keys=True, separators=(',', ':'))
        except TypeError as e:
            raise ValueError(*e.args)

    return stringify_query
Пример #27
0
 def required_star(f, *seqs):
     return map(f, *seqs)
Пример #28
0
    def _dnf(where):
        """
        Constructs DNF of where tree consisting of terms in form:
            (alias, attribute, value, negation)
        meaning `alias.attribute = value`
         or `not alias.attribute = value` if negation is False

        Any conditions other then eq are dropped.
        """
        # Lookups appeared in Django 1.7
        if isinstance(where, Lookup):
            # If where.lhs don't refer to a field then don't bother
            if not hasattr(where.lhs, 'target'):
                return SOME_TREE
            # Don't bother with complex right hand side either
            if isinstance(where.rhs, (QuerySet, Query, SQLEvaluator)):
                return SOME_TREE
            # Skip conditions on non-serialized fields
            if isinstance(where.lhs.target, NOT_SERIALIZED_FIELDS):
                return SOME_TREE

            attname = where.lhs.target.attname
            if isinstance(where, Exact):
                return [[(where.lhs.alias, attname, where.rhs, True)]]
            elif isinstance(where, IsNull):
                return [[(where.lhs.alias, attname, None, where.rhs)]]
            elif isinstance(where, In) and len(where.rhs) < LONG_DISJUNCTION:
                return [[(where.lhs.alias, attname, v, True)] for v in where.rhs]
            else:
                return SOME_TREE
        # Django 1.6 and earlier used tuples to encode conditions
        elif isinstance(where, tuple):
            constraint, lookup, annotation, value = where
            # Don't bother with complex right hand side
            if isinstance(value, (QuerySet, Query, SQLEvaluator)):
                return SOME_TREE
            # Skip conditions on non-serialized fields
            if isinstance(constraint.field, NOT_SERIALIZED_FIELDS):
                return SOME_TREE

            attname = attname_of(model, constraint.col)
            if lookup == 'isnull':
                return [[(constraint.alias, attname, None, value)]]
            elif lookup == 'exact':
                return [[(constraint.alias, attname, value, True)]]
            elif lookup == 'in' and len(value) < LONG_DISJUNCTION:
                return [[(constraint.alias, attname, v, True)] for v in value]
            else:
                return SOME_TREE
        elif isinstance(where, EverythingNode):
            return [[]]
        elif isinstance(where, NothingNode):
            return []
        elif isinstance(where, (ExtraWhere, SubqueryConstraint)):
            return SOME_TREE
        elif len(where) == 0:
            return [[]]
        else:
            chilren_dnfs = map(_dnf, where.children)

            if len(chilren_dnfs) == 0:
                return [[]]
            elif len(chilren_dnfs) == 1:
                result = chilren_dnfs[0]
            else:
                # Just unite children joined with OR
                if where.connector == OR:
                    result = cat(chilren_dnfs)
                # Use Cartesian product to AND children
                else:
                    result = map(cat, product(*chilren_dnfs))

            # Negating and expanding brackets
            if where.negated:
                result = [map(negate, p) for p in product(*result)]

            return result
Пример #29
0
def test_juxt():
    assert juxt(__add__, __sub__)(10, 2) == [12, 8]
    assert map(juxt(_ + 1, _ - 1), [2, 3]) == [[3, 1], [4, 2]]