def create_module(name, attrs, cls_attrs=None, pkg=None,
base=LazyModule, prepare_attr=None):
fqdn = '.'.join([pkg.__name__, name]) if pkg else name
cls_attrs = {} if cls_attrs is None else cls_attrs
pkg, _, modname = name.rpartition('.')
cls_attrs['__module__'] = pkg
attrs = {
attr_name: (prepare_attr(attr) if prepare_attr else attr)
for attr_name, attr in items(attrs)
}
module = sys.modules[fqdn] = type(
bytes_if_py2(modname), (base,), cls_attrs)(bytes_if_py2(name))
module.__dict__.update(attrs)
return module
def subclass_with_self(self,
Class,
name=None,
attribute='app',
reverse=None,
keep_reduce=False,
**kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class, ), attrs)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class.
App-compatible means that the class has a class attribute that
provides the default app it should use, for example:
``class Foo: app = None``.
Arguments:
Class (type): The app-compatible class to subclass.
name (str): Custom name for the target class.
attribute (str): Name of the attribute holding the app,
Default is 'app'.
reverse (str): Reverse path to this object used for pickling
purposes. For example, to get ``app.AsyncResult``,
use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation won't be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict(
{attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
:keyword reverse: Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
:keyword keep_reduce: If enabled a custom ``__reduce__`` implementation
will not be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
Arguments:
Class (type): The app-compatible class to subclass.
name (str): Custom name for the target class.
attribute (str): Name of the attribute holding the app,
Default is 'app'.
reverse (str): Reverse path to this object used for pickling
purposes. E.g. for ``app.AsyncResult`` use ``"AsyncResult"``.
keep_reduce (bool): If enabled a custom ``__reduce__``
implementation won't be provided.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict(
{attribute: self},
__module__=Class.__module__,
__doc__=Class.__doc__,
**kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def subclass_with_self(self, Class, name=None, attribute='app',
reverse=None, keep_reduce=False, **kw):
"""Subclass an app-compatible class by setting its app attribute
to be this app instance.
App-compatible means that the class has a class attribute that
provides the default app it should use, e.g.
``class Foo: app = None``.
:param Class: The app-compatible class to subclass.
:keyword name: Custom name for the target class.
:keyword attribute: Name of the attribute holding the app,
default is 'app'.
"""
Class = symbol_by_name(Class)
reverse = reverse if reverse else Class.__name__
def __reduce__(self):
return _unpickle_appattr, (reverse, self.__reduce_args__())
attrs = dict({attribute: self}, __module__=Class.__module__,
__doc__=Class.__doc__, **kw)
if not keep_reduce:
attrs['__reduce__'] = __reduce__
return type(bytes_if_py2(name or Class.__name__), (Class,), attrs)
def __init__(self, key=None, cert=None, cert_store=None,
digest='sha1', serializer='json'):
self._key = key
self._cert = cert
self._cert_store = cert_store
self._digest = bytes_if_py2(digest)
self._serializer = serializer
def mock_memcache(self):
memcache = types.ModuleType(bytes_if_py2('memcache'))
memcache.Client = MemcachedClient
memcache.Client.__module__ = memcache.__name__
prev, sys.modules['memcache'] = sys.modules.get('memcache'), memcache
try:
yield True
finally:
if prev is not None:
sys.modules['memcache'] = prev
def mock_memcache(self):
memcache = types.ModuleType(bytes_if_py2('memcache'))
memcache.Client = MemcachedClient
memcache.Client.__module__ = memcache.__name__
prev, sys.modules['memcache'] = sys.modules.get('memcache'), memcache
try:
yield True
finally:
if prev is not None:
sys.modules['memcache'] = prev
def mock_pylibmc(self):
pylibmc = types.ModuleType(bytes_if_py2('pylibmc'))
pylibmc.Client = MemcachedClient
pylibmc.Client.__module__ = pylibmc.__name__
prev = sys.modules.get('pylibmc')
sys.modules['pylibmc'] = pylibmc
try:
yield True
finally:
if prev is not None:
sys.modules['pylibmc'] = prev
def mock_pylibmc(self):
pylibmc = types.ModuleType(bytes_if_py2('pylibmc'))
pylibmc.Client = MemcachedClient
pylibmc.Client.__module__ = pylibmc.__name__
prev = sys.modules.get('pylibmc')
sys.modules['pylibmc'] = pylibmc
try:
yield True
finally:
if prev is not None:
sys.modules['pylibmc'] = prev
def get_compat_module(pkg, name):
def prepare(attr):
if isinstance(attr, string_t):
return Proxy(getappattr, (attr,))
return attr
attrs = COMPAT_MODULES[pkg.__name__][name]
if isinstance(attrs, string_t):
fqdn = '.'.join([pkg.__name__, name])
module = sys.modules[fqdn] = import_module(attrs)
return module
attrs[bytes_if_py2('__all__')] = list(attrs)
return create_module(name, dict(attrs), pkg=pkg, prepare_attr=prepare)
def test_find_app(self, app):
cmd = MockCommand(app=app)
with patch('celery.utils.imports.symbol_by_name') as sbn:
from types import ModuleType
x = ModuleType(bytes_if_py2('proj'))
def on_sbn(*args, **kwargs):
def after(*args, **kwargs):
x.app = 'quick brown fox'
x.__path__ = None
return x
sbn.side_effect = after
return x
sbn.side_effect = on_sbn
x.__path__ = [True]
assert cmd.find_app('proj') == 'quick brown fox'
def _default_cls_attr(name, type_, cls_value):
# Proxy uses properties to forward the standard
# class attributes __module__, __name__ and __doc__ to the real
# object, but these needs to be a string when accessed from
# the Proxy class directly. This is a hack to make that work.
# -- See Issue #1087.
def __new__(cls, getter):
instance = type_.__new__(cls, cls_value)
instance.__getter = getter
return instance
def __get__(self, obj, cls=None):
return self.__getter(obj) if obj is not None else self
return type(bytes_if_py2(name), (type_,), {
'__new__': __new__, '__get__': __get__,
})
def test_find_app(self):
cmd = MockCommand(app=self.app)
with patch("celery.bin.base.symbol_by_name") as sbn:
from types import ModuleType
x = ModuleType(bytes_if_py2("proj"))
def on_sbn(*args, **kwargs):
def after(*args, **kwargs):
x.app = "quick brown fox"
x.__path__ = None
return x
sbn.side_effect = after
return x
sbn.side_effect = on_sbn
x.__path__ = [True]
self.assertEqual(cmd.find_app("proj"), "quick brown fox")
def test_read_configuration(self):
from types import ModuleType
class ConfigModule(ModuleType):
pass
configname = os.environ.get('CELERY_CONFIG_MODULE') or 'celeryconfig'
celeryconfig = ConfigModule(bytes_if_py2(configname))
celeryconfig.imports = ('os', 'sys')
prevconfig = sys.modules.get(configname)
sys.modules[configname] = celeryconfig
try:
l = default.Loader(app=self.app)
l.find_module = Mock(name='find_module')
settings = l.read_configuration(fail_silently=False)
assert settings.imports == ('os', 'sys')
settings = l.read_configuration(fail_silently=False)
assert settings.imports == ('os', 'sys')
l.on_worker_init()
finally:
if prevconfig:
sys.modules[configname] = prevconfig
def test_read_configuration(self):
from types import ModuleType
class ConfigModule(ModuleType):
pass
configname = os.environ.get("CELERY_CONFIG_MODULE") or "celeryconfig"
celeryconfig = ConfigModule(bytes_if_py2(configname))
celeryconfig.imports = ("os", "sys")
prevconfig = sys.modules.get(configname)
sys.modules[configname] = celeryconfig
try:
l = default.Loader(app=self.app)
l.find_module = Mock(name="find_module")
settings = l.read_configuration(fail_silently=False)
self.assertTupleEqual(settings.imports, ("os", "sys"))
settings = l.read_configuration(fail_silently=False)
self.assertTupleEqual(settings.imports, ("os", "sys"))
l.on_worker_init()
finally:
if prevconfig:
sys.modules[configname] = prevconfig
def test_read_configuration(self):
from types import ModuleType
class ConfigModule(ModuleType):
pass
configname = os.environ.get('CELERY_CONFIG_MODULE') or 'celeryconfig'
celeryconfig = ConfigModule(bytes_if_py2(configname))
celeryconfig.imports = ('os', 'sys')
prevconfig = sys.modules.get(configname)
sys.modules[configname] = celeryconfig
try:
l = default.Loader(app=self.app)
l.find_module = Mock(name='find_module')
settings = l.read_configuration(fail_silently=False)
assert settings.imports == ('os', 'sys')
settings = l.read_configuration(fail_silently=False)
assert settings.imports == ('os', 'sys')
l.on_worker_init()
finally:
if prevconfig:
sys.modules[configname] = prevconfig
from celery.utils.functional import pass1
from celery.utils.serialization import subclass_exception
from celery.utils.serialization import find_pickleable_exception as fnpe
from celery.utils.serialization import UnpickleableExceptionWrapper
from celery.utils.serialization import get_pickleable_exception as gpe
class wrapobject(object):
def __init__(self, *args, **kwargs):
self.args = args
if sys.version_info[0] == 3 or getattr(sys, 'pypy_version_info', None):
Oldstyle = None
else:
Oldstyle = types.ClassType(bytes_if_py2('Oldstyle'), (), {})
Unpickleable = subclass_exception(
bytes_if_py2('Unpickleable'),
KeyError,
'foo.module',
)
Impossible = subclass_exception(
bytes_if_py2('Impossible'),
object,
'foo.module',
)
Lookalike = subclass_exception(
bytes_if_py2('Lookalike'),
wrapobject,
'foo.module',
)
def test_sign(self):
pkey = PrivateKey(KEY1)
pkey.sign('test', bytes_if_py2('sha1'))
with pytest.raises(ValueError):
pkey.sign('test', bytes_if_py2('unknown'))
def subclass_exception(name, parent, module): # noqa
return type(bytes_if_py2(name), (parent,), {'__module__': module})
def subclass_exception(name, parent, module): # noqa
return type(bytes_if_py2(name), (parent, ), {'__module__': module})
def test_must_have_run_every(self):
with pytest.raises(NotImplementedError):
type(bytes_if_py2('Foo'), (PeriodicTask, ), {
'__module__': __name__,
})
def test_qualname():
Class = type(bytes_if_py2('Fox'), (object,), {
'__module__': 'quick.brown',
})
assert qualname(Class) == 'quick.brown.Fox'
assert qualname(Class()) == 'quick.brown.Fox'
def __repr__(self):
return bytes_if_py2('step:{0.name}{{{0.requires!r}}}'.format(self))
def __str__(self):
return bytes_if_py2(self.name)
def test_must_have_run_every(self):
with pytest.raises(NotImplementedError):
type(bytes_if_py2('Foo'), (PeriodicTask,), {
'__module__': __name__,
})
def test_qualname(self):
Class = type(bytes_if_py2('Fox'), (object,), {
'__module__': 'quick.brown',
})
self.assertEqual(qualname(Class), 'quick.brown.Fox')
self.assertEqual(qualname(Class()), 'quick.brown.Fox')
from celery.result import result_from_tuple
from celery.utils import uuid
from celery.utils.functional import pass1
from celery.tests.case import ANY, AppCase, Case, Mock, call, patch, skip
class wrapobject(object):
def __init__(self, *args, **kwargs):
self.args = args
if sys.version_info[0] == 3 or getattr(sys, "pypy_version_info", None):
Oldstyle = None
else:
Oldstyle = types.ClassType(bytes_if_py2("Oldstyle"), (), {})
Unpickleable = subclass_exception(bytes_if_py2("Unpickleable"), KeyError, "foo.module")
Impossible = subclass_exception(bytes_if_py2("Impossible"), object, "foo.module")
Lookalike = subclass_exception(bytes_if_py2("Lookalike"), wrapobject, "foo.module")
class test_nulldict(Case):
def test_nulldict(self):
x = _nulldict()
x["foo"] = 1
x.update(foo=1, bar=2)
x.setdefault("foo", 3)
class test_serialization(AppCase):
def test_create_exception_cls(self):
def test_qualname(self):
Class = type(bytes_if_py2('Fox'), (object, ), {
'__module__': 'quick.brown',
})
self.assertEqual(qualname(Class), 'quick.brown.Fox')
self.assertEqual(qualname(Class()), 'quick.brown.Fox')
def __repr__(self):
return bytes_if_py2('<step: {0.alias}>'.format(self))
from celery.utils.serialization import subclass_exception
from celery.utils.serialization import find_pickleable_exception as fnpe
from celery.utils.serialization import UnpickleableExceptionWrapper
from celery.utils.serialization import get_pickleable_exception as gpe
class wrapobject(object):
def __init__(self, *args, **kwargs):
self.args = args
if sys.version_info[0] == 3 or getattr(sys, 'pypy_version_info', None):
Oldstyle = None
else:
Oldstyle = types.ClassType(bytes_if_py2('Oldstyle'), (), {})
Unpickleable = subclass_exception(
bytes_if_py2('Unpickleable'), KeyError, 'foo.module',
)
Impossible = subclass_exception(
bytes_if_py2('Impossible'), object, 'foo.module',
)
Lookalike = subclass_exception(
bytes_if_py2('Lookalike'), wrapobject, 'foo.module',
)
class test_nulldict:
def test_nulldict(self):
x = _nulldict()
def test_sign(self):
pkey = PrivateKey(KEY1)
pkey.sign('test', bytes_if_py2('sha1'))
with pytest.raises(ValueError):
pkey.sign('test', bytes_if_py2('unknown'))