def generate_task_ids(self, job, kwargs):
'''An internal method to generate task unique identifiers.
:parameter job: The :class:`.Job` creating the task.
:parameter kwargs: dictionary of key-valued parameters passed to the
:ref:`job callable <job-callable>` method.
:return: a two-elements tuple containing the unique id and an
identifier for overlapping tasks if the :attr:`.Job.can_overlap`
results in ``False``.
Called by the :ref:`TaskBackend <apps-taskqueue-backend>` when
creating a new task.
'''
can_overlap = job.can_overlap
if hasattr(can_overlap, '__call__'):
can_overlap = can_overlap(**kwargs)
tid = gen_unique_id()
if can_overlap:
return tid, None
else:
if kwargs:
kw = ('%s=%s' % (k, kwargs[k]) for k in sorted(kwargs))
name = '%s %s' % (self.name, ', '.join(kw))
else:
name = self.name
return tid, sha1(name.encode('utf-8')).hexdigest()
def generate_task_ids(self, job, kwargs):
'''An internal method to generate task unique identifiers.
:parameter job: The :class:`.Job` creating the task.
:parameter kwargs: dictionary of key-valued parameters passed to the
:ref:`job callable <job-callable>` method.
:return: a two-elements tuple containing the unique id and an
identifier for overlapping tasks if the :attr:`.Job.can_overlap`
results in ``False``.
Called by the :ref:`TaskBackend <apps-taskqueue-backend>` when
creating a new task.
'''
can_overlap = job.can_overlap
if hasattr(can_overlap, '__call__'):
can_overlap = can_overlap(**kwargs)
tid = gen_unique_id()
if can_overlap:
return tid, None
else:
if kwargs:
kw = ('%s=%s' % (k, kwargs[k]) for k in sorted(kwargs))
name = '%s %s' % (self.name, ', '.join(kw))
else:
name = self.name
return tid, sha1(name.encode('utf-8')).hexdigest()
def _spawn_actor(cls, monitor, cfg=None, name=None, aid=None, **kw):
# Internal function which spawns a new Actor and return its
# ActorProxyMonitor.
# *cls* is the Actor class
# *monitor* can be either the ariber or a monitor
kind = None
if issubclass(cls, PoolMixin):
kind = 'monitor'
if monitor:
params = monitor.actorparams()
name = params.pop('name', name)
aid = params.pop('aid', aid)
cfg = params.pop('cfg', cfg)
# get config if not available
if cfg is None:
if monitor:
cfg = monitor.cfg.copy()
else:
cfg = Config()
if not monitor: # monitor not available, this is the arbiter
if kind != 'monitor':
raise TypeError('class %s not a valid monitor' % cls)
kind = 'arbiter'
params = {}
if not cfg.exc_id:
if not aid:
aid = gen_unique_id()[:8]
cfg.set('exc_id', aid)
#
for key, value in iteritems(kw):
if key in cfg.settings:
cfg.set(key, value)
else:
params[key] = value
#
if monitor:
if not kind:
if not issubclass(cls, Actor):
raise TypeError('Class %s not a valid actor.' % cls)
kind = cfg.concurrency
if not kind:
raise TypeError('Cannot spawn class %s. not a valid concurrency.'
% cls)
actor_proxy = concurrency(kind, cls, monitor, cfg, name=name,
aid=aid, **params)
# Add to the list of managed actors if this is a remote actor
if isinstance(actor_proxy, Actor):
return actor_proxy
else:
actor_proxy.monitor = monitor
monitor.managed_actors[actor_proxy.aid] = actor_proxy
future = actor_proxy_future(actor_proxy)
actor_proxy.start()
return future
def create_id(self, kwargs):
'''Create a unique id for a task.
Called by the :class:`.TaskBackend` when a new task is about to be
queued.
:parameter kwargs: dictionary of parameters passed to the
:ref:`job callable method <job-callable>`.
'''
return gen_unique_id()[:8]
def make(self, kind, actor_class, monitor, cfg, name=None, aid=None, **kw):
self.__class__._creation_counter += 1
self.aid = aid or gen_unique_id()[:8]
self.age = self.__class__._creation_counter
self.name = name or actor_class.__name__.lower()
self.kind = kind
self.cfg = cfg
self.actor_class = actor_class
self.params = kw
self.params['monitor'] = monitor
return self.get_actor()
def make(self, kind, actor_class, monitor, cfg, name=None, aid=None, **kw):
self.__class__._creation_counter += 1
self.aid = aid or gen_unique_id()[:8]
self.age = self.__class__._creation_counter
self.name = name or actor_class.__name__.lower()
self.kind = kind
self.cfg = cfg
self.actor_class = actor_class
self.params = kw
self.params['monitor'] = monitor
return self.get_actor()
def make(self, kind, actor_class, monitor, commands_set, cfg, name=None,
aid=None, **params):
self.__class__._creation_counter += 1
self.aid = aid or gen_unique_id()[:8]
self.age = self.__class__._creation_counter
self.name = name or actor_class.__name__.lower()
self.kind = kind
self.commands_set = commands_set
self.cfg = cfg
self.actor_class = actor_class
self.params = params
return self.get_actor(monitor)
def command(cls, command, sender, target, args, kwargs):
command = get_command(command)
data = {'command': command.__name__,
'sender': actor_identity(sender),
'target': actor_identity(target),
'args': args if args is not None else (),
'kwargs': kwargs if kwargs is not None else {}}
if command.ack:
future = Future()
data['ack'] = gen_unique_id()[:8]
else:
future = None
return cls(data, future)
def command(cls, command, sender, target, args, kwargs):
command = get_command(command)
data = {'command': command.__name__,
'sender': actorid(sender),
'target': actorid(target),
'args': args if args is not None else (),
'kwargs': kwargs if kwargs is not None else {}}
if command.ack:
future = Future()
data['ack'] = gen_unique_id()[:8]
else:
future = None
return cls(data, future)
def command(cls, command, sender, target, args, kwargs, address=None,
timeout=None):
command = get_command(command)
data = {'command': command.__name__,
'sender': actorid(sender),
'target': actorid(target),
'args': args if args is not None else (),
'kwargs': kwargs if kwargs is not None else {}}
if command.ack:
future = Deferred()
data['ack'] = gen_unique_id()[:8]
else:
future = None
return cls(data, future, address, timeout)
def setUpClass(cls):
cls.exc_id = gen_unique_id()[:8]
name = cls.__name__.lower()
argv = [
__file__, 'pulse', '-b', '127.0.0.1:0', '--concurrency',
cls.concurrency, '--exc-id', cls.exc_id, '--pulse-app-name', name,
'--data-store', 'pulsar://127.0.0.1:6410/1'
]
cls.app_cfg = yield send('arbiter', 'run', start_server, name, argv)
assert cls.app_cfg.exc_id == cls.exc_id, "Bad execution id"
addr = cls.app_cfg.addresses[0]
cls.uri = 'http://{0}:{1}'.format(*addr)
cls.ws = 'ws://{0}:{1}/message'.format(*addr)
cls.http = http.HttpClient()
def setUpClass(cls):
cls.exc_id = gen_unique_id()[:8]
name = cls.__name__.lower()
argv = [__file__, 'pulse',
'-b', '127.0.0.1:0',
'--concurrency', cls.concurrency,
'--exc-id', cls.exc_id,
'--pulse-app-name', name,
'--data-store', 'pulsar://127.0.0.1:6410/1']
cls.app_cfg = yield send('arbiter', 'run', start_server, name, argv)
assert cls.app_cfg.exc_id == cls.exc_id, "Bad execution id"
addr = cls.app_cfg.addresses[0]
cls.uri = 'http://{0}:{1}'.format(*addr)
cls.ws = 'ws://{0}:{1}/message'.format(*addr)
cls.http = http.HttpClient()
def spawn(cfg=None, **kwargs):
'''Spawn a new :class:`Actor` and return an :class:`ActorProxyDeferred`.
This method can be used from any :class:`Actor`.
If not in the :class:`Arbiter` domain, the method sends a request
to the :class:`Arbiter` to spawn a new actor.
Once the arbiter creates the actor it returns the ``proxy`` to the
original caller.
**Parameter kwargs**
These optional parameters are:
* ``actor_class`` a custom :class:`Actor` subclass
* ``aid`` the actor id
* ``name`` the actor name
* :ref:`actor hooks <actor-hooks>` such as ``start``, ``stopping``
and ``stop``
:return: an :class:`ActorProxyDeferred`.
A typical usage::
>>> def do_something(actor):
...
>>> a = spawn(start=do_something, ...)
>>> a.aid
'ba42b02b'
>>> a.called
True
>>> p = a.result
>>> p.address
('127.0.0.1', 46691)
'''
aid = gen_unique_id()[:8]
kwargs['aid'] = aid
actor = get_actor()
# The actor is not the Arbiter domain.
# We send a message to the Arbiter to spawn a new Actor
if not isinstance(actor, Arbiter):
# send the request to the arbiter
msg = actor.send('arbiter', 'spawn', **kwargs)
return proxy.ActorProxyDeferred(aid, msg)
else:
return actor.spawn(**kwargs)
def spawn(**kwargs):
'''Spawn a new :class:`.Actor` and return an :class:`.ActorProxyFuture`.
This method can be used from any :class:`.Actor`.
If not in the :class:`.Arbiter` domain, the method sends a request
to the :class:`.Arbiter` to spawn a new actor.
Once the arbiter creates the actor it returns the ``proxy`` to the
original caller.
**Parameter kwargs**
These optional parameters are:
* ``aid`` the actor id
* ``name`` the actor name
* :ref:`actor hooks <actor-hooks>` such as ``start``, ``stopping``
and ``stop``
* ``actor_class`` a custom :class:`.Actor` subclass (never used)
:return: an :class:`.ActorProxyFuture`.
A typical usage::
>>> def do_something(actor):
...
>>> a = spawn(start=do_something, ...)
>>> a.aid
'ba42b02b'
>>> a.called
True
>>> p = a.result()
>>> p.address
('127.0.0.1', 46691)
'''
aid = gen_unique_id()[:8]
kwargs['aid'] = aid
actor = get_actor()
# The actor is not the Arbiter domain.
# We send a message to the Arbiter to spawn a new Actor
if not isinstance(actor, Arbiter):
# send the request to the arbiter
future = actor.send('arbiter', 'spawn', **kwargs)
return actor_proxy_future(aid, future)
else:
return actor.spawn(**kwargs)
def spawn(cfg=None, **kwargs):
'''Spawn a new :class:`Actor` and return an :class:`ActorProxyDeferred`.
This method can be used from any :class:`Actor`.
If not in the :class:`Arbiter` domain,
the method send a request to the :class:`Arbiter` to spawn a new actor, once
the arbiter creates the actor it returns the proxy to the original caller.
**Parameter kwargs**
These optional parameters are:
* *actor_class* a custom :class:`Actor` subclass.
* *aid* the actor id
* *commands_set* the set of :ref:`remote commands <api-remote_commands>`
the :class:`Actor` can respond to.
:rtype: an :class:`ActorProxyDeferred`.
A typical usage::
>>> a = spawn()
>>> a.aid
'ba42b02b'
>>> a.called
True
>>> p = a.result
>>> p.address
('127.0.0.1', 46691)
'''
aid = gen_unique_id()[:8]
kwargs['aid'] = aid
actor = get_actor()
# The actor is not the Arbiter domain.
# We send a message to the Arbiter to spawn a new Actor
if not isinstance(actor, Arbiter):
msg = actor.send('arbiter', 'spawn', **kwargs)\
.add_callback(actor.link_actor)
return proxy.ActorProxyDeferred(aid, msg)
else:
return actor.spawn(**kwargs)
def setUpClass(cls):
cls.exc_id = gen_unique_id()[:8]
name = cls.__name__.lower()
argv = [
__file__,
"pulse",
"--bind",
"127.0.0.1:0",
"--concurrency",
cls.concurrency,
"--exc-id",
cls.exc_id,
"--pulse-app-name",
name,
"--data-store",
cls.data_store(),
]
cls.app_cfg = yield send("arbiter", "run", start_server, name, argv)
assert cls.app_cfg.exc_id == cls.exc_id, "Bad execution id"
addr = cls.app_cfg.addresses[0]
cls.uri = "http://{0}:{1}".format(*addr)
cls.ws = "ws://{0}:{1}/message".format(*addr)
cls.http = http.HttpClient()
def send_message(self, msg):
id = to_bytes(gen_unique_id()[:8])
self.requests[id] = Deferred()
self.transport.write(id + to_bytes(msg) + self.separator)
return self.requests[id]
def create_task_id():
return gen_unique_id()[:8]
from pulsar.utils.security import gen_unique_id
from pulsar.apps.data import odm
from pulsar.apps.tasks import Task
models = odm.Mapper('couchdb://127.0.0.1:5984/test')
store = models.default_store
try:
ok = store.delete_database()
except Exception:
pass
ok = store.create_database()
models.register(Task)
ok = models.create_tables()
tasks = models.task
task1 = tasks.create(name='bla', id=gen_unique_id())
assert task1.name == 'bla'
task2 = tasks.create(name='foo', id=gen_unique_id())
assert task2.name == 'foo'
task3 = tasks.create(name='foo', id=gen_unique_id())
assert task3.name == 'foo'
objs = store.table_info(Task)
ok = models.drop_tables()
def makeid(self):
'''Can be re-implemented by your own Proxy'''
return gen_unique_id()
__test__ = False
try:
from example.luxweb.settings import *
except ImportError:
import sys
print("Add a settings file in the example.luxweb module")
sys.exit(1)
EXTENSIONS = ["lux.extensions.base"]
from stdnet import getdb
from pulsar.utils.security import gen_unique_id
c = getdb(DATASTORE[""], db=7, namespace="luxtest:%s:" % gen_unique_id()[:8])
DATASTORE = {"": c.connection_string}
def create_task_id():
return gen_unique_id()[:8]
def pubsub(self, tag=None, **kwargs):
tag = tag or gen_unique_id()
ps = PubSub(self.backend(tag), **kwargs)
self._pubsub.append(ps)
return ps
def pubsub(self, tag=None, **kwargs):
tag = tag or gen_unique_id()
ps = PubSub(self.backend(tag), **kwargs)
self._pubsub.append(ps)
return ps
def __call__(self, msg):
id = to_bytes(gen_unique_id()[:8])
self.requests[id] = d = Deferred()
self.transport.write(id + to_bytes(msg) + self.separator)
return d
__test__ = False
try:
from example.luxweb.settings import *
except ImportError:
import sys
print('Add a settings file in the example.luxweb module')
sys.exit(1)
EXTENSIONS = ['lux.extensions.base']
from stdnet import getdb
from pulsar.utils.security import gen_unique_id
c = getdb(DATASTORE[''], db=7, namespace='luxtest:%s:' % gen_unique_id()[:8])
DATASTORE = {'': c.connection_string}
from pulsar.utils.security import gen_unique_id
from pulsar.apps.data import odm
from pulsar.apps.tasks import Task
models = odm.Mapper('couchdb://127.0.0.1:5984/test')
store = models.default_store
try:
ok = store.delete_database()
except Exception:
pass
ok = store.create_database()
models.register(Task)
ok = models.create_tables()
tasks = models.task
task1 = tasks.create(name='bla', id=gen_unique_id())
assert task1.name == 'bla'
task2 = tasks.create(name='foo', id=gen_unique_id())
assert task2.name == 'foo'
task3 = tasks.create(name='foo', id=gen_unique_id())
assert task3.name == 'foo'
objs = store.table_info(Task)
store.
ok = models.drop_tables()
def create_aid():
return gen_unique_id()[:8]
def send_message(self, msg):
id = to_bytes(gen_unique_id()[:8])
self.requests[id] = Deferred()
self.transport.write(id + to_bytes(msg) + self.separator)
return self.requests[id]
