def log(self, message):
if hasattr(Tasklet.current(), 'jobguid'):
jobguid = Tasklet.current().jobguid
q.workflowengine.jobmanager.appendJobLog(jobguid, message)
else:
# No job in the context, do nothing
pass
return True
def log(self, message):
if hasattr(Tasklet.current(), 'jobguid'):
jobguid = Tasklet.current().jobguid
q.workflowengine.jobmanager.appendJobLog(jobguid, message)
else:
# No job in the context, do nothing
pass
return True
def startRootobjectAction(self, rootobjectname, actionname, params, executionparams={}, jobguid=None):
"""
Starts a given RootObject Action. Uses the tasklet engine to run the matching tasklets.
The current job will be determined automatically, it can however be overriden by setting jobguid.
The action will be executed in a new job that is the newest child of the current job, the new job will inherit the following properties of the job: name, description, userErrormsg, internalErrormsg and maxduration.
The executionparams passed to the function will override the properties of the job (if provided).
.
The guid of the job in which the action is executed, will be added to params: params['jobguid'].
The tasklets executing in the Actor Action have to add their result in params['result']. The dictionary returned by startActorAction contains this result, under the 'result' key.
@param rootobjectname: Name of the rootobject of the root object action.
@type type: string
@param actionnmame: Name of the action to execute on the rootobject.
@type actionname: string
@param params: Dictionary containing all parameters required to execute this actions.
@type params: dictionary
@param executionParams: Dictionary can following keys: name, description, userErrormsg, internalErrormsg, maxduration, wait, timetostart, priority, clouduserguid, rootobjecttype, rootobjectguid
@type executionParams: dictionary
@param jobguid: Optional parameter, can be used to override the current job.
@type jobguid: guid
@return: dictionary with action result as result and the action's jobguid: {'result': <result>, 'jobguid': guid}
@rtype: dictionary
@raise ActionNotFoundException: In case no actions are found for the specified rootobjectname and actoraction
@raise e: In case an error occurred, exception is raised
"""
if len(self.__taskletEngine.find(tags=(rootobjectname, actionname), path=RootobjectActionTaskletPath)) == 0:
raise ActionNotFoundException("RootobjectAction", rootobjectname, actionname)
#SETUP THE JOB AND THE PARAMS
currentjobguid = jobguid or (hasattr(Tasklet.current(), 'jobguid') and Tasklet.current().jobguid) or None
if q.workflowengine.jobmanager.isKilled(currentjobguid):
raise Exception("Can't create child jobs: the job is killed !")
params['jobguid'] = jobguid = q.workflowengine.jobmanager.createJob(currentjobguid, rootobjectname+"."+actionname, executionparams, params=params)
#START A NEW TASKLET FOR THE JOB
q.workflowengine.jobmanager.startJob(jobguid)
tasklet = Tasklet.new(self.__execute)(Tasklet.current(), jobguid, params, (rootobjectname, actionname), RootobjectActionTaskletPath)
#WAIT FOR THE ANSWER
(msg, args, kwargs) = Tasklet.receive().next()
if msg.match(MSG_ACTION_NOWAIT):
return { 'jobguid':jobguid, 'result':None }
if msg.match(MSG_ACTION_RETURN):
return { 'jobguid':jobguid, 'result':params.get('result')}
elif msg.match(MSG_ACTION_EXCEPTION):
raise args[0]
def __execute(self, parentTasklet, jobguid, params, tags, path):
#SETUP THE CONTEXT
Tasklet.current().jobguid = jobguid
wait = q.workflowengine.jobmanager.shouldWait(jobguid)
Tasklet.current().tags = tags
if wait is False: MSG_ACTION_NOWAIT.send(parentTasklet)()
#EXECUTE THE TASKLETS
try:
self.__taskletEngine.execute(params, tags=tags, path=path)
except Exception, e:
q.workflowengine.jobmanager.setJobDied(jobguid, e)
if wait is True: MSG_ACTION_EXCEPTION.send(parentTasklet)(WFLException.create(e,jobguid))
def executeScript(self, agentguid, actionname, scriptpath, params, executionparams={}, jobguid=None):
"""
Execute a script on an agent. The action will be executed in a new job that is the newest child of the current job.
The new job will inherit the following properties of the job: name, description, userErrormsg, internalErrormsg, maxduration and customerguid.
The executionparams passed to the function will override the properties of the job (if provided).
@param agentguid: Guid of the agent on which the script will be executed.
@type agentguid: guid
@param actionname: The name of the action, will filled in, in the job
@param actionname: string
@param scriptpath: The path of the script, on the server.
@type scriptpath: string
@param params: Dictionary containing all parameters required to execute this script.
@type params: dictionary
@param executionParams: Dictionary can following keys: name, description, userErrormsg, internalErrormsg, maxduration, wait, timetostart, priority
@type executionParams: dictionary
@param jobguid: Optional parameter, can be used to override the current job.
@type jobguid: guid
@return: Dictionary with action result as result and the action's jobguid: {'result': <result>, 'jobguid': guid}
@rtype: dictionary
@raise IOError: If the scriptpath can't be read.
@raise TimeOutException: If the agent did not respond within the maxduration: the script will be killed and the exception will be raised.
@raise AgentNotAvailableException: If the agent is not available when starting the script, the script is not be started.
@raise ScriptFailedException: If an exception occurres on the agent while executing the script.
"""
#SETUP THE JOB AND THE PARAMS
currentjobguid = jobguid or Tasklet.current().jobguid
if q.workflowengine.jobmanager.isKilled(currentjobguid):
raise Exception("Can't create child jobs: the job is killed !")
params['jobguid'] = jobguid = q.workflowengine.jobmanager.createJob(currentjobguid, actionname, executionparams, agentguid, params=params)
#START A NEW TASKLET FOR THE JOB
q.workflowengine.jobmanager.startJob(jobguid)
tasklet = Tasklet.new(self.__execute)(Tasklet.current(), jobguid, agentguid, scriptpath, params)
#WAIT FOR THE ANSWER
(msg, args, kwargs) = Tasklet.receive().next()
if msg.match(MSG_ACTION_NOWAIT):
return { 'jobguid':jobguid, 'result':None }
if msg.match(MSG_ACTION_RETURN):
return args[0]
elif msg.match(MSG_ACTION_EXCEPTION):
raise args[0]
def register(self, name, task=None):
if task is None:
task = Tasklet.current()
task_id = id(task)
self._task_by_name[name] = task
self._task_id_by_task[task] = task_id
self._task_by_task_id[task_id] = task
def _read_response(self, reader):
cmd, block_channel = self._response_queue.popleft(True)
try:
if cmd == 'get':
result = {} #we will gather 'get' results here
else:
result = True
while True:
response_line = reader.read_line()
if cmd == 'get' and response_line.startswith('VALUE'):
response_fields = response_line.split(' ')
key = response_fields[1]
flags = int(response_fields[2])
n = int(response_fields[3])
encoded_value = reader.read_bytes(n)
reader.read_line() #\r\n
result[key] = pickle.loads(encoded_value)
elif cmd == 'get' and response_line == 'END':
block_channel.send(result)
break
elif cmd == 'set' and response_line == 'STORED':
block_channel.send(result)
break
else:
assert False, "unknown protocol state, cmd: %s, response_line: %s" % (cmd, response_line)
except Exception, e:
block_channel.send_exception(TaskletError, e, Tasklet.current())
raise
def test_local(self):
req = Tasklet.current().req = FakeRequest()
req.lang = "en"
values = "apple/apples"
self.assertEqual(self.app.call("l10n.literal_value", 0, values), "apples")
self.assertEqual(self.app.call("l10n.literal_value", 1, values), "apple")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values), "apples")
self.assertEqual(self.app.call("l10n.literal_value", 101, values), "apples")
req.lang = "ru"
values = "яблоко/яблока/яблок"
self.assertEqual(self.app.call("l10n.literal_value", 0, values), "яблок")
self.assertEqual(self.app.call("l10n.literal_value", 1, values), "яблоко")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values), "яблока")
self.assertEqual(self.app.call("l10n.literal_value", 2, values), "яблока")
self.assertEqual(self.app.call("l10n.literal_value", 3, values), "яблока")
self.assertEqual(self.app.call("l10n.literal_value", 5, values), "яблок")
self.assertEqual(self.app.call("l10n.literal_value", 11, values), "яблок")
self.assertEqual(self.app.call("l10n.literal_value", 21, values), "яблоко")
values = "зелёное яблоко/зелёных яблока/зелёных яблок/зелёного яблока"
self.assertEqual(self.app.call("l10n.literal_value", 0, values), "зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 1, values), "зелёное яблоко")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values), "зелёного яблока")
self.assertEqual(self.app.call("l10n.literal_value", 2, values), "зелёных яблока")
self.assertEqual(self.app.call("l10n.literal_value", 3, values), "зелёных яблока")
self.assertEqual(self.app.call("l10n.literal_value", 5, values), "зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 11, values), "зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 21, values), "зелёное яблоко")
def testMessageSend(self):
class MSG_PONG(Message): pass
class MSG_PING(Message): pass
def c(parent):
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_PING):
self.assertEquals((10, ), args)
MSG_PONG.send(parent)(20)
parent = Tasklet.current()
child = Tasklet.new(c)(parent)
i = 0
MSG_PING.send(child)(10)
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_PONG):
self.assertEquals((20, ), args)
i += 1
if i > 5: break
MSG_PING.send(child)(10)
self.assertEquals(6, i)
try:
start = time.time()
for msg, args, kwargs in Tasklet.receive(2.0):
self.fail('expected timeout error')
except TimeoutError:
end = time.time()
self.assertAlmostEqual(2.0, end - start, places = 1)
child.kill()
def __init__(self,
mc,
keys,
patience=20,
delay=0.1,
ttl=30,
value_prefix="",
reason=None):
"""
mc - Memcached instance
keys - list of keys to lock
"""
# Filter out keys that are already locked by the current tasklet
tasklet_locks = self.tasklet_locks()
self.keys = []
for key in sorted(keys):
mkey = "LOCK-" + str(key)
if mkey not in tasklet_locks:
self.keys.append(mkey)
self.mc = mc
self.patience = patience
self.delay = delay
self.locked = None
self.ttl = ttl
self.reason = reason
self.value = str(value_prefix) + str(id(Tasklet.current()))
if self.reason != None:
self.value += "-" + str(self.reason)
lock_serial[0] += 1
self.value += "-" + str(lock_serial[0])
def __execute(self, parentTasklet, jobguid, params, tags, path):
#SETUP THE CONTEXT
Tasklet.current().jobguid = jobguid
wait = q.workflowengine.jobmanager.shouldWait(jobguid)
Tasklet.current().tags = tags
if wait is False: MSG_ACTION_NOWAIT.send(parentTasklet)()
#EXECUTE THE TASKLETS
try:
self.__taskletEngine.execute(params, tags=tags, path=path)
except Exception, e:
q.workflowengine.jobmanager.setJobDied(jobguid, e)
if wait is True:
MSG_ACTION_EXCEPTION.send(parentTasklet)(WFLException.create(
e, jobguid))
def load_all(self):
"Load all available modules"
with self.modules_lock:
self.modules_locked_by = traceback.format_stack()
t = Tasklet.current()
t.modules_locked = True
# removing automatically loaded modules
modules = []
complete = set()
for mod in self.loaded_modules.keys():
if mod in self.not_auto_loaded:
modules.append(mod)
self.loaded_modules.clear()
self.app().hooks.clear()
self._load(modules)
repeat = True
while repeat:
repeat = False
for name, mod in self.loaded_modules.items():
if name not in complete:
children = mod.child_modules()
self._load(children, auto_loaded=True, silent=True)
complete.add(name)
repeat = True
t.modules_locked = False
self.modules_locked_by = None
def callback(socket, count, event, args, kwargs):
print count, event, Tasklet.current()
if (count, event) == (1, "write"):
pass
elif (count, event) == (2, "read"):
Tasklet.sleep(1.0)
return "OK\r\n"
def store(self):
"""
This method iterates over installed handlers and stores group => struct(name => modules_list)
into the database
"""
if not self.dynamic:
return
rec = dict()
for name, handlers in self.handlers.items():
m = re_hook_path.match(name)
if not m:
raise HookFormatError("Invalid hook name: %s" % name)
(hook_group, hook_name) = m.group(1, 2)
if hook_group != "core":
grpset = rec.get(hook_group)
if grpset is None:
grpset = rec[hook_group] = set()
for handler in handlers:
grpset.add(handler[2])
with self.app().hook_lock:
with self.app().lock(["HOOK-GROUPS"]):
t = Tasklet.current()
t.hooks_locked = True
old_groups = self.app().objlist(DBHookGroupModulesList, query_index="all")
for obj in old_groups:
if not obj.uuid in rec:
obj.remove()
groups = self.app().objlist(DBHookGroupModulesList, [])
for group, grpset in rec.iteritems():
if group != "all":
obj = self.app().obj(DBHookGroupModules, group, data={})
obj.set("list", list(grpset))
groups.append(obj)
groups.store(dont_load=True)
t.hooks_locked = False
def load_all(self):
"Load all available modules"
with self.modules_lock:
self.modules_locked_by = traceback.format_stack()
t = Tasklet.current()
t.modules_locked = True
# removing automatically loaded modules
modules = []
complete = set()
for mod in self.loaded_modules.keys():
if mod in self.not_auto_loaded:
modules.append(mod)
self.loaded_modules.clear()
self.app().hooks.clear()
self._load(modules)
repeat = True
while repeat:
repeat = False
for name, mod in self.loaded_modules.items():
if name not in complete:
children = mod.child_modules()
self._load(children, auto_loaded=True, silent=True)
complete.add(name)
repeat = True
t.modules_locked = False
self.modules_locked_by = None
def request(self, environ, start_response):
"Process single HTTP request"
request = Request(environ, start_response)
Tasklet.current().req = request
try:
# remove doubling, leading and trailing slashes, unquote and convert to utf-8
try:
uri = re.sub(r'^/*(.*?)/*$', r'\1', re.sub(r'/{2+}', '/', mg.core.tools.urldecode(request.uri())))
except UnicodeDecodeError:
return request.send_response("404 Not Found", request.headers, "<html><body><h1>404 Not Found</h1></body></html>")
else:
return self.request_uri(request, uri)
except RuntimeError as e:
self.error(e)
e = u"%s" % e
try:
if getattr(request, "upload_handler", None):
return request.uresponse(htmlescape(json.dumps({"success": False, "errormsg": e})))
except Exception as e2:
self.exception(e2)
if type(e) == unicode:
e = e.encode("utf-8")
return request.send_response("500 Internal Server Error", request.headers, "<html><body><h1>500 Internal Server Error</h1>%s</body></html>" % htmlescape(e))
except Exception as e:
try:
self.exception(e)
except Exception as e2:
print "Unhandled exception during logging: %s" % e2
print traceback.format_exc()
try:
if getattr(request, "upload_handler", None):
return request.uresponse(htmlescape(json.dumps({"success": False, "errormsg": "Internal Server Error"})))
except Exception as e2:
self.exception(e2)
return request.internal_server_error()
def _read_response(self, reader):
cmd, block_channel = self._response_queue.popleft(True)
try:
if cmd == 'get':
result = {} #we will gather 'get' results here
else:
result = True
while True:
response_line = reader.read_line()
if cmd == 'get' and response_line.startswith('VALUE'):
response_fields = response_line.split(' ')
key = response_fields[1]
flags = int(response_fields[2])
n = int(response_fields[3])
encoded_value = reader.read_bytes(n)
reader.read_line() #\r\n
result[key] = pickle.loads(encoded_value)
elif cmd == 'get' and response_line == 'END':
block_channel.send(result)
break
elif cmd == 'set' and response_line == 'STORED':
block_channel.send(result)
break
else:
assert False, "unknown protocol state, cmd: %s, response_line: %s" % (
cmd, response_line)
except Exception, e:
block_channel.send_exception(TaskletError, e, Tasklet.current())
raise
def sendToDrp(self, name, *args, **kwargs):
MSG_DRP_CALL.send(self.__tasklet)(Tasklet.current(), name, *args, **kwargs)
(msg, args, kwargs) = Tasklet.receive().next()
if msg.match(MSG_DRP_RETURN):
return args[0]
elif msg.match(MSG_DRP_EXCEPTION):
raise args[0]
def testMessageSend(self):
class MSG_PONG(Message):
pass
class MSG_PING(Message):
pass
def c(parent):
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_PING):
self.assertEquals((10, ), args)
MSG_PONG.send(parent)(20)
parent = Tasklet.current()
child = Tasklet.new(c)(parent)
i = 0
MSG_PING.send(child)(10)
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_PONG):
self.assertEquals((20, ), args)
i += 1
if i > 5: break
MSG_PING.send(child)(10)
self.assertEquals(6, i)
try:
start = time.time()
for msg, args, kwargs in Tasklet.receive(2.0):
self.fail('expected timeout error')
except TimeoutError:
end = time.time()
self.assertAlmostEqual(2.0, end - start, places=1)
child.kill()
def register(self, name, task=None):
if task is None:
task = Tasklet.current()
task_id = id(task)
self._task_by_name[name] = task
self._task_id_by_task[task] = task_id
self._task_by_task_id[task_id] = task
def callback(socket, count, event, args, kwargs):
print count, event, Tasklet.current()
if (count, event) == (1, "write"):
pass
elif (count, event) == (2, "read"):
Tasklet.sleep(1.0)
return "OK\r\n"
def handle(self, socket, application):
stream = BufferedStream(socket)
#implements http1.1 keep alive handler
#there are several concurrent tasks for each connection;
#1 for reading requests, 1 or more for handling requests and 1 for writing responses
#the current task (the one created to handle the socket connection) is the controller task,
#e.g. it coordinates the actions of it's children by message passing
control = Tasklet.current()
#writes responses back to the client when they are ready:
response_writer = Tasklet.new(self.write_responses,
name='response_writer')(control, stream)
#reads requests from clients:
request_reader = Tasklet.new(self.read_requests,
name='request_reader')(control, stream)
#typical flow:
#1. reader reads in request, sends notification to control (MSG_REQUEST_READ)
#2. control starts handler for the request
#3. handler works on request and sends notification to control when finished (MSG_REQUEST_HANDLED)
#4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE)
#5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN)
#control wait for msgs to arrive:
for msg, (request, response), kwargs in Tasklet.receive():
if msg.match(self.MSG_REQUEST_READ):
#we use reque to be able to send the responses back in the correct order later
self._reque.start(request)
Tasklet.new(self.handle_request,
name='request_handler')(control, request,
application)
elif msg.match(self.MSG_REQUEST_HANDLED):
#we use reque to retire (send out) the responses in the correct order
for request, response in self._reque.finish(request, response):
self.MSG_WRITE_RESPONSE.send(response_writer)(request,
response)
elif msg.match(self.MSG_RESPONSE_WRITTEN):
if request.version == 'HTTP/1.0':
break #no keep-alive support in http 1.0
elif request.get_response_header('Connection') == 'close':
break #response indicated to close after response
elif request.get_request_header('Connection') == 'close':
break #request indicated to close after response
elif msg.match(self.MSG_READ_ERROR):
break #stop and close the connection
elif msg.match(self.MSG_WRITE_ERROR):
break #stop and close the connection
else:
assert False, "unexpected msg in control loop"
#kill reader and writer
#any outstanding request will continue, but will exit by themselves
response_writer.kill()
request_reader.kill()
#close our side of the socket
stream.close()
def tasklet_locks(self):
tasklet = Tasklet.current()
try:
return tasklet.memcached_locks
except AttributeError:
locks = set()
tasklet.memcached_locks = locks
return locks
def close(self, exception = None, kill_reader = True, kill_writer = True):
#assert False, reason
if kill_reader:
self._response_reader_task.kill()
if kill_writer:
self._command_writer_task.kill()
self._response_reader_task = None
self._command_writer_task = None
#raise exception on all waiting tasks still in the queues
for cmd, args, block_channel in self._command_queue:
block_channel.send_exception(TaskletError, e, Tasklet.current())
for cmd, block_channel in self._response_queue:
block_channel.send_exception(TaskletError, e, Tasklet.current())
self._command_queue = None
self._response_queue = None
self._stream.close()
self._stream = None
def sendToDrp(self, name, *args, **kwargs):
MSG_DRP_CALL.send(self.__tasklet)(Tasklet.current(), name, *args,
**kwargs)
(msg, args, kwargs) = Tasklet.receive().next()
if msg.match(MSG_DRP_RETURN):
return args[0]
elif msg.match(MSG_DRP_EXCEPTION):
raise args[0]
def __delattr__(self, key):
#note: del attr is not recursive!
d = self._d
current = Tasklet.current()
if (not current in d) or (not hasattr(d[current], key)):
raise AttributeError(key)
else:
delattr(d[current], key)
def __delattr__(self, key):
#note: del attr is not recursive!
d = self._d
current = Tasklet.current()
if (not current in d) or (not hasattr(d[current], key)):
raise AttributeError(key)
else:
delattr(d[current], key)
def close(self, exception=None, kill_reader=True, kill_writer=True):
#assert False, reason
if kill_reader:
self._response_reader_task.kill()
if kill_writer:
self._command_writer_task.kill()
self._response_reader_task = None
self._command_writer_task = None
#raise exception on all waiting tasks still in the queues
for cmd, args, block_channel in self._command_queue:
block_channel.send_exception(TaskletError, e, Tasklet.current())
for cmd, block_channel in self._response_queue:
block_channel.send_exception(TaskletError, e, Tasklet.current())
self._command_queue = None
self._response_queue = None
self._stream.close()
self._stream = None
def __enter__(self):
task = id(Tasklet.current())
if self.locked_by and self.locked_by == task:
self.depth += 1
return self
Lock.__enter__(self)
self.locked_by = task
self.depth = 0
return self
def __setattr__(self, key, value):
if key in ['_d', '_recursive']: #to protect from infinite recursion during __init__
self.__dict__[key] = value
else:
d = self._d
current = Tasklet.current()
if not current in d: #task specific attributes instance not created yet
d[current] = self._LocalAttributes()
setattr(d[current], key, value)
def __enter__(self):
task = id(Tasklet.current())
if self.locked_by and self.locked_by == task:
self.depth += 1
return self
Lock.__enter__(self)
self.locked_by = task
self.depth = 0
return self
def __setattr__(self, key, value):
if key in ['_d', '_recursive'
]: #to protect from infinite recursion during __init__
self.__dict__[key] = value
else:
d = self._d
current = Tasklet.current()
if not current in d: #task specific attributes instance not created yet
d[current] = self._LocalAttributes()
setattr(d[current], key, value)
def handle(self, socket, application):
stream = BufferedStream(socket)
#implements http1.1 keep alive handler
#there are several concurrent tasks for each connection;
#1 for reading requests, 1 or more for handling requests and 1 for writing responses
#the current task (the one created to handle the socket connection) is the controller task,
#e.g. it coordinates the actions of it's children by message passing
control = Tasklet.current()
#writes responses back to the client when they are ready:
response_writer = Tasklet.new(self.write_responses, name = 'response_writer')(control, stream)
#reads requests from clients:
request_reader = Tasklet.new(self.read_requests, name = 'request_reader')(control, stream)
#typical flow:
#1. reader reads in request, sends notification to control (MSG_REQUEST_READ)
#2. control starts handler for the request
#3. handler works on request and sends notification to control when finished (MSG_REQUEST_HANDLED)
#4. control sends message to writer to start writing the response (MSG_WRITE_RESPONSE)
#5. writer notififies control when response is wriiten (MSG_RESPONSE_WRITTEN)
#control wait for msgs to arrive:
for msg, (request, response), kwargs in Tasklet.receive():
if msg.match(self.MSG_REQUEST_READ):
#we use reque to be able to send the responses back in the correct order later
self._reque.start(request)
Tasklet.new(self.handle_request, name = 'request_handler')(control, request, application)
elif msg.match(self.MSG_REQUEST_HANDLED):
#request.environ["wsgi.input"].read(request.environ["wsgi.input"]._n)
#we use reque to retire (send out) the responses in the correct order
for request, response in self._reque.finish(request, response):
self.MSG_WRITE_RESPONSE.send(response_writer)(request, response)
elif msg.match(self.MSG_RESPONSE_WRITTEN):
if request.version == 'HTTP/1.0':
break #no keep-alive support in http 1.0
elif request.get_response_header('Connection') == 'close':
break #response indicated to close after response
elif request.get_request_header('Connection') == 'close':
break #request indicated to close after response
elif msg.match(self.MSG_READ_ERROR):
break #stop and close the connection
elif msg.match(self.MSG_WRITE_ERROR):
break #stop and close the connection
else:
assert False, "unexpected msg in control loop"
#kill reader and writer
#any outstanding request will continue, but will exit by themselves
response_writer.kill()
request_reader.kill()
#close our side of the socket
stream.close()
def handler(client_socket):
"""writes the familiar greeting to client"""
cur_thread = threading.current_thread()
stream = BufferedStream(client_socket)
writer = stream.writer
writer.write_bytes("HTTP/1.0 200 OK\r\n")
writer.write_bytes("Content-Length: 12\r\n")
writer.write_bytes("\r\n")
writer.write_bytes("[%s:%s] - " % (cur_thread, Tasklet.current()))
writer.flush()
stream.close()
def testJoinChildren(self):
def t():
return 1
for i in range(4):
Tasklet.new(t)()
self.assertEquals(4, len(Tasklet.current().children()))
result = Tasklet.join_children()
self.assertEquals([1, 1, 1, 1], result)
def request(self, environ, start_response):
"Process single HTTP request"
request = Request(environ, start_response)
Tasklet.current().req = request
try:
# remove doubling, leading and trailing slashes, unquote and convert to utf-8
try:
uri = re.sub(
r'^/*(.*?)/*$', r'\1',
re.sub(r'/{2+}', '/',
mg.core.tools.urldecode(request.uri())))
except UnicodeDecodeError:
return request.send_response(
"404 Not Found", request.headers,
"<html><body><h1>404 Not Found</h1></body></html>")
else:
return self.request_uri(request, uri)
except RuntimeError as e:
self.error(e)
e = u"%s" % e
try:
if getattr(request, "upload_handler", None):
return request.uresponse(
htmlescape(
json.dumps({
"success": False,
"errormsg": e
})))
except Exception as e2:
self.exception(e2)
if type(e) == unicode:
e = e.encode("utf-8")
return request.send_response(
"500 Internal Server Error", request.headers,
"<html><body><h1>500 Internal Server Error</h1>%s</body></html>"
% htmlescape(e))
except Exception as e:
try:
self.exception(e)
except Exception as e2:
print "Unhandled exception during logging: %s" % e2
print traceback.format_exc()
try:
if getattr(request, "upload_handler", None):
return request.uresponse(
htmlescape(
json.dumps({
"success": False,
"errormsg": "Internal Server Error"
})))
except Exception as e2:
self.exception(e2)
return request.internal_server_error()
def tasks(self):
cur = Tasklet.current()
while cur.parent():
cur = cur.parent()
tasks = []
for task, level in cur.tree():
rtask = {"id": id(task), "name": task.name}
try:
rtask["stack"] = traceback.format_stack(task.frame)
except AttributeError:
pass
tasks.append({"level": level, "task": rtask})
self.call("web.response_json", {"retval": tasks})
def load_groups(self, groups):
"""
Load all modules handling any hooks in the given groups
groups - list of hook group names
"""
t = Tasklet.current()
if getattr(t, "hooks_locked", False):
self._load_groups(groups)
else:
with self.app().hook_lock:
t.hooks_locked = True
self._load_groups(groups)
t.hooks_locked = False
def load_groups(self, groups):
"""
Load all modules handling any hooks in the given groups
groups - list of hook group names
"""
t = Tasklet.current()
if getattr(t, "hooks_locked", False):
self._load_groups(groups)
else:
with self.app().hook_lock:
t.hooks_locked = True
self._load_groups(groups)
t.hooks_locked = False
def testJoinChildren(self):
def t():
return 1
for i in range(4):
Tasklet.new(t)()
self.assertEquals(4, len(Tasklet.current().children()))
result = Tasklet.join_children()
self.assertEquals([1,1,1,1], result)
def executeActorActionScript(self,
agentguid,
scriptname,
params,
executionparams={},
jobguid=None):
"""
Execute an actor action script on an agent. The action will be executed in a new job that is the newest child of the current job.
The new job will inherit the following properties of the job: name, description, userErrormsg, internalErrormsg, maxduration and customerguid.
The executionparams passed to the function will override the properties of the job (if provided).
The scripts are found in $actoractionpath/$actorname/$actionname/scripts/${scriptname}.rscript
@param agentguid: Guid of the agent on which the script will be executed.
@type agentguid: guid
@param scriptname: The name of the script. Extension .rscript will be added automatically.
@type scriptpath: string
@param params: Dictionary containing all parameters required to execute this script.
@type params: dictionary
@param executionParams: Dictionary can following keys: name, description, userErrormsg, internalErrormsg, maxduration, wait, timetostart, priority
@type executionParams: dictionary
@param jobguid: Optional parameter, can be used to override the current job.
@type jobguid: guid
@return: Dictionary with action result as result and the action's jobguid: {'result': <result>, 'jobguid': guid}
@rtype: dictionary
@raise ActionNotFoundException If the script was not found.
@raise IOError: If the scriptpath can't be read.
@raise TimeOutException: If the agent did not respond within the maxduration: the script will be killed and the exception will be raised.
@raise AgentNotAvailableException: If the agent is not available when starting the script, the script is not be started.
@raise ScriptFailedException: If an exception occurres on the agent while executing the script.
"""
(actorname, actionname) = Tasklet.current().tags
scriptpath = q.system.fs.joinPaths(ActorActionTaskletPath, actorname,
actionname, ActorActionScriptFolder,
scriptname + ".rscript")
if not q.system.fs.exists(scriptpath):
raise ActionNotFoundException("ActorActionScript",
actorname + "." + actionname,
scriptname)
else:
return self.executeScript(
agentguid, actorname + "." + actionname + "." + scriptname,
scriptpath, params, executionparams, jobguid)
def __getattr__(self, key):
#TODO PROFILING this method seems to be quite expensive in profiling
#can we cache the route up into the parent or something?
d = self._d
current = Tasklet.current()
while current is not None:
if (not current in d) or (not hasattr(d[current], key)):
if not self._recursive:
break
else:
current = current.parent() #continue checking parent task locals
else:
return getattr(d[current], key)
raise AttributeError(key)
def __getattr__(self, key):
#TODO PROFILING this method seems to be quite expensive in profiling
#can we cache the route up into the parent or something?
d = self._d
current = Tasklet.current()
while current is not None:
if (not current in d) or (not hasattr(d[current], key)):
if not self._recursive:
break
else:
current = current.parent(
) #continue checking parent task locals
else:
return getattr(d[current], key)
raise AttributeError(key)
def test_local(self):
req = Tasklet.current().req = FakeRequest()
req.lang = "en"
values = "apple/apples"
self.assertEqual(self.app.call("l10n.literal_value", 0, values),
"apples")
self.assertEqual(self.app.call("l10n.literal_value", 1, values),
"apple")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values),
"apples")
self.assertEqual(self.app.call("l10n.literal_value", 101, values),
"apples")
req.lang = "ru"
values = "яблоко/яблока/яблок"
self.assertEqual(self.app.call("l10n.literal_value", 0, values),
"яблок")
self.assertEqual(self.app.call("l10n.literal_value", 1, values),
"яблоко")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values),
"яблока")
self.assertEqual(self.app.call("l10n.literal_value", 2, values),
"яблока")
self.assertEqual(self.app.call("l10n.literal_value", 3, values),
"яблока")
self.assertEqual(self.app.call("l10n.literal_value", 5, values),
"яблок")
self.assertEqual(self.app.call("l10n.literal_value", 11, values),
"яблок")
self.assertEqual(self.app.call("l10n.literal_value", 21, values),
"яблоко")
values = "зелёное яблоко/зелёных яблока/зелёных яблок/зелёного яблока"
self.assertEqual(self.app.call("l10n.literal_value", 0, values),
"зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 1, values),
"зелёное яблоко")
self.assertEqual(self.app.call("l10n.literal_value", 1.5, values),
"зелёного яблока")
self.assertEqual(self.app.call("l10n.literal_value", 2, values),
"зелёных яблока")
self.assertEqual(self.app.call("l10n.literal_value", 3, values),
"зелёных яблока")
self.assertEqual(self.app.call("l10n.literal_value", 5, values),
"зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 11, values),
"зелёных яблок")
self.assertEqual(self.app.call("l10n.literal_value", 21, values),
"зелёное яблоко")
def push(self, timeout):
current_timeout = self._timeout_stack[-1]
assert current_timeout != TIMEOUT_CURRENT
if timeout == TIMEOUT_CURRENT:
self._timeout_stack.append(current_timeout)
elif timeout == TIMEOUT_NEVER and current_timeout == TIMEOUT_NEVER:
self._timeout_stack.append(timeout)
elif timeout == TIMEOUT_NEVER and current_timeout != TIMEOUT_NEVER:
self._timeout_stack.append(current_timeout)
else:
_timeout_time = time.time() + timeout
if current_timeout == TIMEOUT_NEVER:
self._timeout_stack.append(_timeout_time)
else:
self._timeout_stack.append(min(_timeout_time, current_timeout))
Tasklet.current()._timeout_time = self._timeout_stack[-1]
def __execute(self, parentTasklet, jobguid, agentguid, scriptpath, params):
#SETUP THE CONTEXT
Tasklet.current().jobguid = jobguid
wait = q.workflowengine.jobmanager.shouldWait(jobguid)
if wait is False: MSG_ACTION_NOWAIT.send(parentTasklet)()
try:
params = self.__agentController.executeScript(agentguid, jobguid, scriptpath, params)
except TimeOutException, te:
try:
self.__agentController.killScript(agentguid, jobguid, 10)
except TimeOutException:
q.logger.log("Failed to kill Script '" + scriptpath + "' on agent '" + agentguid + "' for job '" + jobguid, 1)
except ScriptFailedException:
q.logger.log("Failed to execute Script '" + scriptpath + "' on agent '" + agentguid + "' for job '" + jobguid, 1)
q.workflowengine.jobmanager.setJobDied(jobguid, te)
if wait is True: MSG_ACTION_EXCEPTION.send(parentTasklet)(WFLException.create(te,jobguid))
def testTimeout(self):
self.assertEquals(TIMEOUT_NEVER, Tasklet.get_current_timeout())
Tasklet.set_current_timeout(10)
self.assertAlmostEqual(10.0, Tasklet.get_current_timeout(), places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(9.0, Tasklet.get_current_timeout(), places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(8.0, Tasklet.get_current_timeout(), places = 1)
current = Tasklet.current()
current.timeout = 10.0
self.assertAlmostEqual(10.0, current.timeout, places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(9.0, current.timeout, places = 1)
current.timeout = TIMEOUT_NEVER
self.assertEquals(TIMEOUT_NEVER, current.timeout)
def testTimeout(self):
self.assertEquals(TIMEOUT_NEVER, Tasklet.get_current_timeout())
Tasklet.set_current_timeout(10)
self.assertAlmostEqual(10.0, Tasklet.get_current_timeout(), places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(9.0, Tasklet.get_current_timeout(), places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(8.0, Tasklet.get_current_timeout(), places = 1)
current = Tasklet.current()
current.timeout = 10.0
self.assertAlmostEqual(10.0, current.timeout, places = 1)
Tasklet.sleep(1.0)
self.assertAlmostEqual(9.0, current.timeout, places = 1)
current.timeout = TIMEOUT_NEVER
self.assertEquals(TIMEOUT_NEVER, current.timeout)
def testTree(self):
def child(prefix, level, i):
if level < 2:
for j in range(2):
name = prefix + str(j)
Tasklet.new(child, name = name)(name, level + 1, j)
Tasklet.sleep(2)
Tasklet.new(child, 'child')('child', 0, 0)
Tasklet.sleep(1)
#for task, level in Tasklet.current().tree():
# print '\t' * level, task.name, level
flattened = set([(task.name, level) for (task, level) in Tasklet.current().tree()][1:])
self.assertEquals(set([('child', 1), ('child0', 2), ('child00', 3), ('child01', 3),
('child1', 2), ('child10', 3), ('child11', 3)]), flattened)
def testTree(self):
def child(prefix, level, i):
if level < 2:
for j in range(2):
name = prefix + str(j)
Tasklet.new(child, name = name)(name, level + 1, j)
Tasklet.sleep(2)
Tasklet.new(child, 'child')('child', 0, 0)
Tasklet.sleep(1)
#for task, level in Tasklet.current().tree():
# print '\t' * level, task.name, level
flattened = set([(task.name, level) for (task, level) in Tasklet.current().tree()][1:])
self.assertEquals(set([('child', 1), ('child0', 2), ('child00', 3), ('child01', 3),
('child1', 2), ('child10', 3), ('child11', 3)]), flattened)
def handle(client_socket):
"""handles a single client connected to the chat server"""
stream = BufferedStream(client_socket)
client_task = Tasklet.current() #this is the current task as started by server
connected_clients.add(client_task)
def writer():
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_WRITE_LINE):
stream.writer.write_bytes(args[0] + '\n')
stream.writer.flush()
def reader():
for line in stream.reader.read_lines():
line = line.strip()
if line == 'quit':
MSG_QUIT.send(client_task)()
else:
MSG_LINE_READ.send(client_task)(line)
reader_task = Tasklet.new(reader)()
writer_task = Tasklet.new(writer)()
MSG_WRITE_LINE.send(writer_task)("type 'quit' to exit..")
for msg, args, kwargs in Tasklet.receive():
if msg.match(MSG_QUIT):
break
elif msg.match(MSG_LINE_READ):
#a line was recv from our client, multicast it to the other clients
for task in connected_clients:
if task != client_task: #don't echo the line back to myself
MSG_WRITE_LINE.send(task)(args[0])
elif msg.match(MSG_WRITE_LINE):
MSG_WRITE_LINE.send(writer_task)(args[0])
connected_clients.remove(client_task)
reader_task.kill()
writer_task.kill()
client_socket.close()
def tasks(self):
cur = Tasklet.current()
while cur.parent():
cur = cur.parent()
tasks = []
for task, level in cur.tree():
rtask = {
"id": id(task),
"name": task.name,
}
try:
rtask["stack"] = traceback.format_stack(task.frame)
except AttributeError:
pass
tasks.append({
"level": level,
"task": rtask
})
self.call("web.response_json", {
"retval": tasks
})
def _write_command(self, writer):
try:
cmd, args, block_channel = self._command_queue.popleft(True)
writer.clear()
if cmd == 'get':
writer.write_bytes("get")
for key in args[0]:
writer.write_bytes(" " + key)
elif cmd in ['set']:
key, value, flags = args
encoded_value = pickle.dumps(value, -1)
writer.write_bytes("%s %s %d 0 %d\r\n" % (cmd, key, flags, len(encoded_value)))
writer.write_bytes(encoded_value)
else:
assert False, "unknown command %s" % cmd
writer.write_bytes('\r\n')
writer.flush()
self._response_queue.append((cmd, block_channel))
except Exception, e:
block_channel.send_exception(TaskletError, e, Tasklet.current())
raise
def functions(self):
tasklet = Tasklet.current()
try:
return tasklet._globfunctions
except AttributeError:
pass
# Cache miss
funcs = []
for fn_id in self.conf("globfunc.list", []):
funcs.append({
"id": fn_id,
})
for func in funcs:
conf = self.conf("globfunc.%s" % func["id"])
if conf:
for k, v in conf.iteritems():
func[k] = v
funcs.sort(cmp=lambda x, y: cmp(x.get("order", 0), y.get("order", 0)) or cmp(x["title"], y["title"]))
# Cache store
tasklet._globfunctions = funcs
return funcs
def filter(self, record):
try:
try:
if record.args[0] == "200 OK" and record.args[1] == "/core/ping":
return 0
except KeyError:
pass
except IndexError:
pass
except AttributeError:
pass
req = Tasklet.current().req
record.host = req.environ.get("HTTP_X_REAL_HOST")
record.ip = req.environ.get("HTTP_X_REAL_IP")
app = req.app
record.app = app.tag
if req.__dict__.has_key("_session"):
record.user = req.user()
except AttributeError:
pass
return 1
def load(self, modules, silent=False, auto_loaded=False):
"""
Load requested modules.
modules - list of module names (format: "mg.group.Class" means
silent - don't fail on ImportError
auto_loaded - remove this modules on full reload
"import Class from mg.group")
"""
t = Tasklet.current()
if getattr(t, "modules_locked", False):
return self._load(modules, silent, auto_loaded)
else:
wasLocked = False
if self.modules_lock.is_locked():
wasLocked = True
with self.modules_lock:
self.modules_locked_by = traceback.format_stack()
t.modules_locked = True
res = self._load(modules, silent, auto_loaded)
t.modules_locked = False
self.modules_locked_by = None
return res
def functions(self):
tasklet = Tasklet.current()
try:
return tasklet._globfunctions
except AttributeError:
pass
# Cache miss
funcs = []
for fn_id in self.conf("globfunc.list", []):
funcs.append({
"id": fn_id,
})
for func in funcs:
conf = self.conf("globfunc.%s" % func["id"])
if conf:
for k, v in conf.iteritems():
func[k] = v
funcs.sort(cmp=lambda x, y: cmp(x.get("order", 0), y.get("order", 0))
or cmp(x["title"], y["title"]))
# Cache store
tasklet._globfunctions = funcs
return funcs
def filter(self, record):
try:
try:
if record.args[0] == "200 OK" and record.args[
1] == "/core/ping":
return 0
except KeyError:
pass
except IndexError:
pass
except AttributeError:
pass
req = Tasklet.current().req
record.host = req.environ.get("HTTP_X_REAL_HOST")
record.ip = req.environ.get("HTTP_X_REAL_IP")
app = req.app
record.app = app.tag
if req.__dict__.has_key("_session"):
record.user = req.user()
except AttributeError:
pass
return 1
