def stop_worker(self, json_request):
""" Stops the god watch for queues of an application on the current
node.
Args:
json_request: A JSON string with the queue name for which we're
stopping its queues.
Returns:
A JSON string with the result.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.STOP_WORKERS_TAGS)
if 'error' in request:
return json.dumps(request)
app_id = request['app_id']
watch = "celery-" + str(app_id)
try:
if god_interface.stop(watch):
stop_command = self.get_worker_stop_command(app_id)
os.system(stop_command)
TaskQueueConfig.remove_config_files(app_id)
result = {'error': False}
else:
result = {'error': True, 'reason': "Unable to stop watch %s" % watch}
except OSError, os_error:
result = {'error': True, 'reason' : str(os_error)}
def stop_worker(self, json_request):
""" Stops the monit watch for queues of an application on the current
node.
Args:
json_request: A JSON string with the queue name for which we're
stopping its queues.
Returns:
A JSON string with the result.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.STOP_WORKERS_TAGS)
logging.info("Stopping worker: {0}".format(request))
if 'error' in request:
return json.dumps(request)
app_id = request['app_id']
watch = "celery-" + str(app_id)
try:
if monit_interface.stop(watch):
stop_command = self.get_worker_stop_command(app_id)
os.system(stop_command)
TaskQueueConfig.remove_config_files(app_id)
result = {'error': False}
else:
result = {
'error': True,
'reason': "Unable to stop watch %s" % watch
}
except OSError, os_error:
result = {'error': True, 'reason': str(os_error)}
def reload_worker(self, json_request):
""" Reloads taskqueue workers as needed.
A worker can be started on both a master and slave node.
Args:
json_request: A JSON string with the application id.
Returns:
A JSON string with the error status and error reason.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.SETUP_WORKERS_TAGS)
logging.info("Reload worker request: {0}".format(request))
if 'error' in request:
return json.dumps(request)
app_id = self.__cleanse(request['app_id'])
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
old_queues = self.__queue_info_cache.get(app_id, {'queue': []})
old_queue_dict = {}
for queue in old_queues['queue']:
old_queue_dict[queue['name']] = queue
new_queue_dict = {}
# Load the new queue info.
try:
new_queues = config.load_queues_from_file(app_id)
for queue in new_queues['queue']:
new_queue_dict[queue['name']] = queue
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
def test_load_queues_from_xml_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_xml)
flexmock(file_io) \
.should_receive("exists").and_return(False).and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(
queue_info, {
'queue': [{
'max_concurrent_requests': '300',
'rate': '100/s',
'bucket_size': '100',
'name': 'default',
'retry_parameters': {
'task_age_limit': '3d'
}
}, {
'max_concurrent_requests': '100',
'rate': '100/s',
'bucket_size': '100',
'name': 'mapreduce-workers',
'retry_parameters': {
'task_age_limit': '3d'
}
}]
})
def __enqueue_push_task(self, request):
""" Enqueues a batch of push tasks.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
"""
self.__validate_push_task(request)
self.__check_and_store_task_names(request)
args = self.get_task_args(request)
headers = self.get_task_headers(request)
countdown = int(headers['X-AppEngine-TaskETA']) - \
int(datetime.datetime.now().strftime("%s"))
task_func = self.__get_task_function(request)
result = task_func.apply_async(
kwargs={
'headers': headers,
'args': args
},
expires=args['expires'],
acks_late=True,
countdown=countdown,
queue=TaskQueueConfig.get_celery_queue_name(
request.app_id(), request.queue_name()),
routing_key=TaskQueueConfig.get_celery_queue_name(
request.app_id(), request.queue_name()))
def reload_worker(self, json_request):
""" Reloads taskqueue workers as needed.
A worker can be started on both a master and slave node.
Args:
json_request: A JSON string with the application id.
Returns:
A JSON string with the error status and error reason.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.SETUP_WORKERS_TAGS)
logging.info("Reload worker request: {0}".format(request))
if 'error' in request:
return json.dumps(request)
app_id = self.__cleanse(request['app_id'])
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
old_queues = self.__queue_info_cache.get(app_id, {'queue': []})
old_queue_dict = {}
for queue in old_queues['queue']:
old_queue_dict[queue['name']] = queue
new_queue_dict = {}
# Load the new queue info.
try:
new_queues = config.load_queues_from_file(app_id)
for queue in new_queues['queue']:
new_queue_dict[queue['name']] = queue
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
def test_load_queues_from_xml_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_xml)
flexmock(file_io) \
.should_receive("exists").and_return(False).and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(queue_info, {'queue': [{'max_concurrent_requests': '300', 'rate': '100/s', 'bucket_size': '100', 'name': 'default', 'retry_parameters': {'task_age_limit': '3d'}}, {'max_concurrent_requests': '100', 'rate': '100/s', 'bucket_size': '100', 'name': 'mapreduce-workers', 'retry_parameters': {'task_age_limit': '3d'}}]})
def start_worker(self, json_request):
""" Starts taskqueue workers if they are not already running.
A worker can be started on both a master and slave node.
Args:
json_request: A JSON string with the application id.
Returns:
A JSON string with the error status and error reason.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.SETUP_WORKERS_TAGS)
if 'error' in request:
return json.dumps(request)
app_id = self.__cleanse(request['app_id'])
hostname = socket.gethostbyname(socket.gethostname())
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
# Load the queue info
try:
config.load_queues_from_file(app_id)
config.create_celery_file(TaskQueueConfig.QUEUE_INFO_FILE)
config.create_celery_worker_scripts(
TaskQueueConfig.QUEUE_INFO_FILE)
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
def test_load_queues(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
def test_load_queues_from_db(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_db()
self.assertEquals(queue_info, {'queue':[{'name': 'foo',
'rate': '10/m'},
]})
def test_load_queues(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
def __get_task_function(self, request):
""" Returns a function pointer to a celery task.
Load the module for the app/queue.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
Returns:
A function pointer to a celery task.
Raises:
taskqueue_service_pb.TaskQueueServiceError
"""
try:
task_module = __import__(TaskQueueConfig.\
get_celery_worker_module_name(request.app_id()))
# If a new queue was added we need to relaod the python code.
if self.__force_reload:
start = time.time()
reload(task_module)
time_taken = time.time() - start
self.__force_reload = False
logging.info("Reloading module for {0} took {1} seconds.".\
format(request.app_id(), time_taken))
task_func = getattr(task_module,
TaskQueueConfig.get_queue_function_name(request.queue_name()))
return task_func
except AttributeError, attribute_error:
logging.exception(attribute_error)
raise apiproxy_errors.ApplicationError(
taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE)
def __get_task_function(self, request):
""" Returns a function pointer to a celery task.
Load the module for the app/queue.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
Returns:
A function pointer to a celery task.
Raises:
taskqueue_service_pb.TaskQueueServiceError
"""
try:
task_module = __import__(TaskQueueConfig.\
get_celery_worker_module_name(request.app_id()))
# If a new queue was added we need to relaod the python code.
if self.__force_reload:
start = time.time()
reload(task_module)
time_taken = time.time() - start
self.__force_reload = False
logging.info("Reloading module for {0} took {1} seconds.".\
format(request.app_id(), time_taken))
task_func = getattr(
task_module,
TaskQueueConfig.get_queue_function_name(request.queue_name()))
return task_func
except AttributeError, attribute_error:
logging.exception(attribute_error)
raise apiproxy_errors.ApplicationError(
taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE)
def test_load_queues_from_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(
queue_info, {
'queue': [{
'name': 'default',
'rate': '5/s'
}, {
'name': 'foo',
'rate': '10/m'
}]
})
flexmock(file_io) \
.should_receive("read").and_return('blah').and_raise(IOError)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(queue_info,
{'queue': [{
'name': 'default',
'rate': '5/s'
}]})
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(
queue_info, {
'queue': [
{
'name': 'foo',
'rate': '10/m'
},
{
'name': 'default',
'rate': '5/s'
},
]
})
def test_constructor(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
def start_worker(self, json_request):
""" Starts taskqueue workers if they are not already running.
A worker can be started on both a master and slave node.
Args:
json_request: A JSON string with the application id.
Returns:
A JSON string with the error status and error reason.
"""
request = self.__parse_json_and_validate_tags(json_request,
self.SETUP_WORKERS_TAGS)
if 'error' in request:
return json.dumps(request)
app_id = self.__cleanse(request['app_id'])
hostname = socket.gethostbyname(socket.gethostname())
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
# Load the queue info
try:
config.load_queues_from_file(app_id)
config.create_celery_file(TaskQueueConfig.QUEUE_INFO_FILE)
config.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_FILE)
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
def test_load_queues_from_db(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
queue_info = tqc.load_queues_from_db()
self.assertEquals(queue_info,
{'queue': [
{
'name': 'foo',
'rate': '10/m'
},
]})
def test_save_queues_to_db(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(file_io) \
.should_receive('exists').and_return(True)
flexmock(datastore).should_receive("Put").\
and_return()
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
try:
queue_info = tqc.save_queues_to_db()
raise
except ValueError:
pass
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.save_queues_to_db()
def test_save_queues_to_db(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(file_io) \
.should_receive('exists').and_return(True)
flexmock(datastore).should_receive("Put").\
and_return()
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
try:
queue_info = tqc.save_queues_to_db()
raise
except ValueError:
pass
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.save_queues_to_db()
def get_task_args(self, request):
""" Gets the task args used when making a task web request.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest
Returns:
A dictionary used by a task worker.
"""
args = {}
args['task_name'] = request.task_name()
args['url'] = request.url()
args['app_id'] = request.app_id()
args['queue_name'] = request.queue_name()
args['method'] = self.__method_mapping(request.method())
args['body'] = request.body()
args['payload'] = request.payload()
args['description'] = request.description()
# Set defaults.
args['max_retries'] = self.DEFAULT_MAX_RETRIES
args['expires'] = self.__when_to_expire(request)
args['max_retries'] = self.DEFAULT_MAX_RETRIES
args['max_backoff_sec'] = self.DEFAULT_MAX_BACKOFF
args['min_backoff_sec'] = self.DEFAULT_MIN_BACKOFF
args['max_doublings'] = self.DEFAULT_MAX_DOUBLINGS
# Load queue info into cache.
if request.app_id() not in self.__queue_info_cache:
try:
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
request.app_id())
self.__queue_info_cache[
request.app_id()] = config.load_queues_from_file(
request.app_id())
except ValueError, value_error:
logging.error("Unable to load queues for app id {0} using defaults."\
.format(request.app_id()))
except NameError, name_error:
logging.error("Unable to load queues for app id {0} using defaults."\
.format(request.app_id()))
def test_create_celery_worker_scripts(self):
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io).should_receive("write").and_return(None)
flexmock(file_io).should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
flexmock(file_io) \
.should_receive("exists").and_return(True)
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
FILE1 = open(
os.path.dirname(os.path.realpath(__file__)) +
'/../../templates/header.py', 'r')
file1 = FILE1.read()
FILE1.close()
FILE2 = open(
os.path.dirname(os.path.realpath(__file__)) +
'/../../templates/task.py', 'r')
file2 = FILE2.read()
FILE2.close()
flexmock(file_io).should_receive('write').and_return(None)
flexmock(file_io).should_receive("read").and_return(file1).and_return(
file2)
self.assertEquals(
tqc.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_DB),
TaskQueueConfig.CELERY_WORKER_DIR + 'app___myapp.py')
self.assertEquals(
tqc.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_FILE),
TaskQueueConfig.CELERY_WORKER_DIR + 'app___myapp.py')
def test_create_celery_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
# making sure it does not throw an exception
self.assertEquals(tqc.create_celery_file(TaskQueueConfig.QUEUE_INFO_DB),
TaskQueueConfig.CELERY_CONFIG_DIR + "myapp" + ".py")
self.assertEquals(tqc.create_celery_file(TaskQueueConfig.QUEUE_INFO_FILE),
TaskQueueConfig.CELERY_CONFIG_DIR + "myapp" + ".py")
def __enqueue_push_task(self, request):
""" Enqueues a batch of push tasks.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
"""
self.__validate_push_task(request)
self.__check_and_store_task_names(request)
args = self.get_task_args(request)
headers = self.get_task_headers(request)
countdown = int(headers['X-AppEngine-TaskETA']) - \
int(datetime.datetime.now().strftime("%s"))
task_func = self.__get_task_function(request)
result = task_func.apply_async(kwargs={'headers':headers,
'args':args},
expires=args['expires'],
acks_late=True,
countdown=countdown,
queue=TaskQueueConfig.get_celery_queue_name(
request.app_id(), request.queue_name()),
routing_key=TaskQueueConfig.get_celery_queue_name(
request.app_id(), request.queue_name()))
def get_task_args(self, request):
""" Gets the task args used when making a task web request.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest
Returns:
A dictionary used by a task worker.
"""
args = {}
args['task_name'] = request.task_name()
args['url'] = request.url()
args['app_id'] = request.app_id()
args['queue_name'] = request.queue_name()
args['method'] = self.__method_mapping(request.method())
args['body'] = request.body()
args['payload'] = request.payload()
args['description'] = request.description()
# Set defaults.
args['max_retries'] = self.DEFAULT_MAX_RETRIES
args['expires'] = self.__when_to_expire(request)
args['max_retries'] = self.DEFAULT_MAX_RETRIES
args['max_backoff_sec'] = self.DEFAULT_MAX_BACKOFF
args['min_backoff_sec'] = self.DEFAULT_MIN_BACKOFF
args['max_doublings'] = self.DEFAULT_MAX_DOUBLINGS
# Load queue info into cache.
if request.app_id() not in self.__queue_info_cache:
try:
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, request.app_id())
self.__queue_info_cache[request.app_id()] = config.load_queues_from_file(
request.app_id())
except ValueError, value_error:
logging.error("Unable to load queues for app id {0} using defaults."\
.format(request.app_id()))
except NameError, name_error:
logging.error("Unable to load queues for app id {0} using defaults."\
.format(request.app_id()))
def test_create_celery_worker_scripts(self):
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io).should_receive("write").and_return(None)
flexmock(file_io).should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
flexmock(file_io) \
.should_receive("exists").and_return(True)
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
FILE1 = open(os.path.dirname(os.path.realpath(__file__)) + '/../../templates/header.py', 'r')
file1 = FILE1.read()
FILE1.close()
FILE2 = open(os.path.dirname(os.path.realpath(__file__)) + '/../../templates/task.py', 'r')
file2 = FILE2.read()
FILE2.close()
flexmock(file_io).should_receive('write').and_return(None)
flexmock(file_io).should_receive("read").and_return(file1).and_return(file2)
self.assertEquals(tqc.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_DB), TaskQueueConfig.CELERY_WORKER_DIR + 'app___myapp.py')
self.assertEquals(tqc.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_FILE), TaskQueueConfig.CELERY_WORKER_DIR + 'app___myapp.py')
def __get_task_function(self, request):
""" Returns a function pointer to a celery task.
Load the module for the app/queue.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
Returns:
A function pointer to a celery task.
Raises:
taskqueue_service_pb.TaskQueueServiceError
"""
try:
task_module = __import__(TaskQueueConfig.\
get_celery_worker_module_name(request.app_id()))
task_func = getattr(task_module,
TaskQueueConfig.get_queue_function_name(request.queue_name()))
return task_func
except ImportError, import_error:
raise apiproxy_errors.ApplicationError(
taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE)
def __get_task_function(self, request):
""" Returns a function pointer to a celery task.
Load the module for the app/queue.
Args:
request: A taskqueue_service_pb.TaskQueueAddRequest.
Returns:
A function pointer to a celery task.
Raises:
taskqueue_service_pb.TaskQueueServiceError
"""
try:
task_module = __import__(TaskQueueConfig.\
get_celery_worker_module_name(request.app_id()))
task_func = getattr(
task_module,
TaskQueueConfig.get_queue_function_name(request.queue_name()))
return task_func
except ImportError, import_error:
raise apiproxy_errors.ApplicationError(
taskqueue_service_pb.TaskQueueServiceError.UNKNOWN_QUEUE)
def test_create_celery_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
queue_info = tqc.load_queues_from_file('app_id')
queue_info = tqc.load_queues_from_db()
# making sure it does not throw an exception
self.assertEquals(
tqc.create_celery_file(TaskQueueConfig.QUEUE_INFO_DB),
TaskQueueConfig.CELERY_CONFIG_DIR + "myapp" + ".py")
self.assertEquals(
tqc.create_celery_file(TaskQueueConfig.QUEUE_INFO_FILE),
TaskQueueConfig.CELERY_CONFIG_DIR + "myapp" + ".py")
def test_load_queues_from_file(self):
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml)
flexmock(file_io) \
.should_receive("exists").and_return(True)
flexmock(file_io) \
.should_receive("write").and_return(None)
flexmock(file_io) \
.should_receive("mkdir").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(queue_info, {'queue':[{'name': 'default',
'rate': '5/s'},
{'name': 'foo',
'rate': '10/m'}]})
flexmock(file_io) \
.should_receive("read").and_return('blah').and_raise(IOError)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(queue_info, {'queue':[{'name': 'default',
'rate': '5/s'}]})
flexmock(file_io) \
.should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io) \
.should_receive("write").and_return(None)
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
queue_info = tqc.load_queues_from_file('app_id')
self.assertEquals(queue_info, {'queue':[{'name': 'foo',
'rate': '10/m'},
{'name': 'default',
'rate': '5/s'},
]})
import constants
from google.appengine.runtime import apiproxy_errors
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import datastore_errors
from google.appengine.api import datastore_distributed
from google.appengine.api import datastore
from google.appengine.ext import db
sys.path.append(TaskQueueConfig.CELERY_CONFIG_DIR)
sys.path.append(TaskQueueConfig.CELERY_WORKER_DIR)
app_id = "APP_ID"
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
module_name = TaskQueueConfig.get_celery_worker_module_name(app_id)
celery = Celery(module_name, broker=config.get_broker_string(), backend="amqp")
celery.config_from_object("CELERY_CONFIGURATION")
logger = get_task_logger(__name__)
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed("appscaledashboard", connection_str, require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub("datastore_v3", ds_distrib)
os.environ["APPLICATION_ID"] = "appscaledashboard"
# This template header and tasks can be found in appscale/AppTaskQueue/templates
def test_validate_queue_name(self):
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io).should_receive("write").and_return(None)
flexmock(file_io).should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ, 'myapp')
tqc.validate_queue_name("hello")
tqc.validate_queue_name("hello_hello5354")
try:
tqc.validate_queue_name("hello-hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello$hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello@hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello&hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello*hello")
raise
except NameError:
pass
def test_validate_queue_name(self):
flexmock(file_io).should_receive("read").and_return(sample_queue_yaml2)
flexmock(file_io).should_receive("write").and_return(None)
flexmock(file_io).should_receive("mkdir").and_return(None)
flexmock(datastore).should_receive("Get").\
and_return({TaskQueueConfig.QUEUE_INFO: '{"queue":[{"name": "foo", "rate": "10/m"}]}'})
tqc = TaskQueueConfig(TaskQueueConfig.RABBITMQ,
'myapp')
tqc.validate_queue_name("hello")
tqc.validate_queue_name("hello_hello5354")
try:
tqc.validate_queue_name("hello-hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello$hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello@hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello&hello")
raise
except NameError:
pass
try:
tqc.validate_queue_name("hello*hello")
raise
except NameError:
pass
# Load the queue info
try:
self.__queue_info_cache[app_id] = config.load_queues_from_file(app_id)
config.create_celery_file(TaskQueueConfig.QUEUE_INFO_FILE)
config.create_celery_worker_scripts(TaskQueueConfig.QUEUE_INFO_FILE)
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
except NameError, name_error:
return json.dumps({"error": True, "reason": str(name_error)})
log_file = self.LOG_DIR + app_id + ".log"
command = ["celery",
"worker",
"--app=" + \
TaskQueueConfig.get_celery_worker_module_name(app_id),
#"--autoscale=" + self.MIN_MAX_CONCURRENCY,
"--hostname=" + hostname + "." + app_id,
"--workdir=" + TaskQueueConfig.CELERY_WORKER_DIR,
"--logfile=" + log_file,
"--time-limit=" + str(self.HARD_TIME_LIMIT),
"--soft-time-limit=" + str(self.TASK_SOFT_TIME_LIMIT),
"--pidfile=" + self.PID_FILE_LOC + 'celery___' + \
app_id + ".pid",
"--autoreload"]
start_command = str(' '.join(command))
stop_command = self.get_worker_stop_command(app_id)
watch = "celery-" + str(app_id)
god_config = god_app_configuration.create_config_file(watch,
start_command,
stop_command,
import appscale_info
import constants
from google.appengine.runtime import apiproxy_errors
from google.appengine.api import apiproxy_stub_map
from google.appengine.api import datastore_errors
from google.appengine.api import datastore_distributed
from google.appengine.api import datastore
from google.appengine.ext import db
sys.path.append(TaskQueueConfig.CELERY_CONFIG_DIR)
sys.path.append(TaskQueueConfig.CELERY_WORKER_DIR)
app_id = 'APP_ID'
config = TaskQueueConfig(TaskQueueConfig.RABBITMQ, app_id)
module_name = TaskQueueConfig.get_celery_worker_module_name(app_id)
celery = Celery(module_name, broker=config.get_broker_string(), backend='amqp')
celery.config_from_object('CELERY_CONFIGURATION')
logger = get_task_logger(__name__)
master_db_ip = appscale_info.get_db_master_ip()
connection_str = master_db_ip + ":" + str(constants.DB_SERVER_PORT)
ds_distrib = datastore_distributed.DatastoreDistributed("appscaledashboard",
connection_str,
require_indexes=False)
apiproxy_stub_map.apiproxy.RegisterStub('datastore_v3', ds_distrib)
os.environ['APPLICATION_ID'] = "appscaledashboard"
config.create_celery_worker_scripts(
TaskQueueConfig.QUEUE_INFO_FILE)
except ValueError, value_error:
return json.dumps({"error": True, "reason": str(value_error)})
except NameError, name_error:
return json.dumps({"error": True, "reason": str(name_error)})
except Exception, exception:
logging.error("******Unknown exception******")
logging.exception(exception)
return json.dumps({"error": True, "reason": str(exception)})
log_file = self.LOG_DIR + app_id + ".log"
command = ["/usr/local/bin/celery",
"worker",
"--app=" + \
TaskQueueConfig.get_celery_worker_module_name(app_id),
#"--autoscale=" + self.MIN_MAX_CONCURRENCY,
"--hostname=" + hostname + "." + app_id,
"--workdir=" + TaskQueueConfig.CELERY_WORKER_DIR,
"--logfile=" + log_file,
"--time-limit=" + str(self.HARD_TIME_LIMIT),
"--concurrency=" + str(TaskQueueConfig.CELERY_CONCURRENCY),
"--soft-time-limit=" + str(self.TASK_SOFT_TIME_LIMIT),
"--pidfile=" + self.PID_FILE_LOC + 'celery___' + \
app_id + ".pid",
"--statedb=" + TaskQueueConfig.CELERY_STATE_DIR + 'worker___' + \
app_id + ".db",
"--autoreload -Ofair"]
start_command = str(' '.join(command))
stop_command = self.get_worker_stop_command(app_id)
watch = "celery-" + str(app_id)
