def execute_task(task):
"""Execute mapper's executor task.
This will try to determine the correct mapper handler for the task, will set
up all mock environment necessary for task execution, and execute the task
itself.
This function can be used for functional-style testing of functionality
depending on mapper framework.
"""
url = task["url"]
handler = None
for (re_str, handler_class) in main.create_handlers_map():
if re.match(re_str, url):
handler = handler_class()
break
if not handler:
raise Exception("Can't determine handler for %s" % task)
handler.initialize(mock_webapp.MockRequest(),
mock_webapp.MockResponse())
handler.request.path = url
for k, v in decode_task_payload(task).items():
handler.request.set(k, v)
for k, v in task["headers"]:
handler.request.headers[k] = v
handler.post()
def execute_task(task, handlers_map=None):
"""Execute mapper's executor task.
This will try to determine the correct mapper handler for the task, will set
up all mock environment necessary for task execution, and execute the task
itself.
This function can be used for functional-style testing of functionality
depending on mapper framework.
"""
if not handlers_map:
handlers_map = main.create_handlers_map()
url = task["url"]
handler = None
for (re_str, handler_class) in handlers_map:
re_str = "^" + re_str + "($|\\?)"
if re.match(re_str, url):
handler = handler_class()
break
if not handler:
raise Exception("Can't determine handler for %s" % task)
handler.initialize(mock_webapp.MockRequest(), mock_webapp.MockResponse())
handler.request.set_url(url)
handler.request.environ["HTTP_HOST"] = "myapp.appspot.com"
for k, v in task.get("headers", []):
handler.request.headers[k] = v
environ_key = "HTTP_" + k.replace("-", "_").upper()
handler.request.environ[environ_key] = v
handler.request.environ["HTTP_X_APPENGINE_TASKNAME"] = (task.get(
"name", "default_task_name"))
handler.request.environ["HTTP_X_APPENGINE_QUEUENAME"] = (task.get(
"queue_name", "default"))
handler.request.environ["PATH_INFO"] = handler.request.path
saved_os_environ = os.environ
try:
os.environ = dict(os.environ)
os.environ.update(handler.request.environ)
if task["method"] == "POST":
for k, v in decode_task_payload(task).items():
handler.request.set(k, v)
handler.post()
elif task["method"] == "GET":
handler.get()
else:
raise Exception("Unsupported method: %s" % task.method)
finally:
os.environ = saved_os_environ
if handler.response.status != 200:
raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
(handler.response.status,
handler.response.status_message, task, handler))
def execute_task(task, handlers_map=None):
"""Execute mapper's executor task.
This will try to determine the correct mapper handler for the task, will set
up all mock environment necessary for task execution, and execute the task
itself.
This function can be used for functional-style testing of functionality
depending on mapper framework.
"""
if not handlers_map:
handlers_map = main.create_handlers_map()
url = task["url"]
handler = None
for (re_str, handler_class) in handlers_map:
if re.match(re_str, url):
handler = handler_class()
break
if not handler:
raise Exception("Can't determine handler for %s" % task)
handler.initialize(mock_webapp.MockRequest(),
mock_webapp.MockResponse())
handler.request.set_url(url)
handler.request.environ["HTTP_HOST"] = "myapp.appspot.com"
for k, v in task["headers"]:
handler.request.headers[k] = v
environ_key = "HTTP_" + k.replace("-", "_").upper()
handler.request.environ[environ_key] = v
handler.request.environ["HTTP_X_APPENGINE_TASKNAME"] = task["name"]
handler.request.environ["HTTP_X_APPENGINE_QUEUENAME"] = task["queue_name"]
handler.request.environ["PATH_INFO"] = handler.request.path
saved_os_environ = os.environ
try:
os.environ = dict(os.environ)
os.environ.update(handler.request.environ)
if task["method"] == "POST":
for k, v in decode_task_payload(task).items():
handler.request.set(k, v)
handler.post()
elif task["method"] == "GET":
handler.get()
else:
raise Exception("Unsupported method: %s" % task.method)
finally:
os.environ = saved_os_environ
if handler.response.status != 200:
raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
(handler.response.status,
handler.response.status_message,
task,
handler))
def register_module():
"""Registers this module in the registry."""
global_handlers = [
('/cron/mapreduce/cleanup', CronMapreduceCleanupHandler),
]
for path, handler_class in mapreduce_main.create_handlers_map():
# The mapreduce and pipeline libraries are pretty casual about
# mixing up their UI support in with their functional paths.
# Here, we separate things and give them different prefixes
# so that the only-admin-access patterns we define in app.yaml
# can be reasonably clean.
if path.startswith('.*/pipeline'):
if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
else:
path = path.replace('.*/pipeline',
'/mapreduce/worker/pipeline')
else:
if '_callback' in path:
path = path.replace('.*', '/mapreduce/worker', 1)
elif '/list_configs' in path:
# This needs mapreduce.yaml, which we don't distribute. Not
# having this prevents part of the mapreduce UI front page
# from loading, but we don't care, because we don't want
# people using the M/R front page to relaunch jobs anyhow.
continue
else:
path = path.replace('.*', '/mapreduce/ui', 1)
# The UI needs to be guarded by a config so that casual users aren't
# exposed to the internals, but advanced users can investigate issues.
if '/ui/' in path or path.endswith('/ui'):
if (hasattr(handler_class, 'dispatch')
and not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = ui_access_wrapper
global_handlers.append((path, handler_class))
# Wrap worker handlers with check that request really is coming
# from task queue.
else:
if (hasattr(handler_class, 'dispatch')
and not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = authorization_wrapper
global_handlers.append((path, handler_class))
# Tell map/reduce internals that this is now the base path to use.
mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'
global custom_module # pylint: disable=global-statement
custom_module = custom_modules.Module(
MODULE_NAME, 'Provides support for analysis jobs based on map/reduce',
global_handlers, [])
return custom_module
def register_module():
"""Registers this module in the registry."""
global_handlers = [
('/cron/mapreduce/cleanup', CronMapreduceCleanupHandler),
]
for path, handler_class in mapreduce_main.create_handlers_map():
# The mapreduce and pipeline libraries are pretty casual about
# mixing up their UI support in with their functional paths.
# Here, we separate things and give them different prefixes
# so that the only-admin-access patterns we define in app.yaml
# can be reasonably clean.
if path.startswith('.*/pipeline'):
if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
else:
path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
else:
if '_callback' in path:
path = path.replace('.*', '/mapreduce/worker', 1)
elif '/list_configs' in path:
# This needs mapreduce.yaml, which we don't distribute. Not
# having this prevents part of the mapreduce UI front page
# from loading, but we don't care, because we don't want
# people using the M/R front page to relaunch jobs anyhow.
continue
else:
path = path.replace('.*', '/mapreduce/ui', 1)
# The UI needs to be guarded by a config so that casual users aren't
# exposed to the internals, but advanced users can investigate issues.
if '/ui/' in path or path.endswith('/ui'):
if (hasattr(handler_class, 'dispatch') and
not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = ui_access_wrapper
global_handlers.append((path, handler_class))
# Wrap worker handlers with check that request really is coming
# from task queue.
else:
if (hasattr(handler_class, 'dispatch') and
not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = authorization_wrapper
global_handlers.append((path, handler_class))
# Tell map/reduce internals that this is now the base path to use.
mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'
global custom_module # pylint: disable=global-statement
custom_module = custom_modules.Module(
MODULE_NAME,
'Provides support for analysis jobs based on map/reduce',
global_handlers, [])
return custom_module
def create_app_test_handlers():
map_reduce_handlers = create_handlers_map()
app_handlers = [
('/delete_data', DeleteDataHandler),
('/upload_algorithms', UploadAlgorithmHandler),
('/upload_data', UploadDataHandler),
('/run_pipeline', ProcessDataHandler),
]
return app_handlers + map_reduce_handlers
def notify_module_enabled():
for path, handler_class in mapreduce_main.create_handlers_map():
# The mapreduce and pipeline libraries are pretty casual about
# mixing up their UI support in with their functional paths.
# Here, we separate things and give them different prefixes
# so that the only-admin-access patterns we define in app.yaml
# can be reasonably clean.
if path.startswith('.*/pipeline'):
if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
else:
path = path.replace('.*/pipeline',
'/mapreduce/worker/pipeline')
else:
if '_callback' in path:
path = path.replace('.*', '/mapreduce/worker', 1)
elif '/list_configs' in path:
# This needs mapreduce.yaml, which we don't distribute. Not
# having this prevents part of the mapreduce UI front page
# from loading, but we don't care, because we don't want
# people using the M/R front page to relaunch jobs anyhow.
continue
else:
path = path.replace('.*', '/mapreduce/ui', 1)
# The UI needs to be guarded by a config so that casual users aren't
# exposed to the internals, but advanced users can
# investigate issues.
if '/ui/' in path or path.endswith('/ui'):
if (hasattr(handler_class, 'dispatch') and
not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = ui_access_wrapper
global_handlers.append((path, handler_class))
# Wrap worker handlers with check that request really is coming
# from task queue.
else:
if (hasattr(handler_class, 'dispatch') and
not hasattr(handler_class, 'real_dispatch')):
handler_class.real_dispatch = handler_class.dispatch
handler_class.dispatch = authorization_wrapper
global_handlers.append((path, handler_class))
# Tell map/reduce internals that this is now the base path to use.
mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'
# Monkey-patch handler_for_name in mapreduce; the implementation does
# not correctly handle map, reduce methods marked as @classmethod.
mapreduce_util.handler_for_name = handler_for_name
# is present, we can be certain that the request is internal and from
# the task queue worker.
if 'X-AppEngine-TaskName' not in self.request.headers:
self.response.out.write('Forbidden')
self.response.set_status(403)
return
self.real_dispatch(*args, **kwargs)
def ui_access_wrapper(self, *args, **kwargs):
self.real_dispatch(*args, **kwargs)
mapreduce_handlers = []
for path, handler_class in mapreduce_main.create_handlers_map():
if path.startswith('.*/pipeline'):
if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
else:
path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
else:
if '_callback' in path:
path = path.replace('.*', '/mapreduce/worker', 1)
elif '/list_configs' in path:
continue
else:
path = path.replace('.*', '/mapreduce/ui', 1)
if '/ui/' in path or path.endswith('/ui'):
if (hasattr(handler_class, 'dispatch')
def execute_task(task, retries=0, handlers_map=None):
"""Execute mapper's executor task.
This will try to determine the correct mapper handler for the task, will set
up all mock environment necessary for task execution, and execute the task
itself.
This function can be used for functional-style testing of functionality
depending on mapper framework.
Args:
task: a taskqueue task.
retries: the current retry of this task.
handlers_map: a dict from url regex to handler.
Returns:
the handler instance used for this task.
Raises:
Exception: whatever the task raises.
"""
# Find the handler class
if not handlers_map:
handlers_map = main.create_handlers_map()
url = task["url"]
handler = None
for (re_str, handler_class) in handlers_map:
re_str = "^" + re_str + "($|\\?)"
if re.match(re_str, url):
break
else:
raise Exception("Can't determine handler for %s" % task)
request = mock_webapp.MockRequest()
request.set_url(url)
# Set dependent env vars if test hasn't set them.
version = "mr-test-support-version.1"
module = "mr-test-support-module"
default_version_hostname = "mr-test-support.appspot.com"
host = "%s.%s.%s" % (version.split(".")[0],
module,
default_version_hostname)
if "CURRENT_VERSION_ID" not in os.environ:
request.environ["CURRENT_VERSION_ID"] = version
if "DEFAULT_VERSION_HOSTNAME" not in os.environ:
request.environ["DEFAULT_VERSION_HOSTNAME"] = (
default_version_hostname)
if "CURRENT_MODULE_ID" not in os.environ:
request.environ["CURRENT_MODULE_ID"] = module
if "HTTP_HOST" not in os.environ:
request.environ["HTTP_HOST"] = host
# Set taskqueue specific headers and env vars.
for k, v in task.get("headers", []):
request.headers[k] = v
environ_key = "HTTP_" + k.replace("-", "_").upper()
request.environ[environ_key] = v
request.headers["X-AppEngine-TaskExecutionCount"] = retries
request.environ["HTTP_X_APPENGINE_TASKNAME"] = (
task.get("name", "default_task_name"))
request.environ["HTTP_X_APPENGINE_QUEUENAME"] = (
task.get("queue_name", "default"))
request.environ["PATH_INFO"] = request.path
if task["method"] == "POST":
# taskqueue_stub base64 encodes body when it returns the task to us.
request.body = base64.b64decode(task["body"])
for k, v in decode_task_payload(task).iteritems():
request.set(k, v)
response = mock_webapp.MockResponse()
try:
# Webapp2 expects request/response in the handler instantiation, and calls
# initialize automatically.
handler = handler_class(request, response)
except TypeError:
# For webapp, setup request before calling initialize.
handler = handler_class()
handler.initialize(request, response)
saved_os_environ = os.environ
try:
os.environ = dict(os.environ)
os.environ.update(request.environ)
if task["method"] == "POST":
handler.post()
elif task["method"] == "GET":
handler.get()
else:
raise Exception("Unsupported method: %s" % task.method)
finally:
os.environ = saved_os_environ
if handler.response.status != 200:
raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
(handler.response.status,
handler.response.status_message,
task,
handler))
return handler
# is present, we can be certain that the request is internal and from
# the task queue worker.
if 'X-AppEngine-TaskName' not in self.request.headers:
self.response.out.write('Forbidden')
self.response.set_status(403)
return
self.real_dispatch(*args, **kwargs)
def ui_access_wrapper(self, *args, **kwargs):
self.real_dispatch(*args, **kwargs)
MAPREDUCE_HANDLERS = []
for path, handler_class in mapreduce_main.create_handlers_map():
if path.startswith('.*/pipeline'):
if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
else:
path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
else:
if '_callback' in path:
path = path.replace('.*', '/mapreduce/worker', 1)
elif '/list_configs' in path:
continue
else:
path = path.replace('.*', '/mapreduce/ui', 1)
if '/ui/' in path or path.endswith('/ui'):
if (hasattr(handler_class, 'dispatch') and
from django.conf.urls import patterns
from djangae.utils import djangae_webapp
from django.views.decorators.csrf import csrf_exempt
try:
from mapreduce.main import create_handlers_map
wrapped_urls = ((url_re.replace('.*/', '^',
1), csrf_exempt(djangae_webapp(func)))
for url_re, func in create_handlers_map())
except ImportError as e:
wrapped_urls = []
urlpatterns = patterns('', *wrapped_urls)
from django.conf.urls import url
from djangae.utils import djangae_webapp
from django.views.decorators.csrf import csrf_exempt
# The Mapreduce status UI uses inline JS, which will fail If we have django-csp
# installed and are not allowing 'unsafe-inline' as a SCRIPT_SRC.
try:
from csp.decorators import csp_update
exempt_from_unsafe_inline = csp_update(SCRIPT_SRC=("'unsafe-inline'",))
except ImportError:
exempt_from_unsafe_inline = lambda func: func
try:
from mapreduce.main import create_handlers_map
wrapped_urls = [
url(
url_re.replace('.*/', '^', 1),
exempt_from_unsafe_inline(csrf_exempt(djangae_webapp(func)))
)
for url_re, func in create_handlers_map()
]
except ImportError as e:
wrapped_urls = []
urlpatterns = wrapped_urls
from django.conf.urls import patterns
from djangae.utils import djangae_webapp
try:
from mapreduce.main import create_handlers_map
wrapped_urls = ((url_re.replace('.*/', '^', 1), djangae_webapp(func)) for url_re, func in create_handlers_map())
except ImportError as e:
wrapped_urls = []
urlpatterns = patterns('', *wrapped_urls)
from django.conf.urls import patterns
from djangae.utils import djangae_webapp
from django.views.decorators.csrf import csrf_exempt
try:
from mapreduce.main import create_handlers_map
wrapped_urls = ((url_re.replace('.*/', '^', 1), csrf_exempt(djangae_webapp(func))) for url_re, func in create_handlers_map())
except ImportError as e:
wrapped_urls = []
urlpatterns = patterns('', *wrapped_urls)
# -*- coding: utf-8 -*- # Copyright 2017 GIG Technology NV # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # # @@license_version:1.3@@ from mapreduce.main import create_handlers_map from rogerthat.wsgi import RogerthatWSGIApplication handlers = create_handlers_map() app = RogerthatWSGIApplication(handlers, True, name="main_mapreduce", google_authenticated=True)