示例#1
0
def execute_task(task):
  """Execute mapper's executor task.

  This will try to determine the correct mapper handler for the task, will set
  up all mock environment necessary for task execution, and execute the task
  itself.

  This function can be used for functional-style testing of functionality
  depending on mapper framework.
  """
  url = task["url"]
  handler = None

  for (re_str, handler_class) in main.create_handlers_map():
    if re.match(re_str, url):
      handler = handler_class()
      break

  if not handler:
    raise Exception("Can't determine handler for %s" % task)

  handler.initialize(mock_webapp.MockRequest(),
                     mock_webapp.MockResponse())
  handler.request.path = url
  for k, v in decode_task_payload(task).items():
    handler.request.set(k, v)

  for k, v in task["headers"]:
    handler.request.headers[k] = v
  handler.post()
示例#2
0
def execute_task(task, handlers_map=None):
    """Execute mapper's executor task.

  This will try to determine the correct mapper handler for the task, will set
  up all mock environment necessary for task execution, and execute the task
  itself.

  This function can be used for functional-style testing of functionality
  depending on mapper framework.
  """
    if not handlers_map:
        handlers_map = main.create_handlers_map()

    url = task["url"]
    handler = None

    for (re_str, handler_class) in handlers_map:
        re_str = "^" + re_str + "($|\\?)"
        if re.match(re_str, url):
            handler = handler_class()
            break

    if not handler:
        raise Exception("Can't determine handler for %s" % task)

    handler.initialize(mock_webapp.MockRequest(), mock_webapp.MockResponse())
    handler.request.set_url(url)

    handler.request.environ["HTTP_HOST"] = "myapp.appspot.com"
    for k, v in task.get("headers", []):
        handler.request.headers[k] = v
        environ_key = "HTTP_" + k.replace("-", "_").upper()
        handler.request.environ[environ_key] = v
    handler.request.environ["HTTP_X_APPENGINE_TASKNAME"] = (task.get(
        "name", "default_task_name"))
    handler.request.environ["HTTP_X_APPENGINE_QUEUENAME"] = (task.get(
        "queue_name", "default"))
    handler.request.environ["PATH_INFO"] = handler.request.path

    saved_os_environ = os.environ
    try:
        os.environ = dict(os.environ)
        os.environ.update(handler.request.environ)
        if task["method"] == "POST":
            for k, v in decode_task_payload(task).items():
                handler.request.set(k, v)
            handler.post()
        elif task["method"] == "GET":
            handler.get()
        else:
            raise Exception("Unsupported method: %s" % task.method)
    finally:
        os.environ = saved_os_environ

    if handler.response.status != 200:
        raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
                        (handler.response.status,
                         handler.response.status_message, task, handler))
def execute_task(task, handlers_map=None):
  """Execute mapper's executor task.

  This will try to determine the correct mapper handler for the task, will set
  up all mock environment necessary for task execution, and execute the task
  itself.

  This function can be used for functional-style testing of functionality
  depending on mapper framework.
  """
  if not handlers_map:
    handlers_map = main.create_handlers_map()

  url = task["url"]
  handler = None

  for (re_str, handler_class) in handlers_map:
    if re.match(re_str, url):
      handler = handler_class()
      break

  if not handler:
    raise Exception("Can't determine handler for %s" % task)

  handler.initialize(mock_webapp.MockRequest(),
                     mock_webapp.MockResponse())
  handler.request.set_url(url)

  handler.request.environ["HTTP_HOST"] = "myapp.appspot.com"
  for k, v in task["headers"]:
    handler.request.headers[k] = v
    environ_key = "HTTP_" + k.replace("-", "_").upper()
    handler.request.environ[environ_key] = v
  handler.request.environ["HTTP_X_APPENGINE_TASKNAME"] = task["name"]
  handler.request.environ["HTTP_X_APPENGINE_QUEUENAME"] = task["queue_name"]
  handler.request.environ["PATH_INFO"] = handler.request.path

  saved_os_environ = os.environ
  try:
    os.environ = dict(os.environ)
    os.environ.update(handler.request.environ)
    if task["method"] == "POST":
      for k, v in decode_task_payload(task).items():
        handler.request.set(k, v)
      handler.post()
    elif task["method"] == "GET":
      handler.get()
    else:
      raise Exception("Unsupported method: %s" % task.method)
  finally:
    os.environ = saved_os_environ

  if handler.response.status != 200:
    raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
                    (handler.response.status,
                     handler.response.status_message,
                     task,
                     handler))
示例#4
0
def register_module():
    """Registers this module in the registry."""

    global_handlers = [
        ('/cron/mapreduce/cleanup', CronMapreduceCleanupHandler),
    ]

    for path, handler_class in mapreduce_main.create_handlers_map():
        # The mapreduce and pipeline libraries are pretty casual about
        # mixing up their UI support in with their functional paths.
        # Here, we separate things and give them different prefixes
        # so that the only-admin-access patterns we define in app.yaml
        # can be reasonably clean.
        if path.startswith('.*/pipeline'):
            if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
                path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
            else:
                path = path.replace('.*/pipeline',
                                    '/mapreduce/worker/pipeline')
        else:
            if '_callback' in path:
                path = path.replace('.*', '/mapreduce/worker', 1)
            elif '/list_configs' in path:
                # This needs mapreduce.yaml, which we don't distribute.  Not
                # having this prevents part of the mapreduce UI front page
                # from loading, but we don't care, because we don't want
                # people using the M/R front page to relaunch jobs anyhow.
                continue
            else:
                path = path.replace('.*', '/mapreduce/ui', 1)

        # The UI needs to be guarded by a config so that casual users aren't
        # exposed to the internals, but advanced users can investigate issues.
        if '/ui/' in path or path.endswith('/ui'):
            if (hasattr(handler_class, 'dispatch')
                    and not hasattr(handler_class, 'real_dispatch')):
                handler_class.real_dispatch = handler_class.dispatch
                handler_class.dispatch = ui_access_wrapper
            global_handlers.append((path, handler_class))

        # Wrap worker handlers with check that request really is coming
        # from task queue.
        else:
            if (hasattr(handler_class, 'dispatch')
                    and not hasattr(handler_class, 'real_dispatch')):
                handler_class.real_dispatch = handler_class.dispatch
                handler_class.dispatch = authorization_wrapper
            global_handlers.append((path, handler_class))

    # Tell map/reduce internals that this is now the base path to use.
    mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'

    global custom_module  # pylint: disable=global-statement
    custom_module = custom_modules.Module(
        MODULE_NAME, 'Provides support for analysis jobs based on map/reduce',
        global_handlers, [])
    return custom_module
def register_module():
    """Registers this module in the registry."""

    global_handlers = [
        ('/cron/mapreduce/cleanup', CronMapreduceCleanupHandler),
    ]

    for path, handler_class in mapreduce_main.create_handlers_map():
        # The mapreduce and pipeline libraries are pretty casual about
        # mixing up their UI support in with their functional paths.
        # Here, we separate things and give them different prefixes
        # so that the only-admin-access patterns we define in app.yaml
        # can be reasonably clean.
        if path.startswith('.*/pipeline'):
            if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
                path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
            else:
                path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
        else:
            if '_callback' in path:
                path = path.replace('.*', '/mapreduce/worker', 1)
            elif '/list_configs' in path:
                # This needs mapreduce.yaml, which we don't distribute.  Not
                # having this prevents part of the mapreduce UI front page
                # from loading, but we don't care, because we don't want
                # people using the M/R front page to relaunch jobs anyhow.
                continue
            else:
                path = path.replace('.*', '/mapreduce/ui', 1)

        # The UI needs to be guarded by a config so that casual users aren't
        # exposed to the internals, but advanced users can investigate issues.
        if '/ui/' in path or path.endswith('/ui'):
            if (hasattr(handler_class, 'dispatch') and
                not hasattr(handler_class, 'real_dispatch')):
                handler_class.real_dispatch = handler_class.dispatch
                handler_class.dispatch = ui_access_wrapper
            global_handlers.append((path, handler_class))

        # Wrap worker handlers with check that request really is coming
        # from task queue.
        else:
            if (hasattr(handler_class, 'dispatch') and
                not hasattr(handler_class, 'real_dispatch')):
                handler_class.real_dispatch = handler_class.dispatch
                handler_class.dispatch = authorization_wrapper
            global_handlers.append((path, handler_class))

    # Tell map/reduce internals that this is now the base path to use.
    mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'

    global custom_module  # pylint: disable=global-statement
    custom_module = custom_modules.Module(
        MODULE_NAME,
        'Provides support for analysis jobs based on map/reduce',
        global_handlers, [])
    return custom_module
def create_app_test_handlers():
    map_reduce_handlers = create_handlers_map()
    app_handlers = [
        ('/delete_data', DeleteDataHandler),
        ('/upload_algorithms', UploadAlgorithmHandler),
        ('/upload_data', UploadDataHandler),
        ('/run_pipeline', ProcessDataHandler),
    ]
    return app_handlers + map_reduce_handlers
示例#7
0
    def notify_module_enabled():
        for path, handler_class in mapreduce_main.create_handlers_map():
            # The mapreduce and pipeline libraries are pretty casual about
            # mixing up their UI support in with their functional paths.
            # Here, we separate things and give them different prefixes
            # so that the only-admin-access patterns we define in app.yaml
            # can be reasonably clean.
            if path.startswith('.*/pipeline'):
                if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
                    path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
                else:
                    path = path.replace('.*/pipeline',
                                        '/mapreduce/worker/pipeline')
            else:
                if '_callback' in path:
                    path = path.replace('.*', '/mapreduce/worker', 1)
                elif '/list_configs' in path:
                    # This needs mapreduce.yaml, which we don't distribute.  Not
                    # having this prevents part of the mapreduce UI front page
                    # from loading, but we don't care, because we don't want
                    # people using the M/R front page to relaunch jobs anyhow.
                    continue
                else:
                    path = path.replace('.*', '/mapreduce/ui', 1)

            # The UI needs to be guarded by a config so that casual users aren't
            # exposed to the internals, but advanced users can
            # investigate issues.
            if '/ui/' in path or path.endswith('/ui'):
                if (hasattr(handler_class, 'dispatch') and
                    not hasattr(handler_class, 'real_dispatch')):
                    handler_class.real_dispatch = handler_class.dispatch
                    handler_class.dispatch = ui_access_wrapper
                global_handlers.append((path, handler_class))

            # Wrap worker handlers with check that request really is coming
            # from task queue.
            else:
                if (hasattr(handler_class, 'dispatch') and
                    not hasattr(handler_class, 'real_dispatch')):
                    handler_class.real_dispatch = handler_class.dispatch
                    handler_class.dispatch = authorization_wrapper
                global_handlers.append((path, handler_class))

        # Tell map/reduce internals that this is now the base path to use.
        mapreduce_parameters.config.BASE_PATH = '/mapreduce/worker'

        # Monkey-patch handler_for_name in mapreduce; the implementation does
        # not correctly handle map, reduce methods marked as @classmethod.
        mapreduce_util.handler_for_name = handler_for_name
示例#8
0
    # is present, we can be certain that the request is internal and from
    # the task queue worker.
    if 'X-AppEngine-TaskName' not in self.request.headers:
        self.response.out.write('Forbidden')
        self.response.set_status(403)
        return
    self.real_dispatch(*args, **kwargs)


def ui_access_wrapper(self, *args, **kwargs):
    self.real_dispatch(*args, **kwargs)


mapreduce_handlers = []

for path, handler_class in mapreduce_main.create_handlers_map():
    if path.startswith('.*/pipeline'):
        if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
            path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
        else:
            path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
    else:
        if '_callback' in path:
            path = path.replace('.*', '/mapreduce/worker', 1)
        elif '/list_configs' in path:
            continue
        else:
            path = path.replace('.*', '/mapreduce/ui', 1)

    if '/ui/' in path or path.endswith('/ui'):
        if (hasattr(handler_class, 'dispatch')
示例#9
0
def execute_task(task, retries=0, handlers_map=None):
  """Execute mapper's executor task.

  This will try to determine the correct mapper handler for the task, will set
  up all mock environment necessary for task execution, and execute the task
  itself.

  This function can be used for functional-style testing of functionality
  depending on mapper framework.

  Args:
    task: a taskqueue task.
    retries: the current retry of this task.
    handlers_map: a dict from url regex to handler.

  Returns:
    the handler instance used for this task.

  Raises:
    Exception: whatever the task raises.
  """
  # Find the handler class
  if not handlers_map:
    handlers_map = main.create_handlers_map()

  url = task["url"]
  handler = None

  for (re_str, handler_class) in handlers_map:
    re_str = "^" + re_str + "($|\\?)"
    if re.match(re_str, url):
      break
  else:
    raise Exception("Can't determine handler for %s" % task)

  request = mock_webapp.MockRequest()
  request.set_url(url)

  # Set dependent env vars if test hasn't set them.
  version = "mr-test-support-version.1"
  module = "mr-test-support-module"
  default_version_hostname = "mr-test-support.appspot.com"
  host = "%s.%s.%s" % (version.split(".")[0],
                       module,
                       default_version_hostname)

  if "CURRENT_VERSION_ID" not in os.environ:
    request.environ["CURRENT_VERSION_ID"] = version
  if "DEFAULT_VERSION_HOSTNAME" not in os.environ:
    request.environ["DEFAULT_VERSION_HOSTNAME"] = (
        default_version_hostname)
  if "CURRENT_MODULE_ID" not in os.environ:
    request.environ["CURRENT_MODULE_ID"] = module
  if "HTTP_HOST" not in os.environ:
    request.environ["HTTP_HOST"] = host

  # Set taskqueue specific headers and env vars.
  for k, v in task.get("headers", []):
    request.headers[k] = v
    environ_key = "HTTP_" + k.replace("-", "_").upper()
    request.environ[environ_key] = v
  request.headers["X-AppEngine-TaskExecutionCount"] = retries
  request.environ["HTTP_X_APPENGINE_TASKNAME"] = (
      task.get("name", "default_task_name"))
  request.environ["HTTP_X_APPENGINE_QUEUENAME"] = (
      task.get("queue_name", "default"))
  request.environ["PATH_INFO"] = request.path

  if task["method"] == "POST":
    # taskqueue_stub base64 encodes body when it returns the task to us.
    request.body = base64.b64decode(task["body"])
    for k, v in decode_task_payload(task).iteritems():
      request.set(k, v)

  response = mock_webapp.MockResponse()
  try:
    # Webapp2 expects request/response in the handler instantiation, and calls
    # initialize automatically.
    handler = handler_class(request, response)
  except TypeError:
    # For webapp, setup request before calling initialize.
    handler = handler_class()
    handler.initialize(request, response)

  saved_os_environ = os.environ
  try:
    os.environ = dict(os.environ)
    os.environ.update(request.environ)
    if task["method"] == "POST":
      handler.post()
    elif task["method"] == "GET":
      handler.get()
    else:
      raise Exception("Unsupported method: %s" % task.method)
  finally:
    os.environ = saved_os_environ

  if handler.response.status != 200:
    raise Exception("Handler failure: %s (%s). \nTask: %s\nHandler: %s" %
                    (handler.response.status,
                     handler.response.status_message,
                     task,
                     handler))
  return handler
示例#10
0
    # is present, we can be certain that the request is internal and from
    # the task queue worker.
    if 'X-AppEngine-TaskName' not in self.request.headers:
        self.response.out.write('Forbidden')
        self.response.set_status(403)
        return
    self.real_dispatch(*args, **kwargs)


def ui_access_wrapper(self, *args, **kwargs):
    self.real_dispatch(*args, **kwargs)


MAPREDUCE_HANDLERS = []

for path, handler_class in mapreduce_main.create_handlers_map():
    if path.startswith('.*/pipeline'):
        if 'pipeline/rpc/' in path or path == '.*/pipeline(/.+)':
            path = path.replace('.*/pipeline', '/mapreduce/ui/pipeline')
        else:
            path = path.replace('.*/pipeline', '/mapreduce/worker/pipeline')
    else:
        if '_callback' in path:
            path = path.replace('.*', '/mapreduce/worker', 1)
        elif '/list_configs' in path:
            continue
        else:
            path = path.replace('.*', '/mapreduce/ui', 1)

    if '/ui/' in path or path.endswith('/ui'):
        if (hasattr(handler_class, 'dispatch') and
from django.conf.urls import patterns
from djangae.utils import djangae_webapp

from django.views.decorators.csrf import csrf_exempt

try:
    from mapreduce.main import create_handlers_map
    wrapped_urls = ((url_re.replace('.*/', '^',
                                    1), csrf_exempt(djangae_webapp(func)))
                    for url_re, func in create_handlers_map())
except ImportError as e:
    wrapped_urls = []

urlpatterns = patterns('', *wrapped_urls)
示例#12
0
from django.conf.urls import url
from djangae.utils import djangae_webapp

from django.views.decorators.csrf import csrf_exempt

# The Mapreduce status UI uses inline JS, which will fail If we have django-csp
# installed and are not allowing 'unsafe-inline' as a SCRIPT_SRC.
try:
    from csp.decorators import csp_update
    exempt_from_unsafe_inline = csp_update(SCRIPT_SRC=("'unsafe-inline'",))
except ImportError:
    exempt_from_unsafe_inline = lambda func: func


try:
    from mapreduce.main import create_handlers_map
    wrapped_urls = [
        url(
            url_re.replace('.*/', '^', 1), 
            exempt_from_unsafe_inline(csrf_exempt(djangae_webapp(func)))
        ) 
        for url_re, func in create_handlers_map()
    ]
except ImportError as e:
    wrapped_urls = []


urlpatterns = wrapped_urls
示例#13
0
from django.conf.urls import patterns
from djangae.utils import djangae_webapp


try:
    from mapreduce.main import create_handlers_map
    wrapped_urls = ((url_re.replace('.*/', '^', 1), djangae_webapp(func)) for url_re, func in create_handlers_map())
except ImportError as e:
    wrapped_urls = []


urlpatterns = patterns('', *wrapped_urls)
示例#14
0
from django.conf.urls import patterns
from djangae.utils import djangae_webapp

from django.views.decorators.csrf import csrf_exempt

try:
    from mapreduce.main import create_handlers_map
    wrapped_urls = ((url_re.replace('.*/', '^', 1), csrf_exempt(djangae_webapp(func))) for url_re, func in create_handlers_map())
except ImportError as e:
    wrapped_urls = []


urlpatterns = patterns('', *wrapped_urls)
# -*- coding: utf-8 -*-
# Copyright 2017 GIG Technology NV
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# @@license_version:1.3@@

from mapreduce.main import create_handlers_map
from rogerthat.wsgi import RogerthatWSGIApplication

handlers = create_handlers_map()

app = RogerthatWSGIApplication(handlers, True, name="main_mapreduce", google_authenticated=True)