def testNoNamespaces(self):
"""Test default namespace case only."""
TestEntity().put()
jobs = utils.RunMapForKinds(
self.operation,
[TestEntity.kind()],
'Test job for %(kind)s%(namespace)s',
'__main__.foo',
self.reader_class_spec,
{'test_param': 1})
testutil.execute_all_tasks(self.taskqueue)
self.assertEquals(1, len(jobs))
job = jobs[0]
state = model.MapreduceState.get_by_job_id(job)
self.assertTrue(state)
spec = state.mapreduce_spec
self.assertTrue(spec)
self.assertEquals("Test job for TestEntity", spec.name)
mapper = spec.mapper
self.assertTrue(mapper)
self.assertEquals({'test_param': 1,
'entity_kind': TestEntity.kind()},
mapper.params)
self.assertEquals('__main__.foo', mapper.handler_spec)
self.assertEquals(self.reader_class_spec, mapper.input_reader_spec)
def _run_map_jobs(job_operation_key, backup_info_key, kinds, job_name,
backup_handler, input_reader, output_writer, mapper_params,
mapreduce_params, queue):
"""Creates backup/restore MR jobs for the given operation.
Args:
job_operation_key: a key of utils.DatastoreAdminOperation entity.
backup_info_key: a key of BackupInformation entity.
kinds: a list of kinds to run the M/R for.
job_name: the M/R job name prefix.
backup_handler: M/R job completion handler.
input_reader: M/R input reader.
output_writer: M/R output writer.
mapper_params: custom parameters to pass to mapper.
mapreduce_params: dictionary parameters relevant to the whole job.
queue: the name of the queue that will be used by the M/R.
Returns:
Ids of all started mapper jobs as list of strings.
"""
backup_info = BackupInformation.get(backup_info_key)
if not backup_info:
return []
jobs = utils.RunMapForKinds(job_operation_key,
kinds,
job_name,
backup_handler,
input_reader,
output_writer,
mapper_params,
mapreduce_params,
queue_name=queue)
backup_info.active_jobs = jobs
backup_info.put(force_writes=True)
return jobs
def post(self):
"""Handler for post requests to datastore_admin/delete.do.
Jobs are executed and user is redirected to the get handler.
"""
namespace = self.request.get('namespace')
kinds = self.request.get('kind', allow_multiple=True)
(namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds)
token = self.request.get('xsrf_token')
jobs = []
if utils.ValidateXsrfToken(token, XSRF_ACTION):
try:
op = utils.StartOperation('Deleting %s%s' %
(kinds_str, namespace_str))
name_template = 'Delete all %(kind)s objects%(namespace)s'
jobs = utils.RunMapForKinds(op.key(), kinds, name_template,
self.DELETE_HANDLER,
self.INPUT_READER, None, {})
error = ''
except Exception, e:
error = self._HandleException(e)
parameters = [('job', job) for job in jobs]
if error:
parameters.append(('error', error))
def post(self):
"""Handler for post requests to datastore_admin/copy.do.
Jobs are executed and user is redirected to the get handler.
"""
namespace = self.request.get('namespace')
kinds = self.request.get_all('kind')
(namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds)
token = self.request.get('xsrf_token')
remote_url = self.request.get('remote_url')
extra_header = self.request.get('extra_header')
jobs = []
if not remote_url:
parameters = [('error', 'Unspecified remote URL.')]
elif not utils.ValidateXsrfToken(token, XSRF_ACTION):
parameters = [('xsrf_error', '1')]
else:
try:
if extra_header:
extra_headers = dict([extra_header.split(':', 1)])
else:
extra_headers = None
target_app = remote_api_put_stub.get_remote_appid(remote_url,
extra_headers)
op = utils.StartOperation(
'Copying %s%s to %s' % (kinds_str, namespace_str, target_app))
name_template = 'Copy all %(kind)s objects%(namespace)s'
mapper_params = {
'target_app': target_app,
'remote_url': remote_url,
'extra_header': extra_header,
}
jobs = utils.RunMapForKinds(
op.key(),
kinds,
name_template,
self.COPY_HANDLER,
self.INPUT_READER,
None,
mapper_params)
error = ''
except Exception, e:
logging.exception('Handling exception.')
error = self._HandleException(e)
parameters = [('job', job) for job in jobs]
if error:
parameters.append(('error', error))
def post(self):
"""Handler for post requests to datastore_admin/delete.do.
Jobs are executed and user is redirected to the get handler.
"""
namespace = self.request.get('namespace')
kinds = self.request.get_all('kind')
(namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds)
token = self.request.get('xsrf_token')
readonly_warning = self.request.get('readonly_warning')
jobs = []
if (readonly_warning
== 'True') and not self.request.get('confirm_readonly_delete'):
parameters = [('noconfirm_error', '1')]
else:
if utils.ValidateXsrfToken(token, XSRF_ACTION):
try:
op = utils.StartOperation('Deleting %s%s' %
(kinds_str, namespace_str))
name_template = 'Delete all %(kind)s objects%(namespace)s'
mapreduce_params = {'force_ops_writes': True}
queue = self.request.get('queue')
queue = queue or os.environ.get(
'HTTP_X_APPENGINE_QUEUENAME', 'default')
if queue[0] == '_':
queue = 'default'
jobs = utils.RunMapForKinds(
op.key(),
kinds,
name_template,
self.DELETE_HANDLER,
self.INPUT_READER,
None, {},
mapreduce_params=mapreduce_params,
queue_name=queue,
max_shard_count=utils.MAPREDUCE_DEFAULT_SHARDS)
error = ''
except Exception as e:
error = self._HandleException(e)
parameters = [('job', job) for job in jobs]
if error:
parameters.append(('error', error))
else:
parameters = [('xsrf_error', '1')]
query = urllib.parse.urlencode(parameters)
self.redirect('%s/%s?%s' % (config.BASE_PATH, self.SUFFIX, query))
def testProcessNamespace(self):
"""Test ProcessNamespace function."""
namespace_manager.set_namespace("1")
TestEntity().put()
namespace_manager.set_namespace(None)
namespaces_jobs = utils.RunMapForKinds(
self.operation,
[TestEntity.kind()],
'Test job for %(kind)s%(namespace)s',
'__main__.foo',
self.reader_class_spec,
{'test_param': 1})
testutil.execute_all_tasks(self.taskqueue)
m = mox.Mox()
m.StubOutWithMock(context, "get", use_mock_anything=True)
ctx = context.Context(
model.MapreduceState.get_by_job_id(namespaces_jobs[0]).mapreduce_spec,
None)
context.get().AndReturn(ctx)
context.get().AndReturn(ctx)
m.ReplayAll()
try:
jobs = utils.ProcessNamespace('1')
jobs.extend(utils.ProcessNamespace('1'))
m.VerifyAll()
finally:
m.UnsetStubs()
testutil.execute_all_tasks(self.taskqueue)
self.assertEquals(1, len(jobs))
job = jobs[0]
state = model.MapreduceState.get_by_job_id(job)
self.assertTrue(state)
spec = state.mapreduce_spec
self.assertTrue(spec)
self.assertEquals("Test job for TestEntity in namespace 1", spec.name)
mapper = spec.mapper
self.assertTrue(mapper)
self.assertEquals({'test_param': 1,
'entity_kind': TestEntity.kind(),
'namespaces': '1'},
mapper.params)
self.assertEquals('__main__.foo', mapper.handler_spec)
self.assertEquals(self.reader_class_spec, mapper.input_reader_spec)
def _run_map_jobs(job_operation_key, backup_info_key, kinds, job_name,
backup_handler, input_reader, output_writer, mapper_params,
mapreduce_params, queue):
backup_info = BackupInformation.get(backup_info_key)
if not backup_info:
return []
jobs = utils.RunMapForKinds(job_operation_key,
kinds,
job_name,
backup_handler,
input_reader,
output_writer,
mapper_params,
mapreduce_params,
queue_name=queue)
backup_info.active_jobs = jobs
backup_info.put(config=datastore_rpc.Configuration(force_writes=True))
return jobs
def testNamespaces(self):
"""Test non-default namespaces present."""
namespace_manager.set_namespace("1")
TestEntity().put()
namespace_manager.set_namespace(None)
jobs = utils.RunMapForKinds(
self.operation,
[TestEntity.kind()],
'Test job for %(kind)s%(namespace)s',
'__main__.foo',
self.reader_class_spec,
{'test_param': 1})
testutil.execute_all_tasks(self.taskqueue)
self.assertEquals(1, len(jobs))
job = jobs[0]
state = model.MapreduceState.get_by_job_id(job)
self.assertTrue(state)
spec = state.mapreduce_spec
self.assertTrue(spec)
self.assertEquals('Test job for TestEntity: discovering namespaces',
spec.name)
mapper = spec.mapper
self.assertTrue(mapper)
self.assertEquals({'entity_kind': '__namespace__'},
mapper.params)
self.assertEquals(utils.__name__ + "." + utils.ProcessNamespace.__name__,
mapper.handler_spec)
self.assertEquals(
'google.appengine.ext.mapreduce.input_readers.NamespaceInputReader',
mapper.input_reader_spec)
self.assertEquals({'kinds': [TestEntity.kind()],
'reader_spec': self.reader_class_spec,
'datastore_admin_operation': str(self.operation.key()),
'mapper_params': {'test_param': 1},
'handler_spec': '__main__.foo',
'done_callback': '/_ah/datastore_admin/mapreduce_done',
'job_name': 'Test job for %(kind)s%(namespace)s',
},
spec.params)
def post(self):
"""Handler for post requests to datastore_admin/delete.do.
Jobs are executed and user is redirected to the get handler.
"""
namespace = self.request.get('namespace')
kinds = self.request.get_all('kind')
(namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds)
token = self.request.get('xsrf_token')
jobs = []
if utils.ValidateXsrfToken(token, XSRF_ACTION):
try:
op = utils.StartOperation('Deleting %s%s' %
(kinds_str, namespace_str))
name_template = 'Delete all %(kind)s objects%(namespace)s'
queue = self.request.get('queue')
queue = queue or os.environ.get('HTTP_X_APPENGINE_QUEUENAME',
'default')
if queue[0] == '_':
queue = 'default'
jobs = utils.RunMapForKinds(
op.key(),
kinds,
name_template,
self.DELETE_HANDLER,
self.INPUT_READER,
None, {},
queue_name=queue,
max_shard_count=utils.MAPREDUCE_DEFAULT_SHARDS)
error = ''
except Exception, e:
error = self._HandleException(e)
parameters = [('job', job) for job in jobs]
if error:
parameters.append(('error', error))
