def testNoNamespaces(self): """Test default namespace case only.""" TestEntity().put() jobs = utils.RunMapForKinds( self.operation, [TestEntity.kind()], 'Test job for %(kind)s%(namespace)s', '__main__.foo', self.reader_class_spec, {'test_param': 1}) testutil.execute_all_tasks(self.taskqueue) self.assertEquals(1, len(jobs)) job = jobs[0] state = model.MapreduceState.get_by_job_id(job) self.assertTrue(state) spec = state.mapreduce_spec self.assertTrue(spec) self.assertEquals("Test job for TestEntity", spec.name) mapper = spec.mapper self.assertTrue(mapper) self.assertEquals({'test_param': 1, 'entity_kind': TestEntity.kind()}, mapper.params) self.assertEquals('__main__.foo', mapper.handler_spec) self.assertEquals(self.reader_class_spec, mapper.input_reader_spec)
def _run_map_jobs(job_operation_key, backup_info_key, kinds, job_name, backup_handler, input_reader, output_writer, mapper_params, mapreduce_params, queue): """Creates backup/restore MR jobs for the given operation. Args: job_operation_key: a key of utils.DatastoreAdminOperation entity. backup_info_key: a key of BackupInformation entity. kinds: a list of kinds to run the M/R for. job_name: the M/R job name prefix. backup_handler: M/R job completion handler. input_reader: M/R input reader. output_writer: M/R output writer. mapper_params: custom parameters to pass to mapper. mapreduce_params: dictionary parameters relevant to the whole job. queue: the name of the queue that will be used by the M/R. Returns: Ids of all started mapper jobs as list of strings. """ backup_info = BackupInformation.get(backup_info_key) if not backup_info: return [] jobs = utils.RunMapForKinds(job_operation_key, kinds, job_name, backup_handler, input_reader, output_writer, mapper_params, mapreduce_params, queue_name=queue) backup_info.active_jobs = jobs backup_info.put(force_writes=True) return jobs
def post(self): """Handler for post requests to datastore_admin/delete.do. Jobs are executed and user is redirected to the get handler. """ namespace = self.request.get('namespace') kinds = self.request.get('kind', allow_multiple=True) (namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds) token = self.request.get('xsrf_token') jobs = [] if utils.ValidateXsrfToken(token, XSRF_ACTION): try: op = utils.StartOperation('Deleting %s%s' % (kinds_str, namespace_str)) name_template = 'Delete all %(kind)s objects%(namespace)s' jobs = utils.RunMapForKinds(op.key(), kinds, name_template, self.DELETE_HANDLER, self.INPUT_READER, None, {}) error = '' except Exception, e: error = self._HandleException(e) parameters = [('job', job) for job in jobs] if error: parameters.append(('error', error))
def post(self): """Handler for post requests to datastore_admin/copy.do. Jobs are executed and user is redirected to the get handler. """ namespace = self.request.get('namespace') kinds = self.request.get_all('kind') (namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds) token = self.request.get('xsrf_token') remote_url = self.request.get('remote_url') extra_header = self.request.get('extra_header') jobs = [] if not remote_url: parameters = [('error', 'Unspecified remote URL.')] elif not utils.ValidateXsrfToken(token, XSRF_ACTION): parameters = [('xsrf_error', '1')] else: try: if extra_header: extra_headers = dict([extra_header.split(':', 1)]) else: extra_headers = None target_app = remote_api_put_stub.get_remote_appid(remote_url, extra_headers) op = utils.StartOperation( 'Copying %s%s to %s' % (kinds_str, namespace_str, target_app)) name_template = 'Copy all %(kind)s objects%(namespace)s' mapper_params = { 'target_app': target_app, 'remote_url': remote_url, 'extra_header': extra_header, } jobs = utils.RunMapForKinds( op.key(), kinds, name_template, self.COPY_HANDLER, self.INPUT_READER, None, mapper_params) error = '' except Exception, e: logging.exception('Handling exception.') error = self._HandleException(e) parameters = [('job', job) for job in jobs] if error: parameters.append(('error', error))
def post(self): """Handler for post requests to datastore_admin/delete.do. Jobs are executed and user is redirected to the get handler. """ namespace = self.request.get('namespace') kinds = self.request.get_all('kind') (namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds) token = self.request.get('xsrf_token') readonly_warning = self.request.get('readonly_warning') jobs = [] if (readonly_warning == 'True') and not self.request.get('confirm_readonly_delete'): parameters = [('noconfirm_error', '1')] else: if utils.ValidateXsrfToken(token, XSRF_ACTION): try: op = utils.StartOperation('Deleting %s%s' % (kinds_str, namespace_str)) name_template = 'Delete all %(kind)s objects%(namespace)s' mapreduce_params = {'force_ops_writes': True} queue = self.request.get('queue') queue = queue or os.environ.get( 'HTTP_X_APPENGINE_QUEUENAME', 'default') if queue[0] == '_': queue = 'default' jobs = utils.RunMapForKinds( op.key(), kinds, name_template, self.DELETE_HANDLER, self.INPUT_READER, None, {}, mapreduce_params=mapreduce_params, queue_name=queue, max_shard_count=utils.MAPREDUCE_DEFAULT_SHARDS) error = '' except Exception as e: error = self._HandleException(e) parameters = [('job', job) for job in jobs] if error: parameters.append(('error', error)) else: parameters = [('xsrf_error', '1')] query = urllib.parse.urlencode(parameters) self.redirect('%s/%s?%s' % (config.BASE_PATH, self.SUFFIX, query))
def testProcessNamespace(self): """Test ProcessNamespace function.""" namespace_manager.set_namespace("1") TestEntity().put() namespace_manager.set_namespace(None) namespaces_jobs = utils.RunMapForKinds( self.operation, [TestEntity.kind()], 'Test job for %(kind)s%(namespace)s', '__main__.foo', self.reader_class_spec, {'test_param': 1}) testutil.execute_all_tasks(self.taskqueue) m = mox.Mox() m.StubOutWithMock(context, "get", use_mock_anything=True) ctx = context.Context( model.MapreduceState.get_by_job_id(namespaces_jobs[0]).mapreduce_spec, None) context.get().AndReturn(ctx) context.get().AndReturn(ctx) m.ReplayAll() try: jobs = utils.ProcessNamespace('1') jobs.extend(utils.ProcessNamespace('1')) m.VerifyAll() finally: m.UnsetStubs() testutil.execute_all_tasks(self.taskqueue) self.assertEquals(1, len(jobs)) job = jobs[0] state = model.MapreduceState.get_by_job_id(job) self.assertTrue(state) spec = state.mapreduce_spec self.assertTrue(spec) self.assertEquals("Test job for TestEntity in namespace 1", spec.name) mapper = spec.mapper self.assertTrue(mapper) self.assertEquals({'test_param': 1, 'entity_kind': TestEntity.kind(), 'namespaces': '1'}, mapper.params) self.assertEquals('__main__.foo', mapper.handler_spec) self.assertEquals(self.reader_class_spec, mapper.input_reader_spec)
def _run_map_jobs(job_operation_key, backup_info_key, kinds, job_name, backup_handler, input_reader, output_writer, mapper_params, mapreduce_params, queue): backup_info = BackupInformation.get(backup_info_key) if not backup_info: return [] jobs = utils.RunMapForKinds(job_operation_key, kinds, job_name, backup_handler, input_reader, output_writer, mapper_params, mapreduce_params, queue_name=queue) backup_info.active_jobs = jobs backup_info.put(config=datastore_rpc.Configuration(force_writes=True)) return jobs
def testNamespaces(self): """Test non-default namespaces present.""" namespace_manager.set_namespace("1") TestEntity().put() namespace_manager.set_namespace(None) jobs = utils.RunMapForKinds( self.operation, [TestEntity.kind()], 'Test job for %(kind)s%(namespace)s', '__main__.foo', self.reader_class_spec, {'test_param': 1}) testutil.execute_all_tasks(self.taskqueue) self.assertEquals(1, len(jobs)) job = jobs[0] state = model.MapreduceState.get_by_job_id(job) self.assertTrue(state) spec = state.mapreduce_spec self.assertTrue(spec) self.assertEquals('Test job for TestEntity: discovering namespaces', spec.name) mapper = spec.mapper self.assertTrue(mapper) self.assertEquals({'entity_kind': '__namespace__'}, mapper.params) self.assertEquals(utils.__name__ + "." + utils.ProcessNamespace.__name__, mapper.handler_spec) self.assertEquals( 'google.appengine.ext.mapreduce.input_readers.NamespaceInputReader', mapper.input_reader_spec) self.assertEquals({'kinds': [TestEntity.kind()], 'reader_spec': self.reader_class_spec, 'datastore_admin_operation': str(self.operation.key()), 'mapper_params': {'test_param': 1}, 'handler_spec': '__main__.foo', 'done_callback': '/_ah/datastore_admin/mapreduce_done', 'job_name': 'Test job for %(kind)s%(namespace)s', }, spec.params)
def post(self): """Handler for post requests to datastore_admin/delete.do. Jobs are executed and user is redirected to the get handler. """ namespace = self.request.get('namespace') kinds = self.request.get_all('kind') (namespace_str, kinds_str) = utils.GetPrintableStrs(namespace, kinds) token = self.request.get('xsrf_token') jobs = [] if utils.ValidateXsrfToken(token, XSRF_ACTION): try: op = utils.StartOperation('Deleting %s%s' % (kinds_str, namespace_str)) name_template = 'Delete all %(kind)s objects%(namespace)s' queue = self.request.get('queue') queue = queue or os.environ.get('HTTP_X_APPENGINE_QUEUENAME', 'default') if queue[0] == '_': queue = 'default' jobs = utils.RunMapForKinds( op.key(), kinds, name_template, self.DELETE_HANDLER, self.INPUT_READER, None, {}, queue_name=queue, max_shard_count=utils.MAPREDUCE_DEFAULT_SHARDS) error = '' except Exception, e: error = self._HandleException(e) parameters = [('job', job) for job in jobs] if error: parameters.append(('error', error))