def __init__(self, request=None, response=None):
super(OnDemandTableBackupHandler, self).__init__(request, response)
# now let's check if this task is not a retry of some previous (which
# failed for some reason) if so - let's log when it hits the defined
# mark so we can catch it on monitoring:
Tasks.log_task_metadata_for(request=self.request)
def schedule_backup_task(self, projectId, datasetId, tableId):
logging.info("Schedule_backup_task: '%s:%s.%s'", projectId, datasetId,
tableId)
task = Tasks.create(method='GET',
url='/tasks/backups/table/{0}/{1}/{2}'.format(
projectId, datasetId, tableId))
Tasks.schedule('backup-worker', task)
def _schedule_partitioned_table_backup_scheduler_task(table_reference):
Tasks.schedule(
'backup-scheduler',
TaskCreator.create_partitioned_table_backup_scheduler_task(
project_id=table_reference.get_project_id(),
dataset_id=table_reference.get_dataset_id(),
table_id=table_reference.get_table_id()))
def __backup_dataset(self, project_id, dataset_id):
logging.info('Backing up dataset: %s', dataset_id)
task = Tasks.create(url='/tasks/backups/dataset',
params={
'projectId': project_id,
'datasetId': dataset_id
},
headers={
request_correlation_id.HEADER_NAME:
self.request_correlation_id
})
Tasks.schedule('backup-scheduler', task)
def create_slo_recalculation_tasks():
tasks = [
Tasks.create(method='POST',
url='/sli/latency_for_x_days',
params={'x_days': x_days})
for x_days in [4, 5, 7]
]
tasks.append(
Tasks.create(
method='POST',
url='/sli/quality'))
return tasks
def _schedule_tasks_for_tables_backup(project_id, dataset_id, table_ids):
logging.info(
u'Scheduling Table Backup tasks for %s %s:%s dataset tables: %s',
len(table_ids), project_id, dataset_id, table_ids)
tasks = []
for table_id in table_ids:
tasks.append(
TaskCreator.create_table_backup_task(project_id=project_id,
dataset_id=dataset_id,
table_id=table_id))
Tasks.schedule('backup-worker', tasks)
def test_schedule_can_work_with_single_task(self):
# given
task1 = Task(
url='/example/task1',
)
# when
Tasks.schedule("default", task1)
# then
executed_tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names="default"
)
self.assertEqual(len(executed_tasks), 1, "Should create one task in queue")
self.assertEqual(executed_tasks[0].url, '/example/task1')
def _schedule_dataset_backup_scheduler_tasks(project_id, dataset_ids):
logging.info(
u'Scheduling Dataset Backup Scheduler tasks for %s %s project datasets: %s.',
len(dataset_ids), project_id, dataset_ids)
tasks = []
for dataset_id in dataset_ids:
tasks.append(
TaskCreator.create_dataset_backup_scheduler_task(
project_id=project_id,
dataset_id=dataset_id))
Tasks.schedule('backup-scheduler', tasks)
def schedule_tasks_for_partition_backup(project_id, dataset_id, table_id,
partition_ids):
logging.info(
"Scheduling backup tasks for table: '%s/%s/%s partitions: %s'",
project_id, dataset_id, table_id, partition_ids)
tasks = []
for partition_id in partition_ids:
task = Tasks.create(method='GET',
url='/tasks/backups/table/{}/{}/{}/{}'.format(
project_id, dataset_id, table_id,
partition_id))
tasks.append(task)
Tasks.schedule('backup-worker', tasks)
def create_table_backup_task(project_id, dataset_id, table_id):
logging.debug(u'Create Table Backup task for %s:%s.%s', project_id,
dataset_id, table_id)
table_id = urllib2.quote(table_id.encode('UTF-8'))
return Tasks.create(method='GET',
url=u'/tasks/backups/table/{0}/{1}/{2}'.format(
project_id, dataset_id, table_id))
def test_should_create_task_with_additional_correlation_id_header(self, _):
# when
task = Tasks.create(url='/example/task1',
headers={'key': 'value'})
# then
header_value = task.headers[request_correlation_id.HEADER_NAME]
self.assertEqual(header_value, 'correlation-id')
self.assertEqual(task.headers['key'], 'value')
def schedule_backup(self, project_id, page_token=None):
dataset_ids_to_backup, next_page_token = self.big_query.list_dataset_ids(
project_id=project_id,
page_token=page_token)
self._schedule_dataset_backup_scheduler_tasks(project_id,
dataset_ids_to_backup)
if next_page_token:
logging.info(
u'Scheduling Project Backup Scheduler task for %s, page_token: %s',
project_id, next_page_token)
Tasks.schedule('backup-scheduler',
TaskCreator.create_project_backup_scheduler_task(
project_id,
next_page_token)
)
def _schedule_project_backup_scheduler_tasks(self, project_ids):
logging.info(
u'Scheduling Project Backup Scheduler tasks for %s projects: %s',
len(project_ids), project_ids)
tasks = []
for project_id in project_ids:
if project_id not in self.projects_to_skip:
tasks.append(
TaskCreator.create_project_backup_scheduler_task(
project_id=project_id))
else:
logging.info(u'Project %s is skipped.', project_id)
Tasks.schedule('backup-scheduler', tasks)
def _schedule_tasks_for_partition_backup(project_id, dataset_id, table_id,
partiton_ids):
logging.info(
u'Scheduling Partition Table Backup tasks for %s %s:%s.%s table partitions: %s.',
len(partiton_ids), project_id, dataset_id, table_id, partiton_ids)
tasks = []
for partition_id in partiton_ids:
tasks.append(
TaskCreator.create_partition_table_backup_task(
project_id=project_id,
dataset_id=dataset_id,
table_id=table_id,
partition_id=partition_id)
)
Tasks.schedule('backup-worker', tasks)
def schedule_backup(self, page_token=None):
if self.custom_projects_list:
self._schedule_project_backup_scheduler_for_custom_project_list()
return
projects_ids_to_backup, next_page_token = self.big_query.list_project_ids(
page_token=page_token)
self._schedule_project_backup_scheduler_tasks(projects_ids_to_backup)
if next_page_token:
logging.info(
u'Scheduling Organisation Backup Scheduler task for page_token: %s',
next_page_token)
Tasks.schedule(
'backup-scheduler',
TaskCreator.create_organisation_backup_scheduler_task(
page_token=next_page_token))
def recalculate_sli(self):
logging.info("Recalculating quality SLI has been started.")
all_tables, snapshot_time = self.querier.query()
self.streamer.stream(
[self.__create_snapshot_marker_row(snapshot_time)])
logging.info("Snapshot marker sent %s", snapshot_time)
tasks = []
for table in all_tables:
tasks.append(
Task(method='POST',
url='/sli/quality/violation',
payload=json.dumps({'table': table}),
headers={'Content-Type': 'application/json'}))
if tasks:
Tasks.schedule('sli-table-quality-violations', tasks)
def test_schedule_can_work_with_array_of_tasks(self):
# given
task1 = Task(
url='/example/task1',
)
task2 = Task(
url='/example/task2',
)
# when
Tasks.schedule("default", [task1, task2])
# then
executed_tasks = self.taskqueue_stub.get_filtered_tasks(
queue_names="default"
)
self.assertEqual(len(executed_tasks), 2, "Should create two tasks in queue")
self.assertEqual(executed_tasks[0].url, '/example/task1')
self.assertEqual(executed_tasks[1].url, '/example/task2')
def create_post_copy_action(cls, copy_job_type_id,
post_copy_action_request, job_json):
queue_name = copy_job_type_id + '-post-copy-action'
logging.info(
"Creating task on queue '%s' for post copy action ('%s'): %s ",
queue_name, copy_job_type_id, post_copy_action_request)
task = Task(method='POST',
url=post_copy_action_request.url,
payload=json.dumps(
{
"data": post_copy_action_request.data,
"jobJson": job_json
},
cls=RequestEncoder))
Tasks.schedule(queue_name, task)
logging.info('Task %s enqueued, ETA %s', task.name, task.eta)
def create_copy_job_result_check(cls, result_check_request):
assert result_check_request.retry_count >= 0
queue_name = result_check_request.copy_job_type_id + '-result-check'
logging.info(
"Schedule copy_job_result_check task for %s in queue '%s'",
result_check_request, queue_name)
task = Task(
method='POST',
url='/tasks/copy_job_async/result_check',
countdown=configuration.copy_job_result_check_countdown_in_sec,
params={
"resultCheckRequest":
json.dumps(result_check_request, cls=RequestEncoder)
})
Tasks.schedule(queue_name, task)
logging.info('Task %s enqueued, ETA %s', task.name, task.eta)
def create_copy_job(cls, copy_job_request):
task_name = CopyJobTaskName(copy_job_request).create()
queue_name = copy_job_request.copy_job_type_id + '-copy-job'
logging.info("Schedule copy_job_task for '%s' in queue '%s'",
copy_job_request, queue_name)
# try:
task = Task(
method='POST',
url='/tasks/copy_job_async/copy_job',
name=task_name,
params={
"copyJobRequest": json.dumps(copy_job_request,
cls=RequestEncoder)
},
)
Tasks.schedule(queue_name, task)
logging.info('Task %s enqueued, ETA %s', task.name, task.eta)
def create_project_backup_scheduler_task(project_id, page_token=None):
logging.debug(
u'Create Project Backup Scheduler task for %s, page_token: %s',
project_id, page_token)
params = {'projectId': project_id}
if page_token:
params['pageToken'] = page_token
return Tasks.create(url='/tasks/schedulebackup/project', params=params)
def create_partitioned_table_backup_scheduler_task(project_id, dataset_id,
table_id):
logging.debug(
u'Create Partitioned Table Backup Scheduler task for %s:%s.%s',
project_id, dataset_id, table_id)
return Tasks.create(url='/tasks/schedulebackup/partitionedtable',
params={
'projectId': project_id,
'datasetId': dataset_id,
'tableId': table_id
})
def create_organisation_backup_scheduler_task(page_token=None):
logging.debug(
u'Create Organisation Backup Scheduler task for page_token: %s',
page_token)
params = {}
if page_token:
params['pageToken'] = page_token
return Tasks.create(method='GET',
url='/tasks/schedulebackup/organization',
params=params)
def test_should_create_task_without_correlation_id_header(self, _):
# when
task = Tasks.create(url='/example/task1',
headers={'key': 'value'})
# then
self.assertEqual(task.headers['key'], 'value')
def test_should_create_task_with_correlation_id_header(self, _):
# when
task = Tasks.create(url='/example/task1')
# then
header_value = task.headers[request_correlation_id.HEADER_NAME]
self.assertEqual(header_value, 'correlation-id')
def get(self):
logging.info("Recalculating SLIs has been started.")
Tasks.schedule("sli-worker", self.create_slo_recalculation_tasks())