def get_tasks(self, release_id):
"""Retrieve all task information.
Args:
release_id: str, unique release identifier
Returns:
Dictionary of Task objects.
"""
if release_id in self._releases:
tasks = json.loads(self._releases['release_id'].tasks)
task_objects = []
for k, v in tasks:
task_objects.append(Task(v))
return task_objects
dag_id, execution_date = release_id_parser(release_id)
execution_date = time.mktime(execution_date.timetuple())
# build SQl query
task_query = to_sql_tasks(execution_date)
# get data from SQL
task_objects = {}
for config in self._configs:
raw_task_data = config.db.query(task_query) # it's a tuple, so no .append()
if len(raw_task_data):
# package the data into release objects with all neccessary info
task_object = read_tasks(raw_task_data)
return task_objects
def __init__(self, releases_file, tasks_file):
with open(releases_file) as f:
json_releases = f.read()
releases_dict = json.loads(json_releases)
self._releases = {}
for key in releases_dict:
releases_dict[key]['id'] = releases_dict[key]['name']
release = Release(releases_dict[key])
self._releases[key] = release
with open(tasks_file) as f:
json_tasks = f.read()
tasks_dict = json.loads(json_tasks)
self._tasks = {}
for key in tasks_dict:
task = Task(tasks_dict[key])
self._tasks[key] = task
branches = set()
for release in self._releases:
branches.add(self._releases[release].branch)
self._branches = list(branches)
release_types = set()
for release in self._releases:
release_types.add(self._releases[release].release_type)
self._release_types = list(release_types)
def from_string(self, string):
sequence = json.loads(string)
self.id = self._get_id()
self.user_id = sequence['user_id']
self.service_type = sequence['service_type']
self.type = sequence['type']
self.status = sequence['status']
self.command = sequence['command']
self.args = sequence['args']
for task_index in sequence['tasks']:
task = Task()
task.seq_id = self.id
task.queue = task_index['queue']
task.command = task_index['command']
task.args = task_index['args']
self.tasks.append(task)
for task in self.tasks:
task.id = self._get_id()
for i in range(len(self.tasks) - 1):
self.tasks[i].next_task_id = self.tasks[i + 1].id
return self
def from_string(self, string):
sequence = json.loads(string)
self.id = self._get_id()
self.user_id = sequence['user_id']
self.service_type = sequence['service_type']
self.type = sequence['type']
self.status = sequence['status']
self.command = sequence['command']
self.args = sequence['args']
for task_index in sequence['tasks']:
task = Task()
task.seq_id = self.id
task.queue = task_index['queue']
task.command = task_index['command']
task.args = task_index['args']
self.tasks.append(task)
for task in self.tasks:
task.id = self._get_id()
for i in range(len(self.tasks) - 1):
self.tasks[i].next_task_id = self.tasks[i + 1].id
return self
def _load_dump(self, filename):
with open(filename) as f:
releases = f.read()
releases = json.loads(releases)
for k, v in releases.iteritems():
task_ids = []
for task in v['tasks']:
task_id = task['task_name'] + '@' + k
self._tasks[task_id] = Task(task)
task_ids.append(task_id)
v['tasks'] = task_ids
self._releases[k] = Release(v)
logging.info('Releases read from %s' % filename)
def task_add(subject_id, section_id):
if not (available("subjects", subject_id)
and available("section", section_id)):
return redirect("index")
form = TaskForm()
if form.validate_on_submit():
db_sess = db_session.create_session()
task = Task(name=form.name.data,
section_id=section_id,
condition=form.condition.data,
formula=form.formula.data)
db_sess.add(task)
db_sess.commit()
return redirect(f"/task/{subject_id}/{section_id}")
return render_template('form_add.html',
title='Добавление задачи',
form=form)
def test_getters(self):
task = Task({
'status': 3,
'log_url': 'https://youtu.be/dQw4w9WgXcQ',
'started': 47752289,
'dependent_on': [],
'last_modified': 637147117,
'task_name': 'task-2',
'error': 'error number 990'
})
self.assertEqual(task.task_name, 'task-2')
self.assertEqual(task.status, State.FAILED)
self.assertEqual(task.log_url, 'https://youtu.be/dQw4w9WgXcQ')
self.assertEqual(task.started, datetime.fromtimestamp(47752289))
self.assertEqual(task.dependent_on, [])
self.assertEqual(task.last_modified, datetime.fromtimestamp(637147117))
self.assertEqual(task.error, 'error number 990')
def read_tasks(task_data):
"""Reads the tasks, transforms them into objects.
Args:
task_data: a tuple of tuples with task info
Returns:
task_objects
"""
task_named_tuple = namedtuple('Row', [
'task_id', 'dag_id', 'execution_date', 'start_date', 'end_date',
'state'
])
task_objects = []
for item in task_data:
task = Task()
logging.debug(item)
item = task_named_tuple(*item)
task.task_name = item.task_id
task.add_dependency = None # TODO(dommarques): figure out how to get this - dag info?
if item.start_date is None:
execution_date = item.execution_date
task.started = int(time.mktime(execution_date.timetuple()))
else:
start_date = item.start_date
task.started = int(time.mktime(start_date.timetuple()))
task.log_url = construct_log_link(
item.dag_id, item.execution_date, item.task_id
) # TODO(dommarques): figure out how to get the log in here
if item.end_date is None:
if item.start_date is None:
execution_date = item.execution_date
task.last_modified = int(
time.mktime(execution_date.timetuple()))
else:
start_date = item.start_date
task.last_modified = int(time.mktime(start_date.timetuple()))
else:
end_date = item.end_date
task.last_modified = int(time.mktime(end_date.timetuple()))
if item.state is None:
task.status = STATE_FROM_STRING.get('not_started')
task.error = 'not started'
elif item.state == 'running':
task.status = STATE_FROM_STRING.get('running')
task.error = STRING_FROM_STATE.get(task.status)
elif item.start_date is None:
task.status = STATE_FROM_STRING.get('queued')
task.error = item.state
else:
task.error = item.state
if str(item.state) in STATE_FROM_STRING:
task.status = STATE_FROM_STRING.get(str(item.state))
else:
task.status = STATE_FROM_STRING.get('none')
task_objects.append(task)
return task_objects
def test_setters(self):
task = Task()
task.task_name = 'new task'
self.assertEqual(task.task_name, 'new task')
with self.assertRaises(ValueError):
task.task_name = 0
task.status = 0
self.assertEqual(task.status, State.UNUSED_STATUS)
with self.assertRaises(ValueError):
task.status = ['task1', 'task2']
task.log_url = 'url'
self.assertEqual(task.log_url, 'url')
with self.assertRaises(ValueError):
task.log_url = 0
task.started = 0
self.assertEqual(task.started, datetime.fromtimestamp(0))
with self.assertRaises(ValueError):
task.started = 'started'
task.add_dependency('task1')
self.assertEqual(task.dependent_on, ['task1'])
with self.assertRaises(ValueError):
task.add_dependency(0)
task.last_modified = 0
self.assertEqual(task.last_modified, datetime.fromtimestamp(0))
with self.assertRaises(ValueError):
task.last_modified = 'last_modified'
task.error = 'error'
self.assertEqual(task.error, 'error')
with self.assertRaises(ValueError):
task.error = 0