def __init__(self, stages, adj_mat, name):
# assert len(stages) == adj_mat.shape[0] == adj_mat.shape[1]
self.name = name
self.stages = stages
self.adj_mat = adj_mat
self.num_stages = len(stages)
self.num_finished_stages = 0
self.executors = utils.OrderedSet()
# assert self.is_dag(self.num_stages, self.adj_mat)
self.frontier_stages = utils.OrderedSet()
for stage in self.stages:
if stage.is_runnable():
self.frontier_stages.add(stage)
for stage in self.stages:
stage.job = self
self.arrived = False
self.finished = False
self.start_time = None # job.start_time is the arrival time of this job, different from task.start_time
self.finish_time = np.inf
self.executor2interval = self.get_executor_interval_map()
def generate_tpch_jobs(np_random, timeline, wall_time):
"""
Randomly generate jobs with different size and shape.
In this func, we generate all jobs with poisson distribution (the gap between the start time of fore-and-aft jobs follows
the exponential distribution).
"""
arrived_jobs = utils.OrderedSet() # store already arrived jobs
tm = 0 # slot index of millisecond
for _ in range(args.num_init_jobs):
query_size = args.tpch_size[np_random.randint(len(args.tpch_size))]
query_idx = np_random.randint(args.tpch_num) + 1
# new a job instance
job = generate_one_tpch_job(args.job_folder, query_size, query_idx, wall_time, np_random)
job.start_time = tm
job.arrived = True
arrived_jobs.add(job)
# generate future jobs without adding to arrived_jobs but pushing into timeline
for _ in range(args.num_stream_jobs):
# job arrival interval follows a exponential distribution
tm += int(np_random.exponential(args.stream_interval))
# query size and idx are sampled from uniform distribution
query_size = args.tpch_size[np_random.randint(len(args.tpch_size))]
query_idx = np_random.randint(args.tpch_num) + 1
job = generate_one_tpch_job(args.job_folder, query_size, query_idx, wall_time, np_random)
job.start_time = tm
timeline.push(tm, job)
return arrived_jobs
def __init__(self):
"""
msg_mat records the parent-children relation in each message passing step.
msg_masks is the set of stages (nodes) doing message passing at each step.
"""
self.jobs = utils.OrderedSet()
self.msg_mats = []
self.msg_masks = []
self.job_summ_backward_map = None
self.running_job_mat = None
def reset(self, max_time=np.inf):
self.max_time = max_time
self.wall_time.reset()
self.timeline.reset()
self.exec_commit.reset()
self.moving_executors.reset()
self.reward_calculator.reset()
self.finished_jobs = utils.OrderedSet()
self.stage_selected.clear()
for executor in self.executors:
executor.reset()
self.free_executors.reset(self.executors)
# regenerate jobs (note that self.jobs are only currently arrived jobs)
self.jobs = generate_jobs(self.np_random, self.timeline,
self.wall_time)
self.action_map = get_act2stage(self.jobs)
for job in self.jobs:
self.add_job(job)
self.src_job = None
# all executors are schedulable
self.num_src_exec = len(self.executors)
self.exec_to_schedule = utils.OrderedSet(self.executors)
def get_msg_path(self, jobs):
"""
Check whether the set of jobs changes. If changed, compute the message passing path.
"""
if len(self.jobs) != len(jobs):
jobs_changed = True
else:
jobs_changed = not (all(
job_i is job_j for (job_i, job_j) in zip(self.jobs, jobs)))
if jobs_changed:
self.msg_mats, self.msg_masks = self.get_msg(jobs)
self.job_summ_backward_map = self.get_job_summ_backward_map(jobs)
self.running_job_mat = self.get_running_job_mat(jobs)
self.jobs = utils.OrderedSet(jobs)
return self.msg_mats, self.msg_masks, self.job_summ_backward_map, self.running_job_mat, jobs_changed
def get_frontier_stages(self):
"""
Get all the frontier stages from all currently not finished jobs.
Distinguish this from each job's job.frontier_stages. This is mainly (only) used for observation.
In this class, 'frontier' can be interpreted as itself is unsaturated but all its parent stages are saturated.
"""
frontier_stages = utils.OrderedSet()
for job in self.jobs:
for stage in job.stages:
if stage not in self.stage_selected and not self.saturated(
stage):
all_parents_saturated = True
for parent in stage.parent_stages:
if not self.saturated(parent):
all_parents_saturated = False
break
if all_parents_saturated:
frontier_stages.add(stage)
return frontier_stages
def redispatch_executor(self, executor, frontier_changed):
"""
Dispatch a finished task's 'executor' to the other stages.
In this function, scheduling event may be invoked directly without consulting self.exec_commit.
"""
if executor.stage is not None and not executor.stage.no_more_task:
# the stage which this executor previously worked on is not finished, just directly schedule this executor to the next task
# This is wired because every scheduling event should be decided by the agent. However, considering the
# principle of locality, do it like this can not only improve the efficiency, but also avoid invoking the
# agent too many times
task = executor.stage.schedule(executor)
self.timeline.push(task.finish_time, task)
return
# if we run here, it means we need to move executor from executor.stage, but move to where is related
# to frontier_changed, thus we start a classified discussion
if frontier_changed:
src_job = executor.job
# self.exec_commit[executor.stage] is equal to self.exec_commit.commit[executor.stage] because of the __getitem__ func
if len(self.exec_commit[executor.stage]) > 0:
# this condition means that this executor has been decided to be dispatched to some job (or stage),
# thus we just directly fulfill the exactly commitment
self.exec_to_schedule = {executor}
self.schedule()
else:
# this executor is not arranged, temporarily store to the previously served job's free pool
self.free_executors.add(src_job, executor)
self.exec_to_schedule = utils.OrderedSet(
self.free_executors[src_job])
self.src_job = src_job
self.num_src_exec = len(self.free_executors[src_job])
else:
# only need to schedule this executor
self.exec_to_schedule = {executor}
if len(self.exec_commit[executor.stage]) > 0:
self.schedule()
else:
# note that self.exec_to_schedule is immediate, self.num_source_exec is for commit
# so len(self.exec_to_schedule) != self.num_source_exec can happen!
self.src_job = executor.job
self.num_src_exec = len(executor.stage.executors)
def __init__(self, uid, app, version, useragent, lang="en"):
self._appversion = version
self._useragent = useragent
self.known_messages = utils.OrderedSet()
self.shown_messages = set()
self.update_data = {}
self.appname = app
self.version_url = ""
if constants.APP_UPDATE_MODE == 2:
self.default_text = _("New version of %s is ready to install.\n"
"Do you want to upgrade?") % app
self.default_title = _("Update available")
self.update_commands = []
self.download_url = None # Update mode 2 uses no download url
self.version_url = constants.URL_UPDATE_INFO_URL
self.version = version
self.lang = lang
self._check = self._newcheck # confusingly enough
self._download = self._newdownload
self._apply = self._newapply
self.platform = platform.platform()
self.must_download = (self.version_url and my_env.is_windows
and my_env.is_frozen)
self.must_check = constants.APP_UPDATE_MODE > 1 or self.must_download
# Prepare download dir
self.download_path = my_env.tempdirpath("update")
self.enabled = self.must_check
if self.must_download:
# Download updates only applies to WinNT frozen exe distributions
self.download_chunk_size = my_env.get_blocksize(self.download_path)
self.remove_old_download()
# Old download cleanup
if os.path.isdir(self.download_path):
shutil.rmtree(self.download_path, True)
def __init__(self):
self.np_random = np.random.RandomState()
self.wall_time = WallTime()
self.timeline = Timeline()
self.executors = utils.OrderedSet()
for exec_idx in range(args.exec_cap):
self.executors.add(Executor(exec_idx))
self.free_executors = FreeExecutors(self.executors)
self.moving_executors = MovingExecutors()
self.exec_commit = ExecutorCommit()
# stages wait to be scheduled, these stages are the return of scheduling algorithms
self.stage_selected = set()
self.reward_calculator = RewardCalculator()
self.exec_to_schedule = None # executors to be scheduled
self.src_job = None
self.num_src_exec = -1 # number of execs to be scheduled
self.jobs = None
self.action_map = None # a ReversibleMap from act to stage
self.finished_jobs = None # we track this for updating the action-to-stage map
self.max_time = None
def reset(self):
self.jobs = utils.OrderedSet()
self.msg_mats = []
self.msg_masks = []
self.job_summ_backward_map = None
self.running_job_mat = None
def add_job(self, job):
self.free_executors[job] = utils.OrderedSet()
def __init__(self, executors):
self.free_executors = {None: utils.OrderedSet()}
for e in executors:
self.free_executors[None].add(e)
def derived_rels_candidates_from_trace(trns: Trace, more_traces: List[Trace],
max_conj_size: int, max_free_vars: int) -> List[Tuple[List[syntax.SortedVar],Expr]]:
first_relax_idx = first_relax_step_idx(trns)
pre_relax_state = trns.as_state(first_relax_idx)
post_relax_state = trns.as_state(first_relax_idx + 1)
assert pre_relax_state.univs == post_relax_state.univs
# relaxed elements
relaxed_elements = []
for sort, univ in pre_relax_state.univs.items():
active_rel_name = 'active_' + sort.name # TODO: de-duplicate
pre_active_interp = dict_val_from_rel_name(active_rel_name, pre_relax_state.rel_interp)
post_active_interp = dict_val_from_rel_name(active_rel_name, post_relax_state.rel_interp)
pre_active_elements = [tup[0] for (tup, b) in pre_active_interp if b]
post_active_elements = [tup[0] for (tup, b) in post_active_interp if b]
assert set(post_active_elements).issubset(set(pre_active_elements))
for relaxed_elem in utils.OrderedSet(pre_active_elements) - set(post_active_elements):
relaxed_elements.append((sort, relaxed_elem))
# pre-relaxation step facts concerning at least one relaxed element (other to be found by UPDR)
relevant_facts: List[Union[RelationFact,FunctionFact,InequalityFact]] = []
for rel, rintp in pre_relax_state.rel_interp.items():
for rfact in rintp:
(elms, polarity) = rfact
relation_fact = RelationFact(rel, elms, polarity)
if set(relation_fact.involved_elms()) & set(ename for (_, ename) in relaxed_elements):
relevant_facts.append(relation_fact)
for func, fintp in pre_relax_state.func_interp.items():
for ffact in fintp:
(els_params, els_res) = ffact
function_fact = FunctionFact(func, els_params, els_res)
if set(function_fact.involved_elms()) & set(ename for (_, ename) in relaxed_elements):
relevant_facts.append(function_fact)
for sort, elm in relaxed_elements: # other inequalities presumably handled by UPDR
for other_elm in pre_relax_state.univs[sort]:
if other_elm == elm:
continue
relevant_facts.append(InequalityFact(elm, other_elm))
# facts blocking this specific relaxation step
diff_conjunctions = []
candidates_cache: Set[str] = set()
for fact_lst in itertools.combinations(relevant_facts, max_conj_size):
elements = utils.OrderedSet(itertools.chain.from_iterable(fact.involved_elms() for fact in fact_lst))
relaxed_elements_relevant = [elm for (_, elm) in relaxed_elements if elm in elements]
vars_from_elm = dict((elm, syntax.SortedVar(None, syntax.the_program.scope.fresh("v%d" % i), None))
for (i, elm) in enumerate(elements))
parameter_elements = elements - set(relaxed_elements_relevant)
if len(parameter_elements) > max_free_vars:
continue
conjuncts = [fact.as_expr(lambda elm: vars_from_elm[elm].name) for fact in fact_lst]
# for elm, var in vars_from_elm.items():
# TODO: make the two loops similar
for elm in relaxed_elements_relevant:
var = vars_from_elm[elm]
sort = pre_relax_state.element_sort(elm)
active_element_conj = syntax.Apply('active_%s' % sort.name, [syntax.Id(None, var.name)])
conjuncts.append(active_element_conj)
derived_relation_formula = syntax.Exists([vars_from_elm[elm]
for (_, elm) in relaxed_elements
if elm in vars_from_elm],
syntax.And(*conjuncts))
if str(derived_relation_formula) in candidates_cache:
continue
candidates_cache.add(str(derived_relation_formula))
if closing_qa_cycle(syntax.the_program, [pre_relax_state.element_sort(elm) for elm in parameter_elements],
[pre_relax_state.element_sort(elm) for elm in relaxed_elements_relevant]):
# adding the derived relation would close a quantifier alternation cycle, discard the candidate
continue
# if trns.eval_double_vocab(diffing_formula, first_relax_idx):
if is_rel_blocking_relax(trns, first_relax_idx,
([(vars_from_elm[elm], pre_relax_state.element_sort(elm).name) for elm in parameter_elements],
derived_relation_formula)):
# if all(trs.eval_double_vocab(diffing_formula, first_relax_step_idx(trs)) for trs in more_traces):
diff_conjunctions.append(([vars_from_elm[elm] for elm in parameter_elements],
derived_relation_formula))
return diff_conjunctions
def __init__(self, idx, tasks, task_duration, wall_time, np_random):
"""
Initialize a stage.
:param idx: stage index
:param tasks: tasks included in this stage
:param task_duration: a dict records various task execution time of this stage with different executors allocated to it
under different scenarios.
The dict looks like
{
'fresh_durations': {
e_1: [fresh duration of each task (with warmup delay included) of this stage
when e_1 executors are allocated to this stage],
e_2: [...],
...
e_N: [...]
},
'first_wave': {
e_1: [first wave duration of each task of this stage when e_1 executors are
allocated to this stage],
e_2: [...],
...
e_N: [...]
},
'rest_wave': {
e_1: [rest wave duration of each task of this stage when e_1 executors are
allocated to this stage],
e_2: [...],
...
e_N: [...]
}
}
Here e_1, ..., e_N are 2, 5, 10, 80, 100, respectively.
The authors only collect the task execution time under the number of e_i executors.
In ./data/tpch-queries/task_durations/*.pdf, the data points records the durations collected.
- the green data points mean the 'fresh_durations';
- the red data points mean the 'first_wave';
- the blue data points mean the 'rest_wave'.
Let us take tpch-2g-1-0.pdf as an example. This is the first stage of the job and only has 12 tasks.
It is impossible to have the 'first_wave' data points because it is the entry node.
Let us also take tpch-2g-1-4.pdf as an example. This is the last stage of the job and has 5 tasks.
When only small num of executors, it is almost impossible to have the 'fresh_wave' data points.
:param wall_time: records current time
:param np_random: isolated random generator
"""
self.idx = idx
self.tasks = tasks
for task in self.tasks:
task.stage = self
self.task_duration = task_duration
self.wall_time = wall_time
self.np_random = np_random
self.num_tasks = len(tasks)
self.num_finished_tasks = 0
self.next_task_idx = 0
self.no_more_task = False
self.all_tasks_done = False
self.finish_time = np.inf
self.executors = utils.OrderedSet()
# these vars are initialized when the corresponding job is initialized
# self.parent_stages, self.child_stages, self.descendant_stages = [], [], []
self.parent_stages, self.child_stages = [], []
self.job = None
def step(self, next_stage, limit):
"""
One (scheduling event) step forward:
with given actions as input, submit commitment (invoke scheduling event if necessary),
and return the reward and the new state.
:param next_stage: the next to-be-scheduled stage, generated by scheduling algorithm
:param limit: the exec limit for the job of the next to-be-scheduled stage, generated by scheduling algorithm
"""
assert next_stage not in self.stage_selected
self.stage_selected.add(next_stage)
# note that currently self.exec_to_schedule are all coming from the same src,
# go find the src to make sure whether (executors') moving delay is necessary or not
executor = next(iter(self.exec_to_schedule))
src = executor.job if executor.stage is None else executor.stage
# calculate how many execs we need to reissue to next_stage
# if next_stage is None, we just collect all execs in self.exec_to_schedule and save them to self.free_executors
if next_stage is not None:
use_exec = min(
(next_stage.num_tasks - next_stage.next_task_idx)
- # next_stage really needs
(self.exec_commit.stage_commit[next_stage] +
self.moving_executors.count(next_stage)
), # next_stage already got
limit # the upper limit informed by the scheduling algorithm
)
else:
use_exec = limit
assert use_exec > 0
# here is the interesting thing... We do not schedule next_stage with use_exec executors at present,
# but submit the transaction into commitment! So... when we really invoke the scheduling event?
# When a new scheduling round starts! In that case, we fulfill all the commitments in self.exec_commit
self.exec_commit.add(src, next_stage, use_exec)
self.num_src_exec -= use_exec
assert self.num_src_exec >= 0 # if this not fulfilled, sth. wrong with the limit return from scheduling algorithm
if self.num_src_exec == 0:
# invoke the scheduling event!
self.stage_selected.clear(
) # the left selected stages cannot be scheduled any more, start all over again
self.schedule()
# now we update to the new state...
# Pay attention to the second condition of the while loop: only if self.num_src_exec is 0, which means
# the scheduling event is invoked, we update the state. If scheduling event is not invoked, we do not update state
# The second condition also means every time self.num_src_exec changes, new commitment will be submitted,
# which means a next scheduling event is preparing, thus directly exit the while loop
while len(self.timeline) > 0 and self.num_src_exec == 0:
new_time, item = self.timeline.pop()
self.wall_time.update(new_time)
# according to the type of item, turn into different state
if isinstance(item, Task):
# ==== task finish event ====: update stage's finished task num, job' finished stages num, and theirs finish time
finished_task = item
stage = finished_task.stage
stage.num_finished_tasks += 1
frontier_changed = False
if stage.num_finished_tasks == stage.num_tasks:
assert not stage.all_tasks_done
stage.all_tasks_done = True
stage.job.num_finished_stages += 1
stage.finish_time = self.wall_time.cur_time
frontier_changed = stage.job.update_frontier_stages(stage)
# free up this executor, which may invoke scheduling event directly
self.redispatch_executor(finished_task.executor,
frontier_changed)
if stage.job.num_finished_stages == stage.job.num_stages:
# update job completion status
assert not stage.job.finished
stage.job.finished = True
stage.job.finish_time = self.wall_time.cur_time
self.remove_job(stage.job)
elif isinstance(item, Job):
# ==== new job arrives event ====: update act map, assign all free execs to the newly arrived job
job = item
assert not job.arrived
job.arrived = True
self.jobs.add(job)
self.add_job(job)
self.action_map = get_act2stage(self.jobs)
if len(self.free_executors[None]) > 0:
self.exec_to_schedule = utils.OrderedSet(
self.free_executors[None])
self.src_job = None
self.num_src_exec = len(self.free_executors[None])
elif isinstance(item, Executor):
# ==== executor arrival event ====: bind the exec to the destination job or temporarily store it to the src's free_executors pool
executor = item
# get the destination (stage) of this executor
stage = self.moving_executors.pop(executor)
if stage is not None:
# the job (of this stage) is not yet finished when this executor arrives, bind this 'executor' to its arrived job
executor.job = stage.job
stage.job.executors.add(executor)
if stage is not None and not stage.no_more_task:
# this stage is schedulable, directly schedule it
if stage in stage.job.frontier_stages:
# this stage is immediately runnable
task = stage.schedule(executor)
self.timeline.push(task.finish_time, task)
else:
# add this executor to the free executor pool of this job
self.free_executors.add(executor.job, executor)
else:
# this stage is saturated or this job is finished, but the executor still arrives to it
# in this case, use backup schedule policy
self.backup_schedule(executor)
else:
print('Illegal event type!')
exit(1)
# compute reward
reward = self.reward_calculator.get_reward(self.jobs,
self.wall_time.cur_time)
# no more decision to make: jobs all done or time is up
done = self.num_src_exec == 0 and (len(
self.timeline) == 0 or self.wall_time.cur_time >= self.max_time)
if done:
assert self.wall_time.cur_time >= self.max_time or len(
self.jobs) == 0
# return new state, reward and a flag to indicate whether all jobs are finished
# these return vars are the input of the scheduling algorithm
return self.observe(), reward, done
