def __init__(self, stages, adj_mat, name): # assert len(stages) == adj_mat.shape[0] == adj_mat.shape[1] self.name = name self.stages = stages self.adj_mat = adj_mat self.num_stages = len(stages) self.num_finished_stages = 0 self.executors = utils.OrderedSet() # assert self.is_dag(self.num_stages, self.adj_mat) self.frontier_stages = utils.OrderedSet() for stage in self.stages: if stage.is_runnable(): self.frontier_stages.add(stage) for stage in self.stages: stage.job = self self.arrived = False self.finished = False self.start_time = None # job.start_time is the arrival time of this job, different from task.start_time self.finish_time = np.inf self.executor2interval = self.get_executor_interval_map()
def generate_tpch_jobs(np_random, timeline, wall_time): """ Randomly generate jobs with different size and shape. In this func, we generate all jobs with poisson distribution (the gap between the start time of fore-and-aft jobs follows the exponential distribution). """ arrived_jobs = utils.OrderedSet() # store already arrived jobs tm = 0 # slot index of millisecond for _ in range(args.num_init_jobs): query_size = args.tpch_size[np_random.randint(len(args.tpch_size))] query_idx = np_random.randint(args.tpch_num) + 1 # new a job instance job = generate_one_tpch_job(args.job_folder, query_size, query_idx, wall_time, np_random) job.start_time = tm job.arrived = True arrived_jobs.add(job) # generate future jobs without adding to arrived_jobs but pushing into timeline for _ in range(args.num_stream_jobs): # job arrival interval follows a exponential distribution tm += int(np_random.exponential(args.stream_interval)) # query size and idx are sampled from uniform distribution query_size = args.tpch_size[np_random.randint(len(args.tpch_size))] query_idx = np_random.randint(args.tpch_num) + 1 job = generate_one_tpch_job(args.job_folder, query_size, query_idx, wall_time, np_random) job.start_time = tm timeline.push(tm, job) return arrived_jobs
def __init__(self): """ msg_mat records the parent-children relation in each message passing step. msg_masks is the set of stages (nodes) doing message passing at each step. """ self.jobs = utils.OrderedSet() self.msg_mats = [] self.msg_masks = [] self.job_summ_backward_map = None self.running_job_mat = None
def reset(self, max_time=np.inf): self.max_time = max_time self.wall_time.reset() self.timeline.reset() self.exec_commit.reset() self.moving_executors.reset() self.reward_calculator.reset() self.finished_jobs = utils.OrderedSet() self.stage_selected.clear() for executor in self.executors: executor.reset() self.free_executors.reset(self.executors) # regenerate jobs (note that self.jobs are only currently arrived jobs) self.jobs = generate_jobs(self.np_random, self.timeline, self.wall_time) self.action_map = get_act2stage(self.jobs) for job in self.jobs: self.add_job(job) self.src_job = None # all executors are schedulable self.num_src_exec = len(self.executors) self.exec_to_schedule = utils.OrderedSet(self.executors)
def get_msg_path(self, jobs): """ Check whether the set of jobs changes. If changed, compute the message passing path. """ if len(self.jobs) != len(jobs): jobs_changed = True else: jobs_changed = not (all( job_i is job_j for (job_i, job_j) in zip(self.jobs, jobs))) if jobs_changed: self.msg_mats, self.msg_masks = self.get_msg(jobs) self.job_summ_backward_map = self.get_job_summ_backward_map(jobs) self.running_job_mat = self.get_running_job_mat(jobs) self.jobs = utils.OrderedSet(jobs) return self.msg_mats, self.msg_masks, self.job_summ_backward_map, self.running_job_mat, jobs_changed
def get_frontier_stages(self): """ Get all the frontier stages from all currently not finished jobs. Distinguish this from each job's job.frontier_stages. This is mainly (only) used for observation. In this class, 'frontier' can be interpreted as itself is unsaturated but all its parent stages are saturated. """ frontier_stages = utils.OrderedSet() for job in self.jobs: for stage in job.stages: if stage not in self.stage_selected and not self.saturated( stage): all_parents_saturated = True for parent in stage.parent_stages: if not self.saturated(parent): all_parents_saturated = False break if all_parents_saturated: frontier_stages.add(stage) return frontier_stages
def redispatch_executor(self, executor, frontier_changed): """ Dispatch a finished task's 'executor' to the other stages. In this function, scheduling event may be invoked directly without consulting self.exec_commit. """ if executor.stage is not None and not executor.stage.no_more_task: # the stage which this executor previously worked on is not finished, just directly schedule this executor to the next task # This is wired because every scheduling event should be decided by the agent. However, considering the # principle of locality, do it like this can not only improve the efficiency, but also avoid invoking the # agent too many times task = executor.stage.schedule(executor) self.timeline.push(task.finish_time, task) return # if we run here, it means we need to move executor from executor.stage, but move to where is related # to frontier_changed, thus we start a classified discussion if frontier_changed: src_job = executor.job # self.exec_commit[executor.stage] is equal to self.exec_commit.commit[executor.stage] because of the __getitem__ func if len(self.exec_commit[executor.stage]) > 0: # this condition means that this executor has been decided to be dispatched to some job (or stage), # thus we just directly fulfill the exactly commitment self.exec_to_schedule = {executor} self.schedule() else: # this executor is not arranged, temporarily store to the previously served job's free pool self.free_executors.add(src_job, executor) self.exec_to_schedule = utils.OrderedSet( self.free_executors[src_job]) self.src_job = src_job self.num_src_exec = len(self.free_executors[src_job]) else: # only need to schedule this executor self.exec_to_schedule = {executor} if len(self.exec_commit[executor.stage]) > 0: self.schedule() else: # note that self.exec_to_schedule is immediate, self.num_source_exec is for commit # so len(self.exec_to_schedule) != self.num_source_exec can happen! self.src_job = executor.job self.num_src_exec = len(executor.stage.executors)
def __init__(self, uid, app, version, useragent, lang="en"): self._appversion = version self._useragent = useragent self.known_messages = utils.OrderedSet() self.shown_messages = set() self.update_data = {} self.appname = app self.version_url = "" if constants.APP_UPDATE_MODE == 2: self.default_text = _("New version of %s is ready to install.\n" "Do you want to upgrade?") % app self.default_title = _("Update available") self.update_commands = [] self.download_url = None # Update mode 2 uses no download url self.version_url = constants.URL_UPDATE_INFO_URL self.version = version self.lang = lang self._check = self._newcheck # confusingly enough self._download = self._newdownload self._apply = self._newapply self.platform = platform.platform() self.must_download = (self.version_url and my_env.is_windows and my_env.is_frozen) self.must_check = constants.APP_UPDATE_MODE > 1 or self.must_download # Prepare download dir self.download_path = my_env.tempdirpath("update") self.enabled = self.must_check if self.must_download: # Download updates only applies to WinNT frozen exe distributions self.download_chunk_size = my_env.get_blocksize(self.download_path) self.remove_old_download() # Old download cleanup if os.path.isdir(self.download_path): shutil.rmtree(self.download_path, True)
def __init__(self): self.np_random = np.random.RandomState() self.wall_time = WallTime() self.timeline = Timeline() self.executors = utils.OrderedSet() for exec_idx in range(args.exec_cap): self.executors.add(Executor(exec_idx)) self.free_executors = FreeExecutors(self.executors) self.moving_executors = MovingExecutors() self.exec_commit = ExecutorCommit() # stages wait to be scheduled, these stages are the return of scheduling algorithms self.stage_selected = set() self.reward_calculator = RewardCalculator() self.exec_to_schedule = None # executors to be scheduled self.src_job = None self.num_src_exec = -1 # number of execs to be scheduled self.jobs = None self.action_map = None # a ReversibleMap from act to stage self.finished_jobs = None # we track this for updating the action-to-stage map self.max_time = None
def reset(self): self.jobs = utils.OrderedSet() self.msg_mats = [] self.msg_masks = [] self.job_summ_backward_map = None self.running_job_mat = None
def add_job(self, job): self.free_executors[job] = utils.OrderedSet()
def __init__(self, executors): self.free_executors = {None: utils.OrderedSet()} for e in executors: self.free_executors[None].add(e)
def derived_rels_candidates_from_trace(trns: Trace, more_traces: List[Trace], max_conj_size: int, max_free_vars: int) -> List[Tuple[List[syntax.SortedVar],Expr]]: first_relax_idx = first_relax_step_idx(trns) pre_relax_state = trns.as_state(first_relax_idx) post_relax_state = trns.as_state(first_relax_idx + 1) assert pre_relax_state.univs == post_relax_state.univs # relaxed elements relaxed_elements = [] for sort, univ in pre_relax_state.univs.items(): active_rel_name = 'active_' + sort.name # TODO: de-duplicate pre_active_interp = dict_val_from_rel_name(active_rel_name, pre_relax_state.rel_interp) post_active_interp = dict_val_from_rel_name(active_rel_name, post_relax_state.rel_interp) pre_active_elements = [tup[0] for (tup, b) in pre_active_interp if b] post_active_elements = [tup[0] for (tup, b) in post_active_interp if b] assert set(post_active_elements).issubset(set(pre_active_elements)) for relaxed_elem in utils.OrderedSet(pre_active_elements) - set(post_active_elements): relaxed_elements.append((sort, relaxed_elem)) # pre-relaxation step facts concerning at least one relaxed element (other to be found by UPDR) relevant_facts: List[Union[RelationFact,FunctionFact,InequalityFact]] = [] for rel, rintp in pre_relax_state.rel_interp.items(): for rfact in rintp: (elms, polarity) = rfact relation_fact = RelationFact(rel, elms, polarity) if set(relation_fact.involved_elms()) & set(ename for (_, ename) in relaxed_elements): relevant_facts.append(relation_fact) for func, fintp in pre_relax_state.func_interp.items(): for ffact in fintp: (els_params, els_res) = ffact function_fact = FunctionFact(func, els_params, els_res) if set(function_fact.involved_elms()) & set(ename for (_, ename) in relaxed_elements): relevant_facts.append(function_fact) for sort, elm in relaxed_elements: # other inequalities presumably handled by UPDR for other_elm in pre_relax_state.univs[sort]: if other_elm == elm: continue relevant_facts.append(InequalityFact(elm, other_elm)) # facts blocking this specific relaxation step diff_conjunctions = [] candidates_cache: Set[str] = set() for fact_lst in itertools.combinations(relevant_facts, max_conj_size): elements = utils.OrderedSet(itertools.chain.from_iterable(fact.involved_elms() for fact in fact_lst)) relaxed_elements_relevant = [elm for (_, elm) in relaxed_elements if elm in elements] vars_from_elm = dict((elm, syntax.SortedVar(None, syntax.the_program.scope.fresh("v%d" % i), None)) for (i, elm) in enumerate(elements)) parameter_elements = elements - set(relaxed_elements_relevant) if len(parameter_elements) > max_free_vars: continue conjuncts = [fact.as_expr(lambda elm: vars_from_elm[elm].name) for fact in fact_lst] # for elm, var in vars_from_elm.items(): # TODO: make the two loops similar for elm in relaxed_elements_relevant: var = vars_from_elm[elm] sort = pre_relax_state.element_sort(elm) active_element_conj = syntax.Apply('active_%s' % sort.name, [syntax.Id(None, var.name)]) conjuncts.append(active_element_conj) derived_relation_formula = syntax.Exists([vars_from_elm[elm] for (_, elm) in relaxed_elements if elm in vars_from_elm], syntax.And(*conjuncts)) if str(derived_relation_formula) in candidates_cache: continue candidates_cache.add(str(derived_relation_formula)) if closing_qa_cycle(syntax.the_program, [pre_relax_state.element_sort(elm) for elm in parameter_elements], [pre_relax_state.element_sort(elm) for elm in relaxed_elements_relevant]): # adding the derived relation would close a quantifier alternation cycle, discard the candidate continue # if trns.eval_double_vocab(diffing_formula, first_relax_idx): if is_rel_blocking_relax(trns, first_relax_idx, ([(vars_from_elm[elm], pre_relax_state.element_sort(elm).name) for elm in parameter_elements], derived_relation_formula)): # if all(trs.eval_double_vocab(diffing_formula, first_relax_step_idx(trs)) for trs in more_traces): diff_conjunctions.append(([vars_from_elm[elm] for elm in parameter_elements], derived_relation_formula)) return diff_conjunctions
def __init__(self, idx, tasks, task_duration, wall_time, np_random): """ Initialize a stage. :param idx: stage index :param tasks: tasks included in this stage :param task_duration: a dict records various task execution time of this stage with different executors allocated to it under different scenarios. The dict looks like { 'fresh_durations': { e_1: [fresh duration of each task (with warmup delay included) of this stage when e_1 executors are allocated to this stage], e_2: [...], ... e_N: [...] }, 'first_wave': { e_1: [first wave duration of each task of this stage when e_1 executors are allocated to this stage], e_2: [...], ... e_N: [...] }, 'rest_wave': { e_1: [rest wave duration of each task of this stage when e_1 executors are allocated to this stage], e_2: [...], ... e_N: [...] } } Here e_1, ..., e_N are 2, 5, 10, 80, 100, respectively. The authors only collect the task execution time under the number of e_i executors. In ./data/tpch-queries/task_durations/*.pdf, the data points records the durations collected. - the green data points mean the 'fresh_durations'; - the red data points mean the 'first_wave'; - the blue data points mean the 'rest_wave'. Let us take tpch-2g-1-0.pdf as an example. This is the first stage of the job and only has 12 tasks. It is impossible to have the 'first_wave' data points because it is the entry node. Let us also take tpch-2g-1-4.pdf as an example. This is the last stage of the job and has 5 tasks. When only small num of executors, it is almost impossible to have the 'fresh_wave' data points. :param wall_time: records current time :param np_random: isolated random generator """ self.idx = idx self.tasks = tasks for task in self.tasks: task.stage = self self.task_duration = task_duration self.wall_time = wall_time self.np_random = np_random self.num_tasks = len(tasks) self.num_finished_tasks = 0 self.next_task_idx = 0 self.no_more_task = False self.all_tasks_done = False self.finish_time = np.inf self.executors = utils.OrderedSet() # these vars are initialized when the corresponding job is initialized # self.parent_stages, self.child_stages, self.descendant_stages = [], [], [] self.parent_stages, self.child_stages = [], [] self.job = None
def step(self, next_stage, limit): """ One (scheduling event) step forward: with given actions as input, submit commitment (invoke scheduling event if necessary), and return the reward and the new state. :param next_stage: the next to-be-scheduled stage, generated by scheduling algorithm :param limit: the exec limit for the job of the next to-be-scheduled stage, generated by scheduling algorithm """ assert next_stage not in self.stage_selected self.stage_selected.add(next_stage) # note that currently self.exec_to_schedule are all coming from the same src, # go find the src to make sure whether (executors') moving delay is necessary or not executor = next(iter(self.exec_to_schedule)) src = executor.job if executor.stage is None else executor.stage # calculate how many execs we need to reissue to next_stage # if next_stage is None, we just collect all execs in self.exec_to_schedule and save them to self.free_executors if next_stage is not None: use_exec = min( (next_stage.num_tasks - next_stage.next_task_idx) - # next_stage really needs (self.exec_commit.stage_commit[next_stage] + self.moving_executors.count(next_stage) ), # next_stage already got limit # the upper limit informed by the scheduling algorithm ) else: use_exec = limit assert use_exec > 0 # here is the interesting thing... We do not schedule next_stage with use_exec executors at present, # but submit the transaction into commitment! So... when we really invoke the scheduling event? # When a new scheduling round starts! In that case, we fulfill all the commitments in self.exec_commit self.exec_commit.add(src, next_stage, use_exec) self.num_src_exec -= use_exec assert self.num_src_exec >= 0 # if this not fulfilled, sth. wrong with the limit return from scheduling algorithm if self.num_src_exec == 0: # invoke the scheduling event! self.stage_selected.clear( ) # the left selected stages cannot be scheduled any more, start all over again self.schedule() # now we update to the new state... # Pay attention to the second condition of the while loop: only if self.num_src_exec is 0, which means # the scheduling event is invoked, we update the state. If scheduling event is not invoked, we do not update state # The second condition also means every time self.num_src_exec changes, new commitment will be submitted, # which means a next scheduling event is preparing, thus directly exit the while loop while len(self.timeline) > 0 and self.num_src_exec == 0: new_time, item = self.timeline.pop() self.wall_time.update(new_time) # according to the type of item, turn into different state if isinstance(item, Task): # ==== task finish event ====: update stage's finished task num, job' finished stages num, and theirs finish time finished_task = item stage = finished_task.stage stage.num_finished_tasks += 1 frontier_changed = False if stage.num_finished_tasks == stage.num_tasks: assert not stage.all_tasks_done stage.all_tasks_done = True stage.job.num_finished_stages += 1 stage.finish_time = self.wall_time.cur_time frontier_changed = stage.job.update_frontier_stages(stage) # free up this executor, which may invoke scheduling event directly self.redispatch_executor(finished_task.executor, frontier_changed) if stage.job.num_finished_stages == stage.job.num_stages: # update job completion status assert not stage.job.finished stage.job.finished = True stage.job.finish_time = self.wall_time.cur_time self.remove_job(stage.job) elif isinstance(item, Job): # ==== new job arrives event ====: update act map, assign all free execs to the newly arrived job job = item assert not job.arrived job.arrived = True self.jobs.add(job) self.add_job(job) self.action_map = get_act2stage(self.jobs) if len(self.free_executors[None]) > 0: self.exec_to_schedule = utils.OrderedSet( self.free_executors[None]) self.src_job = None self.num_src_exec = len(self.free_executors[None]) elif isinstance(item, Executor): # ==== executor arrival event ====: bind the exec to the destination job or temporarily store it to the src's free_executors pool executor = item # get the destination (stage) of this executor stage = self.moving_executors.pop(executor) if stage is not None: # the job (of this stage) is not yet finished when this executor arrives, bind this 'executor' to its arrived job executor.job = stage.job stage.job.executors.add(executor) if stage is not None and not stage.no_more_task: # this stage is schedulable, directly schedule it if stage in stage.job.frontier_stages: # this stage is immediately runnable task = stage.schedule(executor) self.timeline.push(task.finish_time, task) else: # add this executor to the free executor pool of this job self.free_executors.add(executor.job, executor) else: # this stage is saturated or this job is finished, but the executor still arrives to it # in this case, use backup schedule policy self.backup_schedule(executor) else: print('Illegal event type!') exit(1) # compute reward reward = self.reward_calculator.get_reward(self.jobs, self.wall_time.cur_time) # no more decision to make: jobs all done or time is up done = self.num_src_exec == 0 and (len( self.timeline) == 0 or self.wall_time.cur_time >= self.max_time) if done: assert self.wall_time.cur_time >= self.max_time or len( self.jobs) == 0 # return new state, reward and a flag to indicate whether all jobs are finished # these return vars are the input of the scheduling algorithm return self.observe(), reward, done