def test_add_node_exception(self): try: dag = DAG() dag.add_node("A") dag.add_node("A") except Exception as e: self.assertEqual(str(e), "Duplicate node in one graph.")
def inital_DAG(self, task_id, config=None, T=None, D=None): self.DAG = DAG(task_id, self.freq, T=T, D=D) if config is not None: self.DAG.create_from_config(config) else: self.DAG.create() self.DAG.get_valid_partition() self.DAG.local_only_compute_energy_time(self.freq, self.k)
def __init__(self, table_names, dotted_fields, dotted_datatype, dag=None, con=None): self.table_names = table_names self.dotted_fields = dotted_fields self.dotted_datatype = dotted_datatype if dag is None: self.DAG = DAG() else: self.DAG = cp.copy(dag) self.con = con self._shape = None self._ndim = None
def where(self, *args, **kwargs): print('caught where', args, kwargs) ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) cond = kwargs.get('cond', args[0]) ret.DAG.add(RA.RA_select(self, cond)) return ret
def new_project(account, graph_name="New Project"): #create a new project g_id = HashMaker().hash_graph() u_id = account # u_id = fetch_uid(account) task = DAG(g_id, u_id, graph_name) #create the new DAG task.rename_graph(graph_name) #rename the DAG values = "({},{})".format(g_id, u_id) insert('DAG_Group', values) #insert to DAG_Group return task
def groupby(self, *args, **kwargs): print('caught groupby', args, kwargs) ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) by = kwargs.get('by', args[0]) ret.DAG.add(RA.RA_groupby(self, by)) return ret
def isnull(self): print('caught isnull') ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) pred = '' ret.DAG.add(RA.RA_select(self, pred)) return ret
def test_connect_edge(self): dag = DAG() dag.add_node("A") dag.add_node("B") dag.connect("A", "B") nodelist = list(dag.traverse(lambda x: x.get_edge_str())) self.assertEqual(nodelist, [["A -> B"], []])
def __getitem__(self, attribute): print('caught get []', self._ndim) if self._ndim == 1: print('1 dim. Reverting to numpy') return self.__array__()[attribute] ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) ret.DAG.add(RA.RA_project(self, attribute)) return ret
def test_simple_adding_nodes_and_edges(self): # Check it adds a node basic_graph = DAG() basic_graph.add_node('A') self.assertEqual(basic_graph.graph, {'A': []}) # Check it doesn't add a node if it already exists self.assertFalse(basic_graph.add_node('A')) # Check it adds another node basic_graph.add_node('B') self.assertEqual(basic_graph.graph, {'A': [], 'B': []}) # Adding a directed edge from 1 to 2 basic_graph.add_edge('A', 'B') self.assertEqual(basic_graph.graph, {'A': ['B'], 'B': []}) # When adding an edge, if you include a node that isn't part of the graph it should trigger a ValueError self.assertRaises(ValueError, basic_graph.add_edge, 'B', 'C')
def drop(self, *args, **kwargs): print('caught drop labels', args, kwargs) if 'labels' in kwargs: labels = kwargs['labels'] else: labels = args[0] ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) ret.DAG.add(RA.RA_drop_labels(self, labels)) return ret
def dropna(self, *args, **kwargs): print('caught drop na subset', args, kwargs) if 'subset' in kwargs: subset = kwargs['subset'] else: subset = args[0] ret = lazyDf(self.table_names, self.dotted_fields, self.dotted_datatype, DAG(), self.con) ret.DAG.add(RA.RA_dropna_subset(self, subset)) return ret
def new_task(account, graph_id, task_name="New Task"): #create a new task # u_id = fetch_uid(account) u_id = account task = DAG(graph_id, u_id) n_id = HashMaker().hash_task() #insert to DAG_Node task.add_node(n_id) node_info = NodeInfo(n_id, u_id, task_name) #add the first node values = "({},{})".format(n_id, u_id) insert('Node_Group', values) #insert to NodeGroup node_info.rename_task(task_name) node_info.save_state() return node_info
def merge(self, *args, **kwargs): print('caught merge') other = args[0] ### add _x _y and user prefix to merge combined_table_names = self.table_names + other.table_names combined_dotted_fields = {**self.dotted_fields, **other.dotted_fields} combined_dotted_datatype = { **self.dotted_datatype, **other.dotted_datatype } ret = lazyDf(combined_table_names, combined_dotted_fields, combined_dotted_datatype, DAG(), self.con) ret.DAG.add( RA.RA_join(self, other, kwargs['right_on'], kwargs['left_on'])) return ret
class GraphPic(object): """docstring for GraphPic""" def __init__(self, graph_ID): self.graph = DAG(graph_ID) def analyse_graph(self): self.node_level = dict() # describe level for node in self.graph.graph.keys(): parents = self.graph.show_all_predecessors(node) level = len(parents) if self.node_level.get(level): self.node_level[level].add(node) else: self.node_level[level] = set() self.node_level[level].add(node) # find the max level self.max_nodes_in_one_level = 0 for level in self.node_level.keys(): self.max_nodes_in_one_level = len(self.node_level[level]) if ( len(self.node_level[level]) > self.max_nodes_in_one_level ) else self.max_nodes_in_one_level
def computeBoundingBox(self): # find top right point to create a bounding box (bottom left is [0, 0]) topRight = Point(0, 0) bottomLeft = Point(float("inf"), float("inf")) for point in self.polygon.V: # find top right if point.get_x() >= topRight.get_x(): topRight.set_x(point.get_x() + 1) if point.get_y() >= topRight.get_y(): topRight.set_y(point.get_y() + 1) # find left bottom if point.get_x() <= bottomLeft.get_x(): bottomLeft.set_x(point.get_x() - 1) if point.get_y() <= bottomLeft.get_y(): bottomLeft.set_y(point.get_y() - 1) # Now add the bounding box as a trapezoid B = Trapezoid(bottomLeft, topRight, LineSegment(Point(bottomLeft.x, topRight.y), topRight), LineSegment(bottomLeft, Point(topRight.x, bottomLeft.y))) self.T.addTrapezoid({B}) self.T.G = DAG(DAGNode(B))
class lazyDf(): def __init__(self, dag = None, con = None): if dag is None: self.DAG = DAG() else: self.DAG = cp.copy(dag) self.con = con def add_engine(self, con): self.con = con def __getitem__(self, attribute): print('caught get []') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('project', self, attribute)) ) return ret def __setitem__(self, attribute, value): print('caught set []') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('add column', self, attribute, value)) ) return ret def __eq__(self, attribute): print('caught eq') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('predicate', 'equal', self, attribute)) ) return ret def merge(self, *args, **kwargs): print('caught merge') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_join( self, args[0], kwargs['right_on'], kwargs['left_on']) ) return ret def groupby(self, *args, **kwargs): print('caught groupby', args, kwargs) ret = lazyDf(DAG(), self.con) by = kwargs.get('by', args[0]) ret.DAG.add( RA.RA_groupby( self, by ) ) return ret def mean(self, *args, **kwargs): print('caught mean') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator( ('mean', self, args, kwargs) ) ) return ret def reset_index(self, *args, **kwargs): print('caught reset index') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator( ('reset_index', self, args, kwargs) ) ) return ret def sort_values(self, *args, **kwargs): print('caught sort_values', args, kwargs) ret = lazyDf(DAG(), self.con) by = kwargs.get('by', args[0]) ret.DAG.add( RA.RA_sort( self, by ) ) return ret def __array__(self): return np.zeros(4) def _as_string(self, depth=0, indent=2): ret = 'lzdf '+str(depth)+'\n' if self.DAG.isempty: ret += " "*depth*indent + 'None' + '\n' else: ret += " "*depth*indent + self.DAG._as_string(depth+1) + '\n' return ret def __repr__(self): return self._as_string() def execute(self, query): return self.con.execute(query)
def __init__(self, graph_ID): self.graph = DAG(graph_ID)
+ "; makespan: " + str(sched_res.makespan) + "; lateness: " + str(lateness) + "\n" deadline_miss_ratio = round( float(miss_deadline) / float(len(self.applications)), 2) result += "DMRs: " + str(deadline_miss_ratio) + "; overall lateness: " + str(overall_lateness) + \ "; total makespan: " + str(total_makespan) + "\n" return result if __name__ == '__main__': print('Start DAG') task_1_path = 'output/task-1.dag' task_1_config = GraphConfig(False, True, None, None, None, None, False, task_1_path, 'output', 0, 0, 0, 0, 0, 0) DAG_1 = DAG(task_1_config) task_2_path = 'output/task-2.dag' task_2_config = GraphConfig(False, True, None, None, None, None, False, task_2_path, 'output', 0, 0, 0, 0, 0, 0) DAG_2 = DAG(task_2_config) task_3_path = 'output/task-3.dag' task_3_config = GraphConfig(False, True, None, None, None, None, False, task_3_path, 'output', 0, 0, 0, 0, 0, 0) DAG_3 = DAG(task_3_config) DAG_1.set_application_priority(1) DAG_2.set_application_priority(2) DAG_3.set_application_priority(3)
def sort_values(self, *args, **kwargs): print('caught sort_values', args, kwargs) ret = lazyDf(DAG(), self.con) by = kwargs.get('by', args[0]) ret.DAG.add( RA.RA_sort( self, by ) ) return ret
def reset_index(self, *args, **kwargs): print('caught reset index') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator( ('reset_index', self, args, kwargs) ) ) return ret
def mean(self, *args, **kwargs): print('caught mean') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator( ('mean', self, args, kwargs) ) ) return ret
def groupby(self, *args, **kwargs): print('caught groupby', args, kwargs) ret = lazyDf(DAG(), self.con) by = kwargs.get('by', args[0]) ret.DAG.add( RA.RA_groupby( self, by ) ) return ret
def merge(self, *args, **kwargs): print('caught merge') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_join( self, args[0], kwargs['right_on'], kwargs['left_on']) ) return ret
def __eq__(self, attribute): print('caught eq') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('predicate', 'equal', self, attribute)) ) return ret
def __setitem__(self, attribute, value): print('caught set []') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('add column', self, attribute, value)) ) return ret
def __getitem__(self, attribute): print('caught get []') ret = lazyDf(DAG(), self.con) ret.DAG.add( RA.RA_operator(('project', self, attribute)) ) return ret
def __init__(self, dag = None, con = None): if dag is None: self.DAG = DAG() else: self.DAG = cp.copy(dag) self.con = con
class Device: def __init__(self, frequency, task_id, H=None, transmission_power=0.5): self.freq = frequency self.task_id = task_id self.DAG = None self.p_max = transmission_power # local information self.k = math.pow(10, -28) self.H = H self.preference = None self.alpha = 0.5 # local and remote after partition self.delta = 0 self.config = None self.local = 0 self.remote = 0 self.remote_deadline = 0 self.local_to_remote_size = 0 self.local_only_energy = 0 self.local_only_enabled = True self.total_computation = 0 # job queue self.queue = [] def rate(self, p, bandwidth, k): return bandwidth * math.log2(1 + p * math.pow(self.H[k], 2) / math.pow(10, -9)) """ Given network and computation resource from the edge node, select optimal DAG partition """ def select_partition(self, full, epsilon, edge_id, f_e=None, bw=None): validation = [] if full == False: for delta in range(len(self.DAG.jobs) - 1): self.local, self.remote = 0, 0 for m in range(0, delta): self.local += self.DAG.jobs[m].computation for m in range(delta, self.DAG.length): self.remote += self.DAG.jobs[m].computation if delta == 0: self.local_to_remote_size = self.DAG.jobs[delta].input_data else: self.local_to_remote_size = self.DAG.jobs[delta - 1].output_data config = self.local_optimal_resource(delta, epsilon, edge_id, f_e, bw) if config is not None and (config[1] < self.local_only_energy or not self.local_only_enabled): validation.append(config) else: delta = 0 self.local, self.remote = 0, 0 for m in range(0, self.DAG.length): self.remote += self.DAG.jobs[m].computation self.local_to_remote_size = self.DAG.jobs[delta].input_data config = self.local_optimal_resource(delta, epsilon, edge_id, f_e, bw) if config is not None and (config[1] < self.local_only_energy or not self.local_only_enabled): validation.append(config) if len(validation) > 0: validation.sort(key=lambda x: x[1]) # print("set config", validation) # self.delta = validation[0][0] return validation[0] else: #print("run task locally") return None def local_optimal_resource(self, delta, epsilon, edge_id, f_e, bw, N=math.pow(10, -9)): # max transmission time TM = self.DAG.D / 1000 - self.remote / f_e - self.local / self.freq # min transmission time TS = self.local_to_remote_size / self.rate( self.p_max / (math.ceil(bw / math.pow(10, 6))), bw, edge_id) if TM < TS: #print("user", self.task_id, "can not be offloading by delta", delta # , "TS, TM", round(TS, 4), round(TM, 4), self.local_to_remote_size / 8000 # , "bw", bw/math.pow(10, 6), "local", self.local/math.pow(10, 9), "remote", self.remote/math.pow(10, 9)) return None #else: #print("user", self.task_id, "can be offloading by delta", delta, "TS, TM", TS, # TM) t = TS tt = [] ee = [] dd = [] config = None it = 0 while t <= TM: # 第一介导数 a = N * self.local_to_remote_size * math.pow(2, self.local_to_remote_size / (bw * t)) * math.log(2) - \ t * bw * N * math.pow(2, self.local_to_remote_size / (bw * t)) + t * bw * N c = math.pow(self.H[edge_id], 2) * t * bw directive = 2 * self.k * math.pow(self.local, 3) / math.pow(self.DAG.D/1000 - t - self.remote / f_e, 3) \ - (math.ceil(bw/math.pow(10, 6))) * a / c # energy a = self.k * self.local * math.pow( self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2) b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1 ee.append(a + math.ceil(bw / math.pow(10, 6)) * t * b * N / math.pow(self.H[edge_id], 2)) dd.append(directive) tt.append(t) # print(it, "directive", directive, "t", t) """ if math.fabs(directive) <= 0.0005: # minimal energy a = self.k * self.local * math.pow(self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2) b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1 energy = a + math.ceil(bw / math.pow(10, 6)) * t * b * N / math.pow(self.H[edge_id], 2) cpu = self.local / (self.DAG.D / 1000 - t - self.remote / f_e) power = b * N / math.pow(self.H[edge_id], 2) config = [delta, energy, cpu, power, t, directive] if self.local != 0: diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t else: diff = self.DAG.D / 1000 - self.remote / f_e - t #print("XXXXXXXXX", "user", self.task_id, "diff", diff, self.remote / f_e, t # , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size # , "time", self.local_to_remote_size / self.rate(power, bw, edge_id), # "directive", directive) break """ t = t + TM * epsilon # t = t - 0.1 * directive it = it + 1 if config is None: sm = 100 si = -1 for i in range(len(ee)): if si == -1 or ee[i] < sm: sm = ee[i] si = tt[i] t = si a = self.k * self.local * math.pow( self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2) b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1 energy = a + math.ceil(bw / math.pow( 10, 6)) * t * b * N / math.pow(self.H[edge_id], 2) cpu = self.local / (self.DAG.D / 1000 - t - self.remote / f_e) power = b * N / math.pow(self.H[edge_id], 2) # 第一介导数 a = N * self.local_to_remote_size * math.pow(2, self.local_to_remote_size / (bw * t)) * math.log(2) - \ t * bw * N * math.pow(2, self.local_to_remote_size / (bw * t)) + t * bw * N c = math.pow(self.H[edge_id], 2) * t * bw directive = 2 * self.k * math.pow(self.local, 3) / math.pow(self.DAG.D/1000 - t - self.remote / f_e, 3) \ - (math.ceil(bw/math.pow(10, 6))) * a / c if self.local != 0: diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t else: diff = self.DAG.D / 1000 - self.remote / f_e - t # print("user", self.task_id, "diff", diff, self.remote / f_e, t # , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size # , "time", self.local_to_remote_size / self.rate(power, bw, edge_id), "directive", directive) config = [delta, energy, cpu, power, t, directive] # print(config) #plt.plot(tt, ee) #plt.plot(tt, dd) #plt.xlim([TS, TM]) #plt.show() return config # [313.0, 61.0, 452.0, 228.0, 478.0] [0.0004, 0.2796, 0.0497, 0.4252, 0.2154] def remote_execution(self, f_e, bw, edge_id): if self.config is None: return -1, 0 cpu = self.config[2] * 1.00005 power = self.config[3] * 1.00005 delta = self.config[0] self.local, self.remote, data = 0, 0, self.DAG.jobs[delta].output_data for m in range(0, delta): self.local += self.DAG.jobs[m].computation for m in range(delta, self.DAG.length): self.remote += self.DAG.jobs[m].computation if delta == 0: self.local_to_remote_size = self.DAG.jobs[delta].input_data computation_time = 0 else: self.local_to_remote_size = self.DAG.jobs[delta - 1].output_data computation_time = self.local / cpu t = self.local_to_remote_size / self.rate(power, bw, edge_id) if self.local != 0: diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t else: diff = self.DAG.D / 1000 - self.remote / f_e - t #print("????", "user", self.task_id, "diff", diff, self.remote / f_e, t # , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size # , "time", self.local_to_remote_size / self.rate(power, bw, edge_id)) # print(self.task_id, self.local / cpu, cpu/math.pow(10, 9), self.local/math.pow(10, 9), computation_time) # minimal energy a = self.k * self.local * math.pow(cpu, 2) energy = a + t * power * math.ceil(bw / math.pow(10, 6)) if t + self.remote / f_e + computation_time <= self.DAG.D / 1000: return energy, 1, t, computation_time, self.remote / f_e else: # print(self.task_id, ">>>>>>>>>>>>>", t + self.remote/f_e + computation_time - self.DAG.D/1000) return energy, 0, t, computation_time, self.remote / f_e def local_only_execution(self): total_computation = 0 for m in range(self.DAG.length): total_computation += self.DAG.jobs[m].computation self.total_computation = total_computation / math.pow(10, 9) cpu = 1000 * total_computation / self.DAG.D if cpu > self.freq: self.local_only_enabled = False self.local_only_energy = self.k * total_computation * math.pow(cpu, 2) def inital_DAG(self, task_id, config=None, T=None, D=None): self.DAG = DAG(task_id, self.freq, T=T, D=D) if config is not None: self.DAG.create_from_config(config) else: self.DAG.create() self.DAG.get_valid_partition() self.DAG.local_only_compute_energy_time(self.freq, self.k)
def analyze_data(): output_root = os.getcwd() + "/" + OUTPUT_FOLDER populate_random = False # COMM RATIO comm_directory = ["comm_ratio_" + str(ratio) for ratio in COMM_COST_RATIO] report_path = os.getcwd() + "/report_paper_fat4.0.txt" f = open(report_path, "w") if not f: print('unable to write to file', report_path) return for comm_dir in comm_directory: comm_full_dir = output_root + '/' + comm_dir # FAT RATIO fat_directories = os.listdir(comm_full_dir) fat_directories = ["fat_4.0"] for fat_dir in fat_directories: fat_full_path = comm_full_dir + '/' + fat_dir # APPLICATION NO IN USE fat_files = [txt_file for txt_file in os.listdir(fat_full_path) if txt_file.endswith(".txt")] for application_count in APPLICATION_NO_USED: for analyze_times in range(ANALYZE_TIMES_PER_COMBINATION): if len(fat_files) >= application_count: application_files = random.sample(fat_files, application_count) dag_list = list() priority = 0 # load dag application for application in application_files: dag_file = fat_full_path + '/' + application graph_config = GraphConfig(populate_random, not populate_random, 0, 0, 0, 'none', False, dag_file, dag_file, 0, 0, 0, 0, 0, 0) dag = DAG(graph_config) priority += 1 dag_list.append(dag) for processor_cnt in PROCESSOR_NO_USED: for deadline_ratio in DEADLINE_RANGE: result = "comm ratio: " + comm_dir + \ " ; FAT ratio: " + fat_dir + \ " ; applications no: " + str(application_count) + \ " ; processor no: " + str(processor_cnt) + \ " ; deadline ratio : " + str(deadline_ratio) + '\n' print("COMM ratio : {} , FAT ratio : {} , applications : {} , " "processor no : {} , deadline : {}" .format(comm_dir, fat_dir, application_count, processor_cnt, deadline_ratio)) fmheft = FMHEFT(processor_cnt) wpmheft = WPMHEFT(processor_cnt) ppmheft = PPMHEFT(processor_cnt) deadline_list = list() deadval_list = list() for app in dag_list: # PROCESSOR app_ref = app app_ref.set_processor_count(processor_cnt) # DEADLINE deadline = app_ref.lowerbound + int(app_ref.lowerbound / deadline_ratio) app_ref.set_deadline(deadline) # app priority is by EDF insert_index = 0 for d in range(len(deadline_list)): if deadline <= deadline_list[d].deadline: break insert_index = d + 1 deadline_list.insert(insert_index, app_ref) deadval_list.insert(insert_index, deadline) for app_index in range(len(deadline_list)): app_ref = deadline_list[app_index] app_ref.set_application_priority(app_index + 1) fmheft.add_applications(copy.deepcopy(app_ref)) wpmheft.add_applications(copy.deepcopy(app_ref)) ppmheft.add_applications(copy.deepcopy(app_ref)) fmheft.find_makespan() wpmheft.find_makespan() ppmheft.find_makespan() result += fmheft.get_result() result += wpmheft.get_result() result += ppmheft.get_result() f.write(result) f.write('\n') f.close()