def test_add_node_exception(self):
try:
dag = DAG()
dag.add_node("A")
dag.add_node("A")
except Exception as e:
self.assertEqual(str(e), "Duplicate node in one graph.")
def inital_DAG(self, task_id, config=None, T=None, D=None):
self.DAG = DAG(task_id, self.freq, T=T, D=D)
if config is not None:
self.DAG.create_from_config(config)
else:
self.DAG.create()
self.DAG.get_valid_partition()
self.DAG.local_only_compute_energy_time(self.freq, self.k)
def __init__(self,
table_names,
dotted_fields,
dotted_datatype,
dag=None,
con=None):
self.table_names = table_names
self.dotted_fields = dotted_fields
self.dotted_datatype = dotted_datatype
if dag is None:
self.DAG = DAG()
else:
self.DAG = cp.copy(dag)
self.con = con
self._shape = None
self._ndim = None
def where(self, *args, **kwargs):
print('caught where', args, kwargs)
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
cond = kwargs.get('cond', args[0])
ret.DAG.add(RA.RA_select(self, cond))
return ret
def new_project(account, graph_name="New Project"):
#create a new project
g_id = HashMaker().hash_graph()
u_id = account
# u_id = fetch_uid(account)
task = DAG(g_id, u_id, graph_name)
#create the new DAG
task.rename_graph(graph_name)
#rename the DAG
values = "({},{})".format(g_id, u_id)
insert('DAG_Group', values)
#insert to DAG_Group
return task
def groupby(self, *args, **kwargs):
print('caught groupby', args, kwargs)
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
by = kwargs.get('by', args[0])
ret.DAG.add(RA.RA_groupby(self, by))
return ret
def isnull(self):
print('caught isnull')
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
pred = ''
ret.DAG.add(RA.RA_select(self, pred))
return ret
def test_connect_edge(self):
dag = DAG()
dag.add_node("A")
dag.add_node("B")
dag.connect("A", "B")
nodelist = list(dag.traverse(lambda x: x.get_edge_str()))
self.assertEqual(nodelist, [["A -> B"], []])
def __getitem__(self, attribute):
print('caught get []', self._ndim)
if self._ndim == 1:
print('1 dim. Reverting to numpy')
return self.__array__()[attribute]
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
ret.DAG.add(RA.RA_project(self, attribute))
return ret
def test_simple_adding_nodes_and_edges(self):
# Check it adds a node
basic_graph = DAG()
basic_graph.add_node('A')
self.assertEqual(basic_graph.graph, {'A': []})
# Check it doesn't add a node if it already exists
self.assertFalse(basic_graph.add_node('A'))
# Check it adds another node
basic_graph.add_node('B')
self.assertEqual(basic_graph.graph, {'A': [], 'B': []})
# Adding a directed edge from 1 to 2
basic_graph.add_edge('A', 'B')
self.assertEqual(basic_graph.graph, {'A': ['B'], 'B': []})
# When adding an edge, if you include a node that isn't part of the graph it should trigger a ValueError
self.assertRaises(ValueError, basic_graph.add_edge, 'B', 'C')
def drop(self, *args, **kwargs):
print('caught drop labels', args, kwargs)
if 'labels' in kwargs:
labels = kwargs['labels']
else:
labels = args[0]
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
ret.DAG.add(RA.RA_drop_labels(self, labels))
return ret
def dropna(self, *args, **kwargs):
print('caught drop na subset', args, kwargs)
if 'subset' in kwargs:
subset = kwargs['subset']
else:
subset = args[0]
ret = lazyDf(self.table_names, self.dotted_fields,
self.dotted_datatype, DAG(), self.con)
ret.DAG.add(RA.RA_dropna_subset(self, subset))
return ret
def new_task(account, graph_id, task_name="New Task"):
#create a new task
# u_id = fetch_uid(account)
u_id = account
task = DAG(graph_id, u_id)
n_id = HashMaker().hash_task()
#insert to DAG_Node
task.add_node(n_id)
node_info = NodeInfo(n_id, u_id, task_name)
#add the first node
values = "({},{})".format(n_id, u_id)
insert('Node_Group', values)
#insert to NodeGroup
node_info.rename_task(task_name)
node_info.save_state()
return node_info
def merge(self, *args, **kwargs):
print('caught merge')
other = args[0]
### add _x _y and user prefix to merge
combined_table_names = self.table_names + other.table_names
combined_dotted_fields = {**self.dotted_fields, **other.dotted_fields}
combined_dotted_datatype = {
**self.dotted_datatype,
**other.dotted_datatype
}
ret = lazyDf(combined_table_names, combined_dotted_fields,
combined_dotted_datatype, DAG(), self.con)
ret.DAG.add(
RA.RA_join(self, other, kwargs['right_on'], kwargs['left_on']))
return ret
class GraphPic(object):
"""docstring for GraphPic"""
def __init__(self, graph_ID):
self.graph = DAG(graph_ID)
def analyse_graph(self):
self.node_level = dict()
# describe level
for node in self.graph.graph.keys():
parents = self.graph.show_all_predecessors(node)
level = len(parents)
if self.node_level.get(level):
self.node_level[level].add(node)
else:
self.node_level[level] = set()
self.node_level[level].add(node)
# find the max level
self.max_nodes_in_one_level = 0
for level in self.node_level.keys():
self.max_nodes_in_one_level = len(self.node_level[level]) if (
len(self.node_level[level]) > self.max_nodes_in_one_level
) else self.max_nodes_in_one_level
def computeBoundingBox(self):
# find top right point to create a bounding box (bottom left is [0, 0])
topRight = Point(0, 0)
bottomLeft = Point(float("inf"), float("inf"))
for point in self.polygon.V:
# find top right
if point.get_x() >= topRight.get_x():
topRight.set_x(point.get_x() + 1)
if point.get_y() >= topRight.get_y():
topRight.set_y(point.get_y() + 1)
# find left bottom
if point.get_x() <= bottomLeft.get_x():
bottomLeft.set_x(point.get_x() - 1)
if point.get_y() <= bottomLeft.get_y():
bottomLeft.set_y(point.get_y() - 1)
# Now add the bounding box as a trapezoid
B = Trapezoid(bottomLeft, topRight,
LineSegment(Point(bottomLeft.x, topRight.y), topRight),
LineSegment(bottomLeft, Point(topRight.x, bottomLeft.y)))
self.T.addTrapezoid({B})
self.T.G = DAG(DAGNode(B))
class lazyDf():
def __init__(self, dag = None, con = None):
if dag is None:
self.DAG = DAG()
else:
self.DAG = cp.copy(dag)
self.con = con
def add_engine(self, con):
self.con = con
def __getitem__(self, attribute):
print('caught get []')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('project', self, attribute)) )
return ret
def __setitem__(self, attribute, value):
print('caught set []')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('add column', self, attribute, value)) )
return ret
def __eq__(self, attribute):
print('caught eq')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('predicate', 'equal', self, attribute)) )
return ret
def merge(self, *args, **kwargs):
print('caught merge')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_join( self, args[0], kwargs['right_on'], kwargs['left_on']) )
return ret
def groupby(self, *args, **kwargs):
print('caught groupby', args, kwargs)
ret = lazyDf(DAG(), self.con)
by = kwargs.get('by', args[0])
ret.DAG.add( RA.RA_groupby( self, by ) )
return ret
def mean(self, *args, **kwargs):
print('caught mean')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator( ('mean', self, args, kwargs) ) )
return ret
def reset_index(self, *args, **kwargs):
print('caught reset index')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator( ('reset_index', self, args, kwargs) ) )
return ret
def sort_values(self, *args, **kwargs):
print('caught sort_values', args, kwargs)
ret = lazyDf(DAG(), self.con)
by = kwargs.get('by', args[0])
ret.DAG.add( RA.RA_sort( self, by ) )
return ret
def __array__(self):
return np.zeros(4)
def _as_string(self, depth=0, indent=2):
ret = 'lzdf '+str(depth)+'\n'
if self.DAG.isempty:
ret += " "*depth*indent + 'None' + '\n'
else:
ret += " "*depth*indent + self.DAG._as_string(depth+1) + '\n'
return ret
def __repr__(self):
return self._as_string()
def execute(self, query):
return self.con.execute(query)
def __init__(self, graph_ID):
self.graph = DAG(graph_ID)
+ "; makespan: " + str(sched_res.makespan) + "; lateness: " + str(lateness) + "\n"
deadline_miss_ratio = round(
float(miss_deadline) / float(len(self.applications)), 2)
result += "DMRs: " + str(deadline_miss_ratio) + "; overall lateness: " + str(overall_lateness) + \
"; total makespan: " + str(total_makespan) + "\n"
return result
if __name__ == '__main__':
print('Start DAG')
task_1_path = 'output/task-1.dag'
task_1_config = GraphConfig(False, True, None, None, None, None, False,
task_1_path, 'output', 0, 0, 0, 0, 0, 0)
DAG_1 = DAG(task_1_config)
task_2_path = 'output/task-2.dag'
task_2_config = GraphConfig(False, True, None, None, None, None, False,
task_2_path, 'output', 0, 0, 0, 0, 0, 0)
DAG_2 = DAG(task_2_config)
task_3_path = 'output/task-3.dag'
task_3_config = GraphConfig(False, True, None, None, None, None, False,
task_3_path, 'output', 0, 0, 0, 0, 0, 0)
DAG_3 = DAG(task_3_config)
DAG_1.set_application_priority(1)
DAG_2.set_application_priority(2)
DAG_3.set_application_priority(3)
def sort_values(self, *args, **kwargs):
print('caught sort_values', args, kwargs)
ret = lazyDf(DAG(), self.con)
by = kwargs.get('by', args[0])
ret.DAG.add( RA.RA_sort( self, by ) )
return ret
def reset_index(self, *args, **kwargs):
print('caught reset index')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator( ('reset_index', self, args, kwargs) ) )
return ret
def mean(self, *args, **kwargs):
print('caught mean')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator( ('mean', self, args, kwargs) ) )
return ret
def groupby(self, *args, **kwargs):
print('caught groupby', args, kwargs)
ret = lazyDf(DAG(), self.con)
by = kwargs.get('by', args[0])
ret.DAG.add( RA.RA_groupby( self, by ) )
return ret
def merge(self, *args, **kwargs):
print('caught merge')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_join( self, args[0], kwargs['right_on'], kwargs['left_on']) )
return ret
def __eq__(self, attribute):
print('caught eq')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('predicate', 'equal', self, attribute)) )
return ret
def __setitem__(self, attribute, value):
print('caught set []')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('add column', self, attribute, value)) )
return ret
def __getitem__(self, attribute):
print('caught get []')
ret = lazyDf(DAG(), self.con)
ret.DAG.add( RA.RA_operator(('project', self, attribute)) )
return ret
def __init__(self, dag = None, con = None):
if dag is None:
self.DAG = DAG()
else:
self.DAG = cp.copy(dag)
self.con = con
class Device:
def __init__(self, frequency, task_id, H=None, transmission_power=0.5):
self.freq = frequency
self.task_id = task_id
self.DAG = None
self.p_max = transmission_power
# local information
self.k = math.pow(10, -28)
self.H = H
self.preference = None
self.alpha = 0.5
# local and remote after partition
self.delta = 0
self.config = None
self.local = 0
self.remote = 0
self.remote_deadline = 0
self.local_to_remote_size = 0
self.local_only_energy = 0
self.local_only_enabled = True
self.total_computation = 0
# job queue
self.queue = []
def rate(self, p, bandwidth, k):
return bandwidth * math.log2(1 + p * math.pow(self.H[k], 2) /
math.pow(10, -9))
"""
Given network and computation resource from the edge node, select optimal DAG partition
"""
def select_partition(self, full, epsilon, edge_id, f_e=None, bw=None):
validation = []
if full == False:
for delta in range(len(self.DAG.jobs) - 1):
self.local, self.remote = 0, 0
for m in range(0, delta):
self.local += self.DAG.jobs[m].computation
for m in range(delta, self.DAG.length):
self.remote += self.DAG.jobs[m].computation
if delta == 0:
self.local_to_remote_size = self.DAG.jobs[delta].input_data
else:
self.local_to_remote_size = self.DAG.jobs[delta -
1].output_data
config = self.local_optimal_resource(delta, epsilon, edge_id,
f_e, bw)
if config is not None and (config[1] < self.local_only_energy
or not self.local_only_enabled):
validation.append(config)
else:
delta = 0
self.local, self.remote = 0, 0
for m in range(0, self.DAG.length):
self.remote += self.DAG.jobs[m].computation
self.local_to_remote_size = self.DAG.jobs[delta].input_data
config = self.local_optimal_resource(delta, epsilon, edge_id, f_e,
bw)
if config is not None and (config[1] < self.local_only_energy
or not self.local_only_enabled):
validation.append(config)
if len(validation) > 0:
validation.sort(key=lambda x: x[1])
# print("set config", validation)
# self.delta = validation[0][0]
return validation[0]
else:
#print("run task locally")
return None
def local_optimal_resource(self,
delta,
epsilon,
edge_id,
f_e,
bw,
N=math.pow(10, -9)):
# max transmission time
TM = self.DAG.D / 1000 - self.remote / f_e - self.local / self.freq
# min transmission time
TS = self.local_to_remote_size / self.rate(
self.p_max / (math.ceil(bw / math.pow(10, 6))), bw, edge_id)
if TM < TS:
#print("user", self.task_id, "can not be offloading by delta", delta
# , "TS, TM", round(TS, 4), round(TM, 4), self.local_to_remote_size / 8000
# , "bw", bw/math.pow(10, 6), "local", self.local/math.pow(10, 9), "remote", self.remote/math.pow(10, 9))
return None
#else:
#print("user", self.task_id, "can be offloading by delta", delta, "TS, TM", TS,
# TM)
t = TS
tt = []
ee = []
dd = []
config = None
it = 0
while t <= TM:
# 第一介导数
a = N * self.local_to_remote_size * math.pow(2, self.local_to_remote_size / (bw * t)) * math.log(2) - \
t * bw * N * math.pow(2, self.local_to_remote_size / (bw * t)) + t * bw * N
c = math.pow(self.H[edge_id], 2) * t * bw
directive = 2 * self.k * math.pow(self.local, 3) / math.pow(self.DAG.D/1000 - t - self.remote / f_e, 3) \
- (math.ceil(bw/math.pow(10, 6))) * a / c
# energy
a = self.k * self.local * math.pow(
self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2)
b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1
ee.append(a + math.ceil(bw / math.pow(10, 6)) * t * b * N /
math.pow(self.H[edge_id], 2))
dd.append(directive)
tt.append(t)
# print(it, "directive", directive, "t", t)
"""
if math.fabs(directive) <= 0.0005:
# minimal energy
a = self.k * self.local * math.pow(self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2)
b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1
energy = a + math.ceil(bw / math.pow(10, 6)) * t * b * N / math.pow(self.H[edge_id], 2)
cpu = self.local / (self.DAG.D / 1000 - t - self.remote / f_e)
power = b * N / math.pow(self.H[edge_id], 2)
config = [delta, energy, cpu, power, t, directive]
if self.local != 0:
diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t
else:
diff = self.DAG.D / 1000 - self.remote / f_e - t
#print("XXXXXXXXX", "user", self.task_id, "diff", diff, self.remote / f_e, t
# , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size
# , "time", self.local_to_remote_size / self.rate(power, bw, edge_id),
# "directive", directive)
break
"""
t = t + TM * epsilon
# t = t - 0.1 * directive
it = it + 1
if config is None:
sm = 100
si = -1
for i in range(len(ee)):
if si == -1 or ee[i] < sm:
sm = ee[i]
si = tt[i]
t = si
a = self.k * self.local * math.pow(
self.local / (self.DAG.D / 1000 - t - self.remote / f_e), 2)
b = math.pow(2, self.local_to_remote_size / (bw * t)) - 1
energy = a + math.ceil(bw / math.pow(
10, 6)) * t * b * N / math.pow(self.H[edge_id], 2)
cpu = self.local / (self.DAG.D / 1000 - t - self.remote / f_e)
power = b * N / math.pow(self.H[edge_id], 2)
# 第一介导数
a = N * self.local_to_remote_size * math.pow(2, self.local_to_remote_size / (bw * t)) * math.log(2) - \
t * bw * N * math.pow(2, self.local_to_remote_size / (bw * t)) + t * bw * N
c = math.pow(self.H[edge_id], 2) * t * bw
directive = 2 * self.k * math.pow(self.local, 3) / math.pow(self.DAG.D/1000 - t - self.remote / f_e, 3) \
- (math.ceil(bw/math.pow(10, 6))) * a / c
if self.local != 0:
diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t
else:
diff = self.DAG.D / 1000 - self.remote / f_e - t
# print("user", self.task_id, "diff", diff, self.remote / f_e, t
# , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size
# , "time", self.local_to_remote_size / self.rate(power, bw, edge_id), "directive", directive)
config = [delta, energy, cpu, power, t, directive]
# print(config)
#plt.plot(tt, ee)
#plt.plot(tt, dd)
#plt.xlim([TS, TM])
#plt.show()
return config
# [313.0, 61.0, 452.0, 228.0, 478.0] [0.0004, 0.2796, 0.0497, 0.4252, 0.2154]
def remote_execution(self, f_e, bw, edge_id):
if self.config is None:
return -1, 0
cpu = self.config[2] * 1.00005
power = self.config[3] * 1.00005
delta = self.config[0]
self.local, self.remote, data = 0, 0, self.DAG.jobs[delta].output_data
for m in range(0, delta):
self.local += self.DAG.jobs[m].computation
for m in range(delta, self.DAG.length):
self.remote += self.DAG.jobs[m].computation
if delta == 0:
self.local_to_remote_size = self.DAG.jobs[delta].input_data
computation_time = 0
else:
self.local_to_remote_size = self.DAG.jobs[delta - 1].output_data
computation_time = self.local / cpu
t = self.local_to_remote_size / self.rate(power, bw, edge_id)
if self.local != 0:
diff = self.DAG.D / 1000 - self.remote / f_e - self.local / cpu - t
else:
diff = self.DAG.D / 1000 - self.remote / f_e - t
#print("????", "user", self.task_id, "diff", diff, self.remote / f_e, t
# , "data", t * self.rate(power, bw, edge_id), self.local_to_remote_size
# , "time", self.local_to_remote_size / self.rate(power, bw, edge_id))
# print(self.task_id, self.local / cpu, cpu/math.pow(10, 9), self.local/math.pow(10, 9), computation_time)
# minimal energy
a = self.k * self.local * math.pow(cpu, 2)
energy = a + t * power * math.ceil(bw / math.pow(10, 6))
if t + self.remote / f_e + computation_time <= self.DAG.D / 1000:
return energy, 1, t, computation_time, self.remote / f_e
else:
# print(self.task_id, ">>>>>>>>>>>>>", t + self.remote/f_e + computation_time - self.DAG.D/1000)
return energy, 0, t, computation_time, self.remote / f_e
def local_only_execution(self):
total_computation = 0
for m in range(self.DAG.length):
total_computation += self.DAG.jobs[m].computation
self.total_computation = total_computation / math.pow(10, 9)
cpu = 1000 * total_computation / self.DAG.D
if cpu > self.freq:
self.local_only_enabled = False
self.local_only_energy = self.k * total_computation * math.pow(cpu, 2)
def inital_DAG(self, task_id, config=None, T=None, D=None):
self.DAG = DAG(task_id, self.freq, T=T, D=D)
if config is not None:
self.DAG.create_from_config(config)
else:
self.DAG.create()
self.DAG.get_valid_partition()
self.DAG.local_only_compute_energy_time(self.freq, self.k)
def analyze_data():
output_root = os.getcwd() + "/" + OUTPUT_FOLDER
populate_random = False
# COMM RATIO
comm_directory = ["comm_ratio_" + str(ratio) for ratio in COMM_COST_RATIO]
report_path = os.getcwd() + "/report_paper_fat4.0.txt"
f = open(report_path, "w")
if not f:
print('unable to write to file', report_path)
return
for comm_dir in comm_directory:
comm_full_dir = output_root + '/' + comm_dir
# FAT RATIO
fat_directories = os.listdir(comm_full_dir)
fat_directories = ["fat_4.0"]
for fat_dir in fat_directories:
fat_full_path = comm_full_dir + '/' + fat_dir
# APPLICATION NO IN USE
fat_files = [txt_file for txt_file in os.listdir(fat_full_path) if txt_file.endswith(".txt")]
for application_count in APPLICATION_NO_USED:
for analyze_times in range(ANALYZE_TIMES_PER_COMBINATION):
if len(fat_files) >= application_count:
application_files = random.sample(fat_files, application_count)
dag_list = list()
priority = 0
# load dag application
for application in application_files:
dag_file = fat_full_path + '/' + application
graph_config = GraphConfig(populate_random, not populate_random,
0, 0, 0, 'none', False,
dag_file, dag_file,
0, 0, 0, 0, 0, 0)
dag = DAG(graph_config)
priority += 1
dag_list.append(dag)
for processor_cnt in PROCESSOR_NO_USED:
for deadline_ratio in DEADLINE_RANGE:
result = "comm ratio: " + comm_dir + \
" ; FAT ratio: " + fat_dir + \
" ; applications no: " + str(application_count) + \
" ; processor no: " + str(processor_cnt) + \
" ; deadline ratio : " + str(deadline_ratio) + '\n'
print("COMM ratio : {} , FAT ratio : {} , applications : {} , "
"processor no : {} , deadline : {}"
.format(comm_dir, fat_dir, application_count, processor_cnt, deadline_ratio))
fmheft = FMHEFT(processor_cnt)
wpmheft = WPMHEFT(processor_cnt)
ppmheft = PPMHEFT(processor_cnt)
deadline_list = list()
deadval_list = list()
for app in dag_list:
# PROCESSOR
app_ref = app
app_ref.set_processor_count(processor_cnt)
# DEADLINE
deadline = app_ref.lowerbound + int(app_ref.lowerbound / deadline_ratio)
app_ref.set_deadline(deadline)
# app priority is by EDF
insert_index = 0
for d in range(len(deadline_list)):
if deadline <= deadline_list[d].deadline:
break
insert_index = d + 1
deadline_list.insert(insert_index, app_ref)
deadval_list.insert(insert_index, deadline)
for app_index in range(len(deadline_list)):
app_ref = deadline_list[app_index]
app_ref.set_application_priority(app_index + 1)
fmheft.add_applications(copy.deepcopy(app_ref))
wpmheft.add_applications(copy.deepcopy(app_ref))
ppmheft.add_applications(copy.deepcopy(app_ref))
fmheft.find_makespan()
wpmheft.find_makespan()
ppmheft.find_makespan()
result += fmheft.get_result()
result += wpmheft.get_result()
result += ppmheft.get_result()
f.write(result)
f.write('\n')
f.close()