def setUp(self):
self.workflow = Workflow(
'test/data/daliugesample.json',
cfg.test_heuristic_data['topcuoglu_graph_system'],
calc_time=False)
self.dense = Workflow(
'test/data/ggen_out_4-denselu.json',
cfg.test_heuristic_data['topcuoglu_graph_system'],
calc_time=False)
self.gnp = Workflow('test/data/ggen_out_20-0.5.json',
cfg.test_heuristic_data['topcuoglu_graph_system'],
calc_time=False)
def setUp(self):
self.wf = Workflow("{0}/{1}".format(
current_dir, cfg.test_metaheuristic_data['topcuoglu_graph']))
env = Environment("{0}/{1}".format(
current_dir, cfg.test_metaheuristic_data['graph_sys_with_costs']))
self.wf.add_environment(env)
self.rng = np.random.default_rng(40)
def run_algorithm(arg, parser):
if arg['algorithm'] == 'heft':
pass
wf = Workflow(arg['workflow'])
env = Environment(arg['environment'])
wf.add_environment(env)
print(heft(wf))
print(wf.machine_alloc)
def test_large_workflow(self):
self.workflow = Workflow(
"/home/rwb/Dropbox/PhD/writeups/observation_graph-model/json/3000Node.json"
)
env = Environment(
"/home/rwb/Dropbox/PhD/writeups/observation_graph-model/json/3000Node_sys.json"
)
self.workflow.add_environment(env)
heft(self.workflow)
def setUp(self):
self.workflow = Workflow("{0}/{1}".format(
current_dir, cfg.test_workflow_data['topcuoglu_graph']))
env = Environment("{0}/{1}".format(
current_dir, cfg.test_workflow_data['topcuoglu_graph_system']))
# self.workflow = Workfow(cfg.test_heuristic_data['topcuoglu_graph'],
# cfg.test_heuristic_data['topcuoglu_graph_system'])
self.workflow.add_environment(env)
def test_time_false(self):
"""
When we read in the environment file and add it to to the workflow, we do some
pre-processing of the default computing values. If we have errors in the workflow or environment
config files, we need to ensure we exit appropriately.
"""
# We have a workflow with time is false, but time is actually true. This means there is more
# than one value in the 'comp' attribute, which is incorrect.
# We havea workflow with time: true, but time is actually false; i.e. there is only one value
# stored in 'comp' attribute, when there should be multiply. REMEMBER, time: true implies that runtime
# has been previously calculated for each machine.
workflow_true_but_false = Workflow('test/data/workflow/exception_raised_timeistrue.json')
self.assertRaises(TypeError, workflow_true_but_false.add_environment, self.env)
pass
def _initialise_shadow_workflows(self, observation, cluster):
"""
Use the SHADOW library workflow model to build the graph
Parameters
----------
observation
Returns
-------
workflow : shadow.models.workflow.Worflow
A wrapper for NetworkX DiGraph object with additional information
"""
workflow = Workflow(observation.workflow)
available_resources = self._cluster_to_shadow_format(cluster)
workflow_env = Environment(available_resources, dictionary=True)
workflow.add_environment(workflow_env)
return workflow
# import config as cfg
from shadow.algorithms.heuristic import heft
from shadow.models.workflow import Workflow
# This workflow calculates the task time for each resource based on the demand
# and supply vectors provided in the 'flop_rep_test.json' fsile.
wf = Workflow('topcuoglu.graphml')
retval = wf.load_attributes('flop_rep_test.json')
print(heft(wf))
wf.pretty_print_allocation()
# Original HEFT workflow; task time on reach resource is provided directly by
# the .json file.
wf = Workflow('topcuoglu.graphml')
retval = wf.load_attributes('heft.json',calc_time=False)
print(heft(wf))
wf.pretty_print_allocation()
# Copyright (C) 17/6/20 RW Bunney
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
# This is adapted and expanded upon in the shadowgen.ipynb notebook
from shadow.models.workflow import Workflow
from shadow.models.environment import Environment
import shadow.algorithms.heuristic as heuristic
workflow = Workflow('dax_files/output/shadow_Epigenomics_24.json')
env = Environment('environments/sys.json')
workflow.add_environment(env)
heuristic.heft(workflow)
from shadow.algorithms.heuristic import heft from shadow.models.workflow import Workflow from shadow.models.environment import Environment from shadow.visualiser.plot import AllocationPlot import matplotlib.pyplot as plt from test import config HEFTWorkflow = Workflow(config.test_heuristic_data['topcuoglu_graph']) env = Environment(config.test_heuristic_data['topcuoglu_graph_system']) HEFTWorkflow.add_environment(env) heft(HEFTWorkflow) sample_allocation = AllocationPlot(solution=HEFTWorkflow.solution) fig, ax = sample_allocation.plot() plt.show()
def setUp(self):
self.wf = Workflow("{0}/{1}".format(current_dir, cfg.test_workflow_data['topcuoglu_graph']))
self.env = Environment("{0}/{1}".format(current_dir, cfg.test_workflow_data['topcuoglu_graph_system']))
def run_workflow(workflow, environment):
workflow.add_environment(environment)
return heft(workflow)
if __name__ == '__main__':
output_dir_path = Path(f"notebooks/sdp_comparison/output")
if not output_dir_path.exists():
raise RuntimeError(f"{output_dir_path} does not exist!")
config_path = output_dir_path / "shadow_config.json"
wfs = []
for wf_path in (output_dir_path / "workflows").iterdir():
wfs.append(Workflow(wf_path))
env = Environment(config_path)
environs = [deepcopy(env) for i in range(3)]
params = list(zip(wfs, environs))
start = time.time()
with Pool(processes=4) as pool:
result = pool.starmap(run_workflow, params)
finish = time.time()
print(f"{finish-start=}")
# start = time.time()
# for x in range(3):
# workflow = Workflow("/home/rwb/github/thesis_experiments/skaworkflows_tests"
with 40 machines.
"""
import pandas as pd
import seaborn as sns
from shadow.models.workflow import Workflow
from shadow.models.environment import Environment
from shadow.algorithms.heuristic import heft, fcfs
# df = pd.DataFrame(columns=["time", "channels", "algorithm", "diff"])
df = pd.DataFrame()
WORKFLOW = "routput/shadow_Continuum_ChannelSplit_10.json"
CLUSTER = "routput/system_spec_40_200-400_1.0"
wf_fcfs = Workflow(WORKFLOW)
env_fcfs = Environment(CLUSTER)
wf_fcfs.add_environment(env_fcfs)
soln = fcfs(wf_fcfs)
seq = -1
#
# for p in range(len(self.processors)):
# comp = 0
# for task in list(self.graph.nodes()):
# comp = comp + self.comp_matrix[task.tid][p]
# if (seq is -1) or (comp < seq):
# seq = comp
for x in range(10, 100, 10):
print("Scheduling {0} Channels".format(x))
from queue import Queue
from shadow.algorithms.heuristic import heft
from shadow.models.workflow import Workflow, Task
from shadow.models.environment import Environment
HEFTWorkflow = Workflow('heft.json')
env = Environment('sys.json')
HEFTWorkflow.add_environment(env)
print(heft(HEFTWorkflow))
DelayWorkflow = Workflow('heft_delay.json')
DelayWorkflow.add_environment(env)
print(heft(DelayWorkflow))
def calc_task_delay(task, delay, workflow):
t = workflow.graph.tasks[task]
aft = t.aft
update_list = list(workflow.graph.successors(t))
# add 10 to the start and finish time of each of these successors, and their successors
update_queue = Queue()
update_queue.put(update_list)
print(update_queue)
return workflow
task = HEFTWorkflow.tasks[0]
calc_task_delay(task, 10, workflow=HEFTWorkflow)
def setUp(self):
self.workflow = Workflow(cfg.test_heuristic_data['pheft_graph'])
env = Environment(cfg.test_workflow_data['topcuoglu_graph_system'])
self.workflow.add_environment(env)
self.up_oct_rank_values = [72, 41, 37, 43, 31, 41, 17, 20, 16, 0]
self.up_rank_values = [169, 114, 102, 110, 129, 119, 52, 92, 42, 20]
# Copyright (C) 26/6/20 RW Bunney
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
from shadow.models.workflow import Workflow
from shadow.models.environment import Environment
import shadow.visualiser.graph as sgraph
from IPython.display import Image
workflow_file = 'TestAskapCont_channels-10_shadow.json'
sys = 'final_heft_sys.json'
workflow = Workflow(workflow_file)
env = Environment(sys)
workflow.add_environment(env)
# png = sgraph.visualise_graph(workflow, workflow_file.strip('.json')+'.png')
graphviz = sgraph.convert_to_graphviz(workflow)
graphviz.render('output.gz')
def setUp(self) -> None:
self.workflow = Workflow("{0}/{1}".format(current_dir, cfg.test_workflow_data['topcuoglu_graph']))