def parseCommandDict(self, startupCommands=[]):
print "PARSING COMMAND DICTIONARY!!!"
from InstanceCommand import InstanceCommand
newCommands = {}
# make commands
print self.command_dict
for id in self.command_dict:
newCommands[id] = []
deps = startupCommands # the first command in all command sequences get startup commands as dependencies
for line in self.command_dict[id]["command"].split("\n"):
line = line.strip()
if line!="":
newCommand = InstanceCommand(self, line, deps, "main")
newCommands[id].append(newCommand)
deps = [newCommand]
print newCommands.keys()
for key in newCommands.keys():
print "KEY:", key, newCommands[key]
# print newCommands.values()
# set dependencies
for id in newCommands:
deps = []
# deps = startupCommands
for newCommand in newCommands[id]:
for did in self.command_dict[id]["dependencies"]:
deps += newCommands[did]
print newCommand, "adding command deps", newCommands[did]
# newCommand.dependencies = [newCommands[did] for did in self.command_dict[id]["dependencies"]]+startupCommands
newCommand.command_dependencies = list(set(newCommand.command_dependencies+deps))
result = []
for commandList in newCommands.values():
result += commandList
return result
def getCommand(self):
user = self.getUser()
instance = self.getInstance()
command = "echo 'test!'"
c = InstanceCommand.findByCommand(SESSION, command, user)
if c == None:
c = InstanceCommand(instance, command, [], "main")
return c
def getCommands(self):
"""
This method communicates with the master node and gets all commands to run.
"""
print self.base_address + "/api/commands"
commandData = self.communicator.get(self.base_address.strip("/") + "/api/commands")
print "commands from server:", commandData
self.commands = InstanceCommand.generateCommandsFromDataDict(commandData)
return self.commands
def getInstanceCommandData(self, workflow_id, instance_id):
from DDServerApp.ORM.Mappers import InstanceCommand
wf = [w for w in self.workflows if str(w.id)==str(workflow_id)]
result = {}
if wf !=[]:
wf = wf[0]
inst = [i for i in wf.instances if str(i.id)==str(instance_id)]
if inst!=[]:
inst = inst[0]
result["colnames"] = InstanceCommand.getTableNames()
result["rows"] = [c.getTableData() for c in inst.commands]
result["numrows"] = len(result["rows"])
return result
def run(self):
# first get commands
self.getCommands()
if VERBOSE:
print self.commands
for c in self.commands:
c.finished = False
# next get first commands
self.first_commands = InstanceCommand.getFirstCommands(self.commands)
# add commands to run to a queue
threads = []
for command in self.first_commands:
threads.append(RunCommandThread(command, worker=self))
# start all threads, when each thread finishes command it starts a new set of
# threads for yet to be completed and ready commands and waits for them to
# complete, so when the initial threads are finished all threads are finished
for thread in threads:
thread.start()
# what for threads to finish
for thread in threads:
thread.join()
def testGetFirstCommands(self):
commands = self.getCommands()
firstcommands = InstanceCommand.getFirstCommands(commands)
# do first commands have any dependencies in all commands
allIDs = [c.id for c in commands]
self.assertTrue(all([all([cd.id not in allIDs for cd in c.command_dependencies]) for c in firstcommands]), "first commands have dependencies in all commands")