def hasSpaceFor(self, launchableTool, calendarName):
"""
Check if this worker has space for the given tool. (this checks the command and every group of commands max_thred settings)
Args:
launchableTool: a tool documents fetched from database that has to be launched
Returns:
Return True if a command of the tool's type can be launched on this worker, False otherwise.
"""
# 1. Find command with command id
command = Command.fetchObject({"name": launchableTool.name},
calendarName)
if command.safe == "False":
return False
# 2. Calculate individual command limit for the server
nb = self.getNbOfLaunchedCommand(command.name) + 1
if nb > int(command.max_thread):
#print("Can't launch "+command.name+" on worker cause command max_trhad "+str(nb)+" > "+str(int(command.max_thread)))
return False
# 3. Get groups of command incorporation command id
command_groups = CommandGroup.fetchObjects(
{"commands": {
"$elemMatch": {
"$eq": command.name
}
}})
# 4. Calculate limites for the group
for group in command_groups:
tots = 0
for commandName in group.commands:
tots += self.getNbOfLaunchedCommand(commandName)
if tots + 1 > int(group.max_thread):
#print("Can't launch "+command.name+" on worker cause group_max_thread "+str(tots + 1)+" > "+str(int(group.max_thread)))
return False
return True
def executeCommand(calendarName, toolId, parser=""):
"""
CELERY remote task
Execute the tool with the given toolId on the given calendar name.
Then execute the plugin corresponding.
Any unhandled exception will result in a task-failed event in the Monitor class.
Args:
calendarName: The calendar to search the given tool id for.
toolId: the mongo Object id corresponding to the tool to execute.
parser: plugin name to execute. If empty, the plugin specified in tools.d will be feteched.
Raises:
Terminated: if the task gets terminated
OSError: if the output directory cannot be created (not if it already exists)
Exception: if an exception unhandled occurs during the bash command execution.
Exception: if a plugin considered a failure.
"""
# Connect to given calendar
mongoInstance = MongoCalendar.getInstance()
mongoInstance.connectToDb(calendarName)
msg = ""
# retrieve tool from tool sid
toolModel = Tool.fetchObject({"_id": ObjectId(toolId)})
if toolModel is None:
raise Exception("Tool does not exist : " + str(toolId))
command = Command.fetchObject({"name": toolModel.name}, calendarName)
# Get time limit and output directory
if toolModel.wave == "Custom commands":
timeLimit = None
else:
timeLimit = getWaveTimeLimit(toolModel.wave)
if command is not None:
timeLimit = min(datetime.now() + timedelta(0, int(command.timeout)),
timeLimit)
outputRelDir = toolModel.getOutputDir(calendarName)
abs_path = os.path.dirname(os.path.abspath(__file__))
outputDir = os.path.join(abs_path, "./results", outputRelDir)
# Create the output directory
try:
os.makedirs(outputDir)
except OSError as exc:
if exc.errno == errno.EEXIST and os.path.isdir(outputDir):
pass
else:
raise exc
# Read Tool config file
tools_infos = Utils.loadToolsConfig()
comm = toolModel.getCommandToExecute(outputDir)
if parser.strip() == "":
if toolModel.name not in list(tools_infos.keys()):
msg = "TASK FAILED Received tool that was not registered : " + \
str(toolModel.name)+" not in "+str(list(tools_infos.keys()))
raise Exception(msg)
# Fetch the command to execute
if tools_infos.get(toolModel.name, None) is None:
bin_path = ""
else:
bin_path = tools_infos[toolModel.name].get("bin")
if bin_path is not None:
if not bin_path.endswith(" "):
bin_path = bin_path + " "
comm = bin_path + comm
if comm != "":
try:
# Load the plugin
if parser.strip() == "":
mod = Utils.loadPlugin(tools_infos[toolModel.name]["plugin"])
elif parser.strip() == "auto-detect":
mod = Utils.loadPluginByBin(toolModel.name.split("::")[0])
else:
mod = Utils.loadPlugin(parser)
# Complete command with file output
toolFileName = toolModel.name+"_" + \
str(time.time())+mod.getFileOutputExt()
comm = mod.changeCommand(comm, outputDir, toolFileName)
print(('TASK STARTED:' + toolModel.name))
print("Will timeout at " + str(timeLimit))
# Execute the command with a timeout
returncode = Utils.execute(comm, timeLimit, True)
except Exception as e:
raise e
# Execute found plugin if there is one
if mod is not None:
filepath = mod.getFileOutputPath(comm)
try:
# Open generated file as utf8
with io.open(filepath, "r", encoding="utf-8",
errors='ignore') as file_opened:
# Check return code by plugin (can be always true if the return code is inconsistent)
if mod.checkReturnCode(returncode):
notes, tags, _, _ = mod.Parse(file_opened)
if notes is None:
notes = "No results found by plugin."
if tags is None:
tags = []
if isinstance(tags, str):
tags = [tags]
# Success could be change to False by the plugin function (evaluating the return code for exemple)
# if the success is validated, mark tool as done
toolModel.markAsDone(
os.path.join(outputRelDir,
os.path.basename(filepath)))
# And update the tool in database
toolModel.notes = notes
toolModel.tags = tags
toolModel.update()
# Upload file to SFTP
mod.centralizeFile(filepath, outputDir)
msg = "TASK SUCCESS : " + toolModel.name
else: # BAS RESULT OF PLUGIN
msg = "TASK FAILED (says the mod) : " + toolModel.name
msg += "The return code was not the expected one. (" + str(
returncode) + ")."
toolModel.markAsError()
raise Exception(msg)
except IOError as e:
toolModel.tags = ["todo"]
toolModel.notes = "Failed to read results file"
toolModel.markAsDone()
else:
msg = "TASK FAILED (no plugin found) : " + toolModel.name
toolModel.markAsNotDone()
raise Exception(msg)
# Delay
if command is not None:
if float(command.sleep_between) > 0.0:
msg += " (will sleep for " + \
str(float(command.sleep_between))+")"
print(msg)
time.sleep(float(command.sleep_between))
return