class JobDistributor(Thread):
def __init__(self, task):
Thread.__init__(self)
self.log = Config().getLogger("distributor")
self.__maxJobs = config["hostJobs"]
self.__maxErrors = config["hostErrors"]
self.totalJobs = 0
self.instances = 0
self.__task = task
self.__status = States.Pending
self.computer_list = self.__getHostfromConfig(self.__maxJobs, self.__maxErrors)
self.__processes = {}
self.__jobQueue = Queue(100)
def getTask(self):
return self.__task
def terminate(self):
self.__status = States.Aborted
self.__task.setStatus(States.Aborted)
self.__stopAll()
def __stopAll(self):
self.log.debug("Terminating all jobs, please standby ...")
for host in self.__processes:
for job in self.__processes[host]:
job.terminate()
def run(self):
commands = self.__task.createCommandList()
self.log.debug("Adding commands to queue...")
for command in commands:
self.__jobQueue.put(command, block=False)
self.__task.setStatus(States.Running)
while not self.__jobQueue.empty() and self.__status not in [States.Completed, States.Aborted]:
self.log.debug("JD.run - jobQ %s and status %s" % (self.__jobQueue.qsize(), self.__status))
self.distribute(self.__jobQueue.get(block=False))
while len(self) != 0:
self.log.debug("JD.run end - len is %s" % len(self))
self.__cleanup()
time.sleep(config["poll_timeout"])
if not self.__task.getStatus() in [States.Completed, States.Aborted]:
self.__task.setStatus(States.Failed)
def distribute(self, command):
if self.__status == States.Pending:
self.__status = States.Running
procNum = command.getID()
host = None
sleepTime = 0
maxSleepTime = config["maxHostWait"]
self.log.info("Searching for host for process %i..." % (procNum))
while host == None and self.__status not in [States.Completed, States.Aborted]:
self.log.debug("Waiting %i seconds for available host!" % sleepTime)
time.sleep(sleepTime)
if sleepTime < maxSleepTime:
sleepTime += 10
host = self.__getHost()
if self.__status not in [States.Completed, States.Aborted]:
self.log.info("Host %s chosen for proc %i." % (host.getHostName(), procNum))
self.__processes[host.getHostName()].append(Job(host, command))
self.log.info("Submited to " + host.getHostName() + ": " + command.getCommand())
self.totalJobs += 1
return True
return False
def __getHost(self):
"""Find a host among the computer_list whose load is less than maxJobs."""
self.log.info("Finding available host...")
self.__cleanup()
for host in self.__processes:
hostInfo = self.__getHostfromList(host)
self.log.debug("Checking host %s" % hostInfo.getHostName())
if hostInfo.getStatus() in [Host.States.Available, Host.States.Running]:
return hostInfo
return None
def __getHostfromConfig(self, maxProcess, maxError):
lst = config["hosts"]
pcList = []
for host in lst:
h = Host(host["name"], host["user"], crypto.decrypt(host["pass"]))
h.setMaxProcess(maxProcess)
h.setMaxErrors(maxError)
self.log.debug("Host %s extracted from config file." % h.getHostName())
pcList.append(h)
self.log.debug("Total number of hosts is: %i" % len(pcList))
return pcList
def __cleanup(self):
processes = {}
for host in self.computer_list:
if host.checkHost() and host.getStatus() != Host.States.Error: # host is alive
self.log.debug("Host %s has status %s" % (host.getHostName(), host.getStatus()))
if host.getStatus() in (Host.States.Running, Host.States.Full): # host is working
if self.__processes.has_key(host.getHostName()): # current process list contains this host
jobs = []
for job in self.__processes[host.getHostName()]:
if job.poll(): # process is still working
jobs.append(job)
else: # process is done
if job.getStatus().get_status() == "Cracked": # check for crack code
self.log.debug("status is Cracked")
self.__status = States.Completed
self.__task.setStatus(States.Completed)
self.__task.setCode(job.getStatus().get_crackCode())
self.__stopAll()
break
if job.getStatus().get_command_xcode() != 0: # check for errors
self.log.debug("exit code is %s" % job.getStatus().get_command_xcode())
self.__jobQueue.put(job.getStatus().get_command(), block=False)
continue
if not self.__status == States.Aborted: # remove completed from task
self.log.debug("removing job %s from queue" % job.getStatus().get_command().getID())
self.__task.delJobID(job.getStatus().get_command().getID())
processes[host.getHostName()] = jobs
else:
self.log.error(
"Something is very wrong!!! There are hosts with assigned tasks that are not in the current jobs list."
)
else: # host is idle, add empty proc list
processes[host.getHostName()] = []
else: # host is error, or dead
if self.__processes.has_key(host.getHostName()):
for job in self.__processes[host.getHostName()]:
# add to error queue
job.terminate()
self.log.debug(
"Host %s is dead returning job %s in the queue!"
% (host.getHostName(), job.getStatus().get_command())
)
self.__jobQueue.put(job.getStatus().get_command(), block=False)
self.__processes = processes
self.__task.setProgress(self.__calcTaskProgress(self.__task.getJobCount(), self.__calculateProgress()))
def __len__(self):
return sum([len(plist) for plist in self.__processes.values()])
def __repr__(self):
errors = 0
for host in self.computer_list:
if host.getStatus() == Host.States.Error:
errors += 1
return (errors, self.__status)
def __calcTaskProgress(self, jCount, fraction):
answer = 100 - jCount + fraction
self.log.debug("progress part is: %f" % answer)
return answer
def __calculateProgress(self):
allProgress = []
for host in self.__processes:
for job in self.__processes[host]:
allProgress.append(job.getStatus().get_progress())
fraction = 0.00
for i in allProgress:
fraction = fraction + math.modf(i)[0]
return fraction
def resetErrorHost(self):
for host in self.computer_list:
if host.getStatus() == Host.States.Error:
host.resetErrors()
def __getHostfromList(self, host):
for hostInfo in self.computer_list:
if hostInfo.getHostName() == host:
return hostInfo
class HashCat(Thread):
"""Connect to remote host with SSH and execute and control hashcat.
This is a facade/wrapper that uses paramiko to spawn and control an SSH client.
You must have OpenSSH installed.
@ivar host_name: Host name or IP address
@ivar user_name: User name
@ivar password: Password
@ivar prompt: Command prompt (or partial string matching the end of the prompt)
@ivar ssh: Instance of a paramiko.SSHClient object
"""
config = Config().getConfig()
def __init__(self, hostInfo,command):
"""
@param host_name: Host name or IP address
@param user_name: User name
@param password: Password
@param command: The hashcat command that have to be executed
@param results: A list that will contain the results once this thread is completed.
"""
Thread.__init__(self)
self.log = Config().getLogger('distributor.'+hostInfo.getHostName(), hostInfo.getHostName())
self.log.debug("Logging has been configured!!!")
self.results=results()
self.results.set_host(hostInfo)
self.results.set_command(command)
self.__chan = None
self.__ssh = None
self.interval = int(HashCat.config["heartbeat_timeout"])
self.be_alive = False
self.aborted = False
def get_command_xcode(self):
return self.results.get_command_xcode()
def __str__(self) :
#return str(self.__dict__)
return str({"host_name":self.results.get_host().getHostName(), "user_name":self.results.get_host().getUserName()})
def __eq__(self, other) :
return self.__dict__ == other.__dict__
def abort(self,value):
self.aborted = value
def isAborted(self):
return self.aborted
def set_command(self,value):
self.results.set_command(value)
def run(self):
if self.run_command(self.results.get_command()):
self.ping()
if self.aborted:
self.stop_proc()
self.results.set_command_xcode(-500)
self.quit()
def run_command(self, command):
"""Run a command on the remote host.
@param command: Unix command
@return: Command output
@rtype: String
"""
self.__ssh=self.results.get_host().getChannel()
if self.__ssh == None:
return False
self.__chan=self.__ssh.invoke_shell()
self.__chan.settimeout(15.0)
self.__chan.setblocking(1)
self.log.debug("sending command: '%s' to host" % command.getCommand())
try:
self.__chan.send(command.getCommand()+'\n')
except:
self.log.exception("Unable to send Command!!! - %s" % command.getCommand())
return False
time.sleep(int(HashCat.config["init_timeout"]))
self.be_alive = True
self.read_proc()
return True
def ping(self):
self.log.debug("Heartbeat is: %s and override is: %s" % (self.be_alive, self.aborted))
while self.be_alive and not self.aborted:
self.write_proc("s")
self.read_proc()
self.log.debug("Sleeping for %d seconds" % self.interval)
time.sleep(self.interval)
def parse(self, lines):
line_arr=lines.splitlines()
self.log.debug("Lines array: %s" % line_arr)
for line in line_arr:
if line.startswith("Status."):
self.results.set_status(line.split(":")[1].strip())
for case in switch(self.results.get_status()):
if case('Running'):
self.be_alive=True
break
if case('Finished'):
self.be_alive=False
break
if case('Cracked'):
self.getCrackCode()
self.be_alive=False
break
if case('Aborted'):
self.be_alive=False
break
if case('Exhausted'):
self.be_alive=False
break
if case('Initializing'):
self.be_alive=True
break
if case():
self.log.warning("Unexpected value: %s" % self.results.get_status())
continue
if line.startswith("Progress."):
try:
prg=float(line[line.find("(")+1:line.find(")")-1])
if prg==float(self.results.get_command().getID())+1.0:
prg=0.99
self.results.set_progress(prg)
except:
self.results.set_progress(-1.0)
continue
if line.startswith("Time.Running."):
self.results.set_elapsed_time(self.parseTime(line.split(":")[1].strip()))
continue
if line.startswith("Time.Left."):
self.results.set_estimated_time(self.parseTime(line.split(":")[1].strip()))
continue
if [True for i in ["$ ","$ s","# ","# s","ss"] if line.endswith(i)]:
self.results.set_last_output(line_arr)
self.be_alive=False
self.evaluate_xcode()
if not self.results.get_command_xcode() in [0,1]:
self.results.set_status("Error")
continue
#self.log.debug("Line cannot be recognized: %s" % line)
def parseTime(self,timeString):
days=0
hours=0
minutes=0
seconds=0
time_arr=timeString.split(",")
for timeLine in time_arr:
if timeLine.strip().split(" ")[1].strip() in ["day","days"]:
days=int(timeLine.strip().split(" ")[0].strip())
if timeLine.strip().split(" ")[1].strip() in ["hour","hours"]:
hours=int(timeLine.strip().split(" ")[0].strip())
if timeLine.strip().split(" ")[1].strip() in ["min","mins"]:
minutes=int(timeLine.strip().split(" ")[0].strip())
if timeLine.strip().split(" ")[1].strip() in ["sec","secs"]:
seconds=int(timeLine.strip().split(" ")[0].strip())
return "%i:%i:%i" %(24*days+hours,minutes,seconds)
def evaluate_xcode(self):
lines=''
command="echo $?"
self.write_proc("\b"*10)
self.log.debug("sending command: '%s' to host" % command)
try:
self.__chan.send(command+'\n')
except:
self.log.error("Unable to send command!!! - %s" % command)
while not [True for i in ["=> ","$ ","$ s","# ","# s","ss"] if lines.endswith(i)]:
try:
for x in range(1,16):
time.sleep(1)
if self.__chan.recv_ready():
line=self.__chan.recv(9999)
lines=lines+''.join(line)
break
if x==15: raise Exception()
except:
self.log.error("Cannot obtain exit code line!!!")
self.log.debug("Exit Code Line: %s" % lines)
line_arr=lines.splitlines()
try:
self.results.set_command_xcode(int(line_arr[1]))
except:
self.log.exception("Line: %s - does not contain an integer value!!!" % line_arr[1])
self.log.debug("Exit Code is: %d" % self.results.get_command_xcode())
def stop_proc(self):
self.log.info("Sending stop command to process...")
self.be_alive = False
self.write_proc("q")
self.read_proc()
def read_proc(self):
lines=''
while not [True for i in ["=> ","$ ","$ s","# ","# s","ss"] if lines.endswith(i)]:
try:
self.log.debug("Channel receive status: %s" % self.__chan.recv_ready())
for x in range(1,16):
time.sleep(1)
if self.__chan.recv_ready():
line=self.__chan.recv(9999)
lines=lines+''.join(line)
break
if x==15: raise Exception()
except:
self.log.exception("Stream not ready!!!")
self.be_alive=False
return
self.parse(lines)
def write_proc(self, message):
try:
self.log.debug("Sending message: %s through the channel..." % message)
self.__chan.send(message)
return True
except:
self.log.exception("Sending message %s failed!!!" % message)
return False
def quit(self):
self.log.debug("Quitting thread on host %s ..."% self.results.get_host().getHostName())
if self.__ssh!= None:
self.results.get_host().closeChannel(self.__ssh)
def getCrackCode(self):
#read code from file
cmdline=self.results.get_command().getCommand().split(" ")
for cmd in cmdline:
if cmd.startswith('--outfile'):
filename=cmd.split("=")[1]
if filename != None:
tmp=self.results.get_host().getFile(filename)
crackCode=tmp[0][:-1].split(':')[2]
if crackCode != None:
self.results.set_crackCode(crackCode)
def get_result(self):
return self.results
class Host(object):
'''
classdocs
'''
class States(Enum):
NotAvailable=0
Running=1
Available=2
Down=3
Error=4
Full=5
def __init__(self, host, user, password, port = 22):
self.__status = Host.States.NotAvailable
self.__errors = 0
self.__maxErrors=3
self.__processes = 0
self.__maxProcess = 0
self.__hostName = host
self.__userName = user
self.__password = password
self.__port = port
self.log = Config().getLogger('distributor.'+host,host)
self.__sched = Scheduler()
self.__sched.start()
def checkHost(self):
self.log.debug("Start checking of host...")
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.settimeout(5)
try:
self.log.debug("Making connection to remote host: %s on port: %i with user: %s " % (self.__hostName,self.__port, self.__userName))
s.connect((self.__hostName, int(self.__port)))
self.log.debug("Closing connection...")
s.shutdown(2)
self.log.debug("Host is alive and running!")
if self.__status==Host.States.NotAvailable:
self.log.debug("Setting host as available!")
self.__status=Host.States.Available
return True
except:
self.log.error("A connection to the host cannot be established!!!")
self.__status=Host.States.Down
timechange=datetime.now()+relativedelta(seconds=300)
job = self.__sched.add_date_job(self.__resetDown, timechange)
return False
def __resetDown(self):
self.__status=Host.States.NotAvailable
self.__processes = 0
self.__errors = 0
def __getSSH(self):
ssh = paramiko.SSHClient()
ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
return ssh
def getChannel(self):
ssh=self.__getSSH()
try:
self.log.debug("Making connection to remote host: %s with user: %s"%(self.__hostName, self.__userName))
ssh.connect(self.__hostName,self.__port, self.__userName, self.__password, timeout=5)
except:
self.log.exception("Connection cannot be established!!!")
return None
self.log.debug("Connection Established!")
return ssh
def getFile(self,fileName):
ssh=self.getChannel()
sftp=ssh.open_sftp()
try:
fl=sftp.open(fileName)
string=fl.readlines()
return string
except IOError:
self.log.debug("No such file!!!")
return None
def closeChannel(self,ssh):
"""
Close the connection to the remote host.
"""
self.log.info("Closing connection to host %s" % self.__hostName)
ssh.close()
self.log.debug("Connection closed!")
def getHostName(self):
return self.__hostName
def getUserName(self):
return self.__userName
def getPassword(self):
return self.__password
def getPort(self):
return self.__port
def setMaxProcess(self,value):
self.__maxProcess=value
def setMaxErrors(self,value):
self.__maxErrors=value
def getStatus(self):
return self.__status
def addError(self):
self.log.debug("Adding error to host")
if self.__errors<self.__maxErrors:
self.log.debug("Error added to host!")
self.__errors+=1
return self.__stateSystem()
return False
def addProcess(self):
self.log.debug("Adding process to host")
if self.__processes<self.__maxProcess:
self.log.debug("Process added to host!")
self.__processes+=1
return self.__stateSystem()
return False
def delProcess(self):
self.log.debug("Removing process from host")
if self.__processes>0:
self.log.debug("Process removed from host!")
self.__processes-=1
return self.__stateSystem()
return False
def __stateSystem(self):
if not self.__status in [Host.States.NotAvailable, Host.States.Down]:
if self.__processes>=self.__maxProcess:
self.__status=Host.States.Full
if self.__processes<=0:
self.__status=Host.States.Available
if 0<self.__processes<self.__maxProcess:
self.__status=Host.States.Running
if self.__errors>=self.__maxErrors:
self.__status=Host.States.Error
self.log.debug("Status of host %s is: %s"% (self.__hostName,self.__status))
return True
else:
self.log.debug("Status of host %s is: %s"% (self.__hostName,self.__status))
return False
def resetErrors(self):
self.log.debug("Errors reset for host: %s !!!"%self.__hostName)
self.__errors=0
self.__stateSystem()
