class JobDistributor(Thread): def __init__(self, task): Thread.__init__(self) self.log = Config().getLogger("distributor") self.__maxJobs = config["hostJobs"] self.__maxErrors = config["hostErrors"] self.totalJobs = 0 self.instances = 0 self.__task = task self.__status = States.Pending self.computer_list = self.__getHostfromConfig(self.__maxJobs, self.__maxErrors) self.__processes = {} self.__jobQueue = Queue(100) def getTask(self): return self.__task def terminate(self): self.__status = States.Aborted self.__task.setStatus(States.Aborted) self.__stopAll() def __stopAll(self): self.log.debug("Terminating all jobs, please standby ...") for host in self.__processes: for job in self.__processes[host]: job.terminate() def run(self): commands = self.__task.createCommandList() self.log.debug("Adding commands to queue...") for command in commands: self.__jobQueue.put(command, block=False) self.__task.setStatus(States.Running) while not self.__jobQueue.empty() and self.__status not in [States.Completed, States.Aborted]: self.log.debug("JD.run - jobQ %s and status %s" % (self.__jobQueue.qsize(), self.__status)) self.distribute(self.__jobQueue.get(block=False)) while len(self) != 0: self.log.debug("JD.run end - len is %s" % len(self)) self.__cleanup() time.sleep(config["poll_timeout"]) if not self.__task.getStatus() in [States.Completed, States.Aborted]: self.__task.setStatus(States.Failed) def distribute(self, command): if self.__status == States.Pending: self.__status = States.Running procNum = command.getID() host = None sleepTime = 0 maxSleepTime = config["maxHostWait"] self.log.info("Searching for host for process %i..." % (procNum)) while host == None and self.__status not in [States.Completed, States.Aborted]: self.log.debug("Waiting %i seconds for available host!" % sleepTime) time.sleep(sleepTime) if sleepTime < maxSleepTime: sleepTime += 10 host = self.__getHost() if self.__status not in [States.Completed, States.Aborted]: self.log.info("Host %s chosen for proc %i." % (host.getHostName(), procNum)) self.__processes[host.getHostName()].append(Job(host, command)) self.log.info("Submited to " + host.getHostName() + ": " + command.getCommand()) self.totalJobs += 1 return True return False def __getHost(self): """Find a host among the computer_list whose load is less than maxJobs.""" self.log.info("Finding available host...") self.__cleanup() for host in self.__processes: hostInfo = self.__getHostfromList(host) self.log.debug("Checking host %s" % hostInfo.getHostName()) if hostInfo.getStatus() in [Host.States.Available, Host.States.Running]: return hostInfo return None def __getHostfromConfig(self, maxProcess, maxError): lst = config["hosts"] pcList = [] for host in lst: h = Host(host["name"], host["user"], crypto.decrypt(host["pass"])) h.setMaxProcess(maxProcess) h.setMaxErrors(maxError) self.log.debug("Host %s extracted from config file." % h.getHostName()) pcList.append(h) self.log.debug("Total number of hosts is: %i" % len(pcList)) return pcList def __cleanup(self): processes = {} for host in self.computer_list: if host.checkHost() and host.getStatus() != Host.States.Error: # host is alive self.log.debug("Host %s has status %s" % (host.getHostName(), host.getStatus())) if host.getStatus() in (Host.States.Running, Host.States.Full): # host is working if self.__processes.has_key(host.getHostName()): # current process list contains this host jobs = [] for job in self.__processes[host.getHostName()]: if job.poll(): # process is still working jobs.append(job) else: # process is done if job.getStatus().get_status() == "Cracked": # check for crack code self.log.debug("status is Cracked") self.__status = States.Completed self.__task.setStatus(States.Completed) self.__task.setCode(job.getStatus().get_crackCode()) self.__stopAll() break if job.getStatus().get_command_xcode() != 0: # check for errors self.log.debug("exit code is %s" % job.getStatus().get_command_xcode()) self.__jobQueue.put(job.getStatus().get_command(), block=False) continue if not self.__status == States.Aborted: # remove completed from task self.log.debug("removing job %s from queue" % job.getStatus().get_command().getID()) self.__task.delJobID(job.getStatus().get_command().getID()) processes[host.getHostName()] = jobs else: self.log.error( "Something is very wrong!!! There are hosts with assigned tasks that are not in the current jobs list." ) else: # host is idle, add empty proc list processes[host.getHostName()] = [] else: # host is error, or dead if self.__processes.has_key(host.getHostName()): for job in self.__processes[host.getHostName()]: # add to error queue job.terminate() self.log.debug( "Host %s is dead returning job %s in the queue!" % (host.getHostName(), job.getStatus().get_command()) ) self.__jobQueue.put(job.getStatus().get_command(), block=False) self.__processes = processes self.__task.setProgress(self.__calcTaskProgress(self.__task.getJobCount(), self.__calculateProgress())) def __len__(self): return sum([len(plist) for plist in self.__processes.values()]) def __repr__(self): errors = 0 for host in self.computer_list: if host.getStatus() == Host.States.Error: errors += 1 return (errors, self.__status) def __calcTaskProgress(self, jCount, fraction): answer = 100 - jCount + fraction self.log.debug("progress part is: %f" % answer) return answer def __calculateProgress(self): allProgress = [] for host in self.__processes: for job in self.__processes[host]: allProgress.append(job.getStatus().get_progress()) fraction = 0.00 for i in allProgress: fraction = fraction + math.modf(i)[0] return fraction def resetErrorHost(self): for host in self.computer_list: if host.getStatus() == Host.States.Error: host.resetErrors() def __getHostfromList(self, host): for hostInfo in self.computer_list: if hostInfo.getHostName() == host: return hostInfo
class HashCat(Thread): """Connect to remote host with SSH and execute and control hashcat. This is a facade/wrapper that uses paramiko to spawn and control an SSH client. You must have OpenSSH installed. @ivar host_name: Host name or IP address @ivar user_name: User name @ivar password: Password @ivar prompt: Command prompt (or partial string matching the end of the prompt) @ivar ssh: Instance of a paramiko.SSHClient object """ config = Config().getConfig() def __init__(self, hostInfo,command): """ @param host_name: Host name or IP address @param user_name: User name @param password: Password @param command: The hashcat command that have to be executed @param results: A list that will contain the results once this thread is completed. """ Thread.__init__(self) self.log = Config().getLogger('distributor.'+hostInfo.getHostName(), hostInfo.getHostName()) self.log.debug("Logging has been configured!!!") self.results=results() self.results.set_host(hostInfo) self.results.set_command(command) self.__chan = None self.__ssh = None self.interval = int(HashCat.config["heartbeat_timeout"]) self.be_alive = False self.aborted = False def get_command_xcode(self): return self.results.get_command_xcode() def __str__(self) : #return str(self.__dict__) return str({"host_name":self.results.get_host().getHostName(), "user_name":self.results.get_host().getUserName()}) def __eq__(self, other) : return self.__dict__ == other.__dict__ def abort(self,value): self.aborted = value def isAborted(self): return self.aborted def set_command(self,value): self.results.set_command(value) def run(self): if self.run_command(self.results.get_command()): self.ping() if self.aborted: self.stop_proc() self.results.set_command_xcode(-500) self.quit() def run_command(self, command): """Run a command on the remote host. @param command: Unix command @return: Command output @rtype: String """ self.__ssh=self.results.get_host().getChannel() if self.__ssh == None: return False self.__chan=self.__ssh.invoke_shell() self.__chan.settimeout(15.0) self.__chan.setblocking(1) self.log.debug("sending command: '%s' to host" % command.getCommand()) try: self.__chan.send(command.getCommand()+'\n') except: self.log.exception("Unable to send Command!!! - %s" % command.getCommand()) return False time.sleep(int(HashCat.config["init_timeout"])) self.be_alive = True self.read_proc() return True def ping(self): self.log.debug("Heartbeat is: %s and override is: %s" % (self.be_alive, self.aborted)) while self.be_alive and not self.aborted: self.write_proc("s") self.read_proc() self.log.debug("Sleeping for %d seconds" % self.interval) time.sleep(self.interval) def parse(self, lines): line_arr=lines.splitlines() self.log.debug("Lines array: %s" % line_arr) for line in line_arr: if line.startswith("Status."): self.results.set_status(line.split(":")[1].strip()) for case in switch(self.results.get_status()): if case('Running'): self.be_alive=True break if case('Finished'): self.be_alive=False break if case('Cracked'): self.getCrackCode() self.be_alive=False break if case('Aborted'): self.be_alive=False break if case('Exhausted'): self.be_alive=False break if case('Initializing'): self.be_alive=True break if case(): self.log.warning("Unexpected value: %s" % self.results.get_status()) continue if line.startswith("Progress."): try: prg=float(line[line.find("(")+1:line.find(")")-1]) if prg==float(self.results.get_command().getID())+1.0: prg=0.99 self.results.set_progress(prg) except: self.results.set_progress(-1.0) continue if line.startswith("Time.Running."): self.results.set_elapsed_time(self.parseTime(line.split(":")[1].strip())) continue if line.startswith("Time.Left."): self.results.set_estimated_time(self.parseTime(line.split(":")[1].strip())) continue if [True for i in ["$ ","$ s","# ","# s","ss"] if line.endswith(i)]: self.results.set_last_output(line_arr) self.be_alive=False self.evaluate_xcode() if not self.results.get_command_xcode() in [0,1]: self.results.set_status("Error") continue #self.log.debug("Line cannot be recognized: %s" % line) def parseTime(self,timeString): days=0 hours=0 minutes=0 seconds=0 time_arr=timeString.split(",") for timeLine in time_arr: if timeLine.strip().split(" ")[1].strip() in ["day","days"]: days=int(timeLine.strip().split(" ")[0].strip()) if timeLine.strip().split(" ")[1].strip() in ["hour","hours"]: hours=int(timeLine.strip().split(" ")[0].strip()) if timeLine.strip().split(" ")[1].strip() in ["min","mins"]: minutes=int(timeLine.strip().split(" ")[0].strip()) if timeLine.strip().split(" ")[1].strip() in ["sec","secs"]: seconds=int(timeLine.strip().split(" ")[0].strip()) return "%i:%i:%i" %(24*days+hours,minutes,seconds) def evaluate_xcode(self): lines='' command="echo $?" self.write_proc("\b"*10) self.log.debug("sending command: '%s' to host" % command) try: self.__chan.send(command+'\n') except: self.log.error("Unable to send command!!! - %s" % command) while not [True for i in ["=> ","$ ","$ s","# ","# s","ss"] if lines.endswith(i)]: try: for x in range(1,16): time.sleep(1) if self.__chan.recv_ready(): line=self.__chan.recv(9999) lines=lines+''.join(line) break if x==15: raise Exception() except: self.log.error("Cannot obtain exit code line!!!") self.log.debug("Exit Code Line: %s" % lines) line_arr=lines.splitlines() try: self.results.set_command_xcode(int(line_arr[1])) except: self.log.exception("Line: %s - does not contain an integer value!!!" % line_arr[1]) self.log.debug("Exit Code is: %d" % self.results.get_command_xcode()) def stop_proc(self): self.log.info("Sending stop command to process...") self.be_alive = False self.write_proc("q") self.read_proc() def read_proc(self): lines='' while not [True for i in ["=> ","$ ","$ s","# ","# s","ss"] if lines.endswith(i)]: try: self.log.debug("Channel receive status: %s" % self.__chan.recv_ready()) for x in range(1,16): time.sleep(1) if self.__chan.recv_ready(): line=self.__chan.recv(9999) lines=lines+''.join(line) break if x==15: raise Exception() except: self.log.exception("Stream not ready!!!") self.be_alive=False return self.parse(lines) def write_proc(self, message): try: self.log.debug("Sending message: %s through the channel..." % message) self.__chan.send(message) return True except: self.log.exception("Sending message %s failed!!!" % message) return False def quit(self): self.log.debug("Quitting thread on host %s ..."% self.results.get_host().getHostName()) if self.__ssh!= None: self.results.get_host().closeChannel(self.__ssh) def getCrackCode(self): #read code from file cmdline=self.results.get_command().getCommand().split(" ") for cmd in cmdline: if cmd.startswith('--outfile'): filename=cmd.split("=")[1] if filename != None: tmp=self.results.get_host().getFile(filename) crackCode=tmp[0][:-1].split(':')[2] if crackCode != None: self.results.set_crackCode(crackCode) def get_result(self): return self.results
class Host(object): ''' classdocs ''' class States(Enum): NotAvailable=0 Running=1 Available=2 Down=3 Error=4 Full=5 def __init__(self, host, user, password, port = 22): self.__status = Host.States.NotAvailable self.__errors = 0 self.__maxErrors=3 self.__processes = 0 self.__maxProcess = 0 self.__hostName = host self.__userName = user self.__password = password self.__port = port self.log = Config().getLogger('distributor.'+host,host) self.__sched = Scheduler() self.__sched.start() def checkHost(self): self.log.debug("Start checking of host...") s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) s.settimeout(5) try: self.log.debug("Making connection to remote host: %s on port: %i with user: %s " % (self.__hostName,self.__port, self.__userName)) s.connect((self.__hostName, int(self.__port))) self.log.debug("Closing connection...") s.shutdown(2) self.log.debug("Host is alive and running!") if self.__status==Host.States.NotAvailable: self.log.debug("Setting host as available!") self.__status=Host.States.Available return True except: self.log.error("A connection to the host cannot be established!!!") self.__status=Host.States.Down timechange=datetime.now()+relativedelta(seconds=300) job = self.__sched.add_date_job(self.__resetDown, timechange) return False def __resetDown(self): self.__status=Host.States.NotAvailable self.__processes = 0 self.__errors = 0 def __getSSH(self): ssh = paramiko.SSHClient() ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy()) return ssh def getChannel(self): ssh=self.__getSSH() try: self.log.debug("Making connection to remote host: %s with user: %s"%(self.__hostName, self.__userName)) ssh.connect(self.__hostName,self.__port, self.__userName, self.__password, timeout=5) except: self.log.exception("Connection cannot be established!!!") return None self.log.debug("Connection Established!") return ssh def getFile(self,fileName): ssh=self.getChannel() sftp=ssh.open_sftp() try: fl=sftp.open(fileName) string=fl.readlines() return string except IOError: self.log.debug("No such file!!!") return None def closeChannel(self,ssh): """ Close the connection to the remote host. """ self.log.info("Closing connection to host %s" % self.__hostName) ssh.close() self.log.debug("Connection closed!") def getHostName(self): return self.__hostName def getUserName(self): return self.__userName def getPassword(self): return self.__password def getPort(self): return self.__port def setMaxProcess(self,value): self.__maxProcess=value def setMaxErrors(self,value): self.__maxErrors=value def getStatus(self): return self.__status def addError(self): self.log.debug("Adding error to host") if self.__errors<self.__maxErrors: self.log.debug("Error added to host!") self.__errors+=1 return self.__stateSystem() return False def addProcess(self): self.log.debug("Adding process to host") if self.__processes<self.__maxProcess: self.log.debug("Process added to host!") self.__processes+=1 return self.__stateSystem() return False def delProcess(self): self.log.debug("Removing process from host") if self.__processes>0: self.log.debug("Process removed from host!") self.__processes-=1 return self.__stateSystem() return False def __stateSystem(self): if not self.__status in [Host.States.NotAvailable, Host.States.Down]: if self.__processes>=self.__maxProcess: self.__status=Host.States.Full if self.__processes<=0: self.__status=Host.States.Available if 0<self.__processes<self.__maxProcess: self.__status=Host.States.Running if self.__errors>=self.__maxErrors: self.__status=Host.States.Error self.log.debug("Status of host %s is: %s"% (self.__hostName,self.__status)) return True else: self.log.debug("Status of host %s is: %s"% (self.__hostName,self.__status)) return False def resetErrors(self): self.log.debug("Errors reset for host: %s !!!"%self.__hostName) self.__errors=0 self.__stateSystem()