def foo(iterations): for i in range(iterations): print threading.currentThread().name, print threading.currentThread().ident, print threading.activeCount(), print threading.enumerate() time.sleep(0.2)
def testA(self): """ _testA_ Handle AddWorkflowToManage events """ myThread = threading.currentThread() config = self.getConfig() testWorkflowManager = WorkflowManager(config) testWorkflowManager.prepareToStart() for i in xrange(0, WorkflowManagerTest._maxMessage): workflow = Workflow(spec = "testSpec.xml", owner = "riahi", \ name = "testWorkflow" + str(i), task = "testTask") workflow.create() for j in xrange(0, 3): workflowManagerdict = {'payload':{'WorkflowId' : workflow.id \ , 'FilesetMatch': 'FILESET_' + str(j) ,'SplitAlgo':'NO SPLITALGO', 'Type':'NO TYPE' }} testWorkflowManager.handleMessage( \ type = 'AddWorkflowToManage' , payload = workflowManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
def scrape(self): self.threads=[] self.LList=[] tic=time.time() response=requests.get(self.artist_URL, headers={'User-Agent': 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_9_4) \ AppleWebKit/537.36 (KHTML, like Gecko) Chrome/35.0.1916.153 Safari/537.36'}) soup=BeautifulSoup(response.text,'lxml') header=soup.select(".results_header") numSongs=header[0].text.split()[0] if numSongs.strip()=="Results": return [] if int(numSongs) > 20000: return [] #Genius.com lists 20 songs per page numPages=int(numSongs)/20 self.pages=numPages for page in range(1,numPages): URL=self.BASE_QUERY+str(page)+"&q="+self.artist thread=pageThread(page,URL,self.artistString,self.LList) thread.start() self.threads.append(thread) #Wait for all threads to finish collecting lyrics before=0 while threading.activeCount()>3: if threading.activeCount()!=before: print "active count is {0}".format(threading.activeCount()) before=threading.activeCount() for t in self.threads: t.join() toc=time.time() print toc-tic,'s' return self.LList
def run(self): try: global threads global max_threads global threads_spawned_count while running == True: if len(threads) < max_threads: try: debugMsg(str(self)+" INITIATOR Spawning new thread since len(threads) is less than " + str(max_threads)) current = someThreadFunction(threads_spawned_count) threads.append([threads_spawned_count,current]) current.start() del current threads_spawned_count = threads_spawned_count + 1 debugMsg(str(self)+" INITIATOR Running " + str(len(threads)) + " threads so far") print "INITIATOR Active Threads " + str(threading.activeCount()) + " including thread timeout timers" except: debugMsg("Unable to spawn thread, probably os limit") time.sleep(1) else: debugMsg(str(self)+" INITIATOR Waiting for a thread to timeout / die which will reduce threads_count") time.sleep(randrange(2,5,1)) debugMsg(str(self)+" INITIATOR Running Threads " + str(threads)) print "INITIATIOR " + str(len(threads)) + " threads in stack" print "INITIATIOR Active Threads:" + str(threading.activeCount()) + " including thread timeout timers" time.sleep(1) except Exception, e: debugMsg("WARNING: a initiator has died") debugMsg(str(e)) global initiator_count initiator_count = initiator_count - 1
def main(): global counter parser=OptionParser() parser.add_option("-n", dest="nhost", type="int",\ help="Number of hosts", metavar="nHost") parser.add_option("-o", "--output", dest="oFile", type="string",\ help="File to save logs", metavar="FILE") parser.add_option("-v", "--verbose", dest="verbose", default=False,\ action="store_true",\ help="Logs everything") parser.add_option("-t", "--timeout", dest="timeout", type="float",\ help="Timeout in seconds") parser.add_option("-m", "--maxthread", dest="max", type="int",\ help="Maximum thread number") (options, args)= parser.parse_args() if options.nhost==None or options.nhost<1 or options.max<1: parser.print_help() os._exit(1) else: counter=options.nhost log="[#]=========================================================================[#]\n" log+=" # Host # Banner # Login #\n" log+="[#]=========================================================================[#]" if options.oFile!=None: logFile=open(options.oFile, "a") ctime=str(datetime.datetime.now()) # returns "yy-mm-dd hh:mm:ss.xx ctime=ctime[:ctime.index(".")] logFile.write("\nScan time: %s\n"%ctime) logFile.write(log+"\n") else: logFile=None verbose=options.verbose tmax=options.max nthreads=threading.activeCount() # get initial number of running threads socket.setdefaulttimeout(options.timeout) print(log) while counter>0: address=randomHost() try: while threading.activeCount()>tmax: time.sleep(10) t=threading.Thread(target=AnonLogin, args=(address, logFile, verbose)) t.start() except: pass while threading.activeCount()>nthreads: # if number of running threads less than initial number time.sleep(10) # then, there are few threads still running log="[#]=========================================================================[#]" print(log) if logFile: logFile.write(log+"\n") logFile.close()
def __init__(self): self.for_upload = [] self.url_stats = {} self.tempdir = 'tmp' self.current_date = datetime.datetime.today().strftime("%Y-%m-%d") self.create_temp_dir() self.get_image_data() for chunk in self.chunks(glob.glob1(self.tempdir, "*.jpg"), 50): worker = Thread(target=self.create_thumbnail, args=(chunk,)) worker.setDaemon(True) worker.start() while (activeCount() > 1): time.sleep(5) s3key = 'AKIAIYZERMTB6Z5NPF5Q' s3secret = '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' bucket_name = "hrachya-test" self.s3_conn = boto.connect_s3(s3key, s3secret) self.bucket_obj = self.s3_conn.get_bucket(bucket_name) for chunk in self.chunks(glob.glob1(self.tempdir, "*.jpg"), 100): worker = Thread(target=self.aws_s3_uploader, args=(chunk,)) worker.setDaemon(True) worker.start() while (activeCount() > 1): time.sleep(5) #self.aws_s3_uploader() self.update_record() self.cleaner()
def go(self, usercount): chats = [] db = _getDB() conn = db.open() app = conn.root()['Application'] chatservice = getattr(app, 'chatservice') for i in range(usercount): user = '******'%i buddy = user + 'buddy' chatservice.register(user, user) chatservice.register(buddy, buddy) transaction.commit() thread = ChatThread(db, user, buddy) chats.append(thread) thread = ChatThread(db, buddy, user) chats.append(thread) for thread in chats: thread.start() time.sleep(0.1) active = threading.activeCount() while active > 0: active = threading.activeCount()-1 print 'waiting for %s threads' % active print "chats: ", numActive(chats), time.sleep(5)
def testMultiThreading( tries ): import random DIRAC.gLogger.info( 'Testing MySQL MultiThreading' ) DIRAC.gLogger.info( 'First adding 10 K records' ) if not DB.checktable()['OK']: return DIRAC.S_ERROR() if not DB.filltable( 10000 )['OK']: return DIRAC.S_ERROR() i = 0 # overthread = 0 DIRAC.gLogger.info( 'Now querying 100 K in MultiThread mode' ) while i < tries: if not i % 1000: DIRAC.gLogger.info( 'Query:', i ) overthread = 0 i += 1 id = int( random.uniform( 0, 10000 ) ) + 1 t = threading.Thread( target = testRetrieve, args = ( id, ) ) semaphore.acquire() t.start() n = threading.activeCount() while n > 1: DIRAC.gLogger.info( 'Waiting for Treads to end:', n ) n = threading.activeCount() time.sleep( 0.1 ) DIRAC.gLogger.info( 'Total retrieved values', Success ) DIRAC.gLogger.info( 'Total Errors', Error ) return DIRAC.S_OK( ( Success, Error ) )
def test_handle_one_request_closes_connection(self): # By default, BaseHTTPServer.py treats all HTTP 1.1 requests as keep-alive. # Intentionally use HTTP 1.0 to prevent this behavior. response = httparchive.ArchivedHttpResponse( version=10, status=200, reason="OK", headers=[], response_data=["bat1"] ) self.set_up_proxy_server(response) t = threading.Thread(target=HttpProxyTest.serve_requests_forever, args=(self,)) t.start() initial_thread_count = threading.activeCount() # Make a bunch of requests. request_count = 10 for _ in range(request_count): conn = httplib.HTTPConnection("localhost", 8889, timeout=10) conn.request("GET", "/index.html") res = conn.getresponse().read() self.assertEqual(res, "bat1") conn.close() # Check to make sure that there is no leaked thread. util.WaitFor(lambda: threading.activeCount() == initial_thread_count, 2) self.assertEqual(request_count, HttpProxyTest.HANDLED_REQUEST_COUNT)
def testA(self): """ _testA_ Handle AddDatasetWatch events """ myThread = threading.currentThread() config = self.getConfig() testFeederManager = FeederManager(config) testFeederManager.prepareToStart() for i in xrange(0, FeederManagerTest._maxMessage): for j in xrange(0, 3): feederManagerdict = {'payload':{'FeederType':'NO Feeder', 'dataset' : 'NO DATASET', 'FileType' : 'NO FILE TYPE', 'StartRun' : 'NO START RUN' }} testFeederManager.handleMessage( type = 'AddDatasetWatch', payload = feederManagerdict ) time.sleep(30) myThread.workerThreadManager.terminateWorkers() while threading.activeCount() > 1: print('Currently: '+str(threading.activeCount())+\ ' Threads. Wait until all our threads have finished') time.sleep(1)
def start(self): """ Start the checker to watch every node in every service :return: """ self.manager.start_working() while self.is_working: self.now_time = time.time() # Check every self.interval = 60 seconds if self.now_time - self.last_check_time >= self.interval: # If last check cost too much time, that means the performance is too low :-( if self.now_time - self.last_check_time >= 90: checker_log.fatal('last check time cost time too long:%s' % (self.now_time - self.last_check_time)) self.last_check_time = self.now_time else: self.last_check_time += 60 checker_log.info('Now Sina Alert Checker Main Start...') # Init each service by service_init function for service in self.watch_service_class_dict.keys(): len_nodes = self.services_init(service) if not len_nodes: checker_log.info('service: [%s] check occur error!' % service) continue checker_log.info('start check service [%s] now... nodes count [%s] ' % (service, len_nodes)) # Wait all check nodes init function finished... self.manager.wait_complete() print 'checking service %s now process status: threads_num:%s cost_time:%s' % \ (self.watch_service_class_dict, threading.activeCount(), (time.time() - self.now_time)) checker_log.info('checking service %s now process status: threads_num:%s cost_time:%s' % (self.watch_service_class_dict, threading.activeCount(), (time.time() - self.now_time))) time.sleep(0.1)
def check(self, payload, agent_config, collection_time, emit_time, cpu_time=None): if threading.activeCount() > MAX_THREADS_COUNT: self.save_sample("datadog.agent.collector.threads.count", threading.activeCount()) self.logger.info("Thread count is high: %d" % threading.activeCount()) if collection_time > MAX_COLLECTION_TIME: self.save_sample("datadog.agent.collector.collection.time", collection_time) self.logger.info( "Collection time (s) is high: %.1f, metrics count: %d, events count: %d" % (collection_time, len(payload["metrics"]), len(payload["events"])) ) if emit_time is not None and emit_time > MAX_EMIT_TIME: self.save_sample("datadog.agent.emitter.emit.time", emit_time) self.logger.info( "Emit time (s) is high: %.1f, metrics count: %d, events count: %d" % (emit_time, len(payload["metrics"]), len(payload["events"])) ) if cpu_time is not None: try: cpu_used_pct = 100.0 * float(cpu_time) / float(collection_time) if cpu_used_pct > MAX_CPU_PCT: self.save_sample("datadog.agent.collector.cpu.used", cpu_used_pct) self.logger.info( "CPU consumed (%%) is high: %.1f, metrics count: %d, events count: %d" % (cpu_used_pct, len(payload["metrics"]), len(payload["events"])) ) except Exception, e: self.logger.debug( "Couldn't compute cpu used by collector with values %s %s %s" % (cpu_time, collection_time, str(e)) )
def list_ping(_set): if ip_str is None or ip_str == '': for _ in _set: for i in range(_[1], _[2] + 1): ping_thread = Ping('%s.%d' % (_[0], i)) ping_thread.start() else: ip = ip_str.split("|") for _ in ip: ping_thread = Ping(_) ping_thread.start() # once run threading.activeCount()=2 !!!maybe itself print threading.activeCount() - init_threading_count, 'threading working...' while threading.activeCount() > init_threading_count: pass ip_list.sort() # I want print by fast!!! this doesn't work. print '%-6s%-4s%-16s%s' % ('loss%', 'avg', 'IP Address', 'dNSName') for ip in ip_list: print '%-6s%-4s%-16s%s' % ip print "you may love this." _ip = [] for _ in ip_list: if int(_[0]) == 0: _ip.append(_[2]) print "|".join(_ip)
def main(): """Main function""" lock = threading.Lock() thread1 = Foo(lock, iterations=10) thread2 = Foo(lock, iterations=15) thread1.start() thread2.start() # Let the main thread do something too... for i in range(5): lock.acquire() print threading.currentThread().name, print threading.currentThread().ident, print threading.activeCount(), print threading.enumerate() lock.release() time.sleep(0.2) # Main thread waits for all threads to complete thread1.join() thread2.join() print "Exiting Main Thread"
def prepareToStop(self, wait=False, stopPayload=""): """ _stopComponent Stops the component, including all worker threads. Allows call from test framework """ # Stop all worker threads logging.info(">>>Terminating worker threads") myThread = threading.currentThread() try: myThread.workerThreadManager.terminateWorkers() except: # We may not have a thread manager pass if wait: logging.info(">>>Shut down of component " + "while waiting for threads to finish") # check if nr of threads is specified. activeThreads = 1 if stopPayload != "": activeThreads = int(stopPayload) if activeThreads < 1: activeThreads = 1 while threading.activeCount() > activeThreads: logging.info(">>>Currently " + str(threading.activeCount()) + " threads active") logging.info(">>>Waiting for less then " + str(activeThreads) + " to be active") time.sleep(5)
def populate_Entities(input_filename,output_filename): global entities global ids global docs global Tokens for line in file(input_filename).readlines(): combo=line.split('\t') ids.append(combo[0]) docs.append(combo[1]) # File=open(output_filename,'w+') threads = [] for key,doc in enumerate(docs): t = threading.Thread(target=worker, args=(key,doc)) threads.append(t) t.start() while threading.activeCount()>1: time.sleep(10) print 'Active thread count: ',threading.activeCount() #Remove the entities that are categoirezed under disambiguation categories remove_disambiguation() #save entities and their counts per text File=open(output_filename,'w+') for key,value in entities.items(): value=[v[0:3] for v in value if v[0] in Tokens.values()] File.write('%s\t%s\n'%(key,value)) File.flush() File.close
def main(): global g_maxLevel args = sys.argv if 2 == len(args): g_maxLevel = int(args[1]) maxThread = 50 if 3 == len(args): g_maxLevel = int(args[1]) maxThread = int(args[2]) if maxThread < 1: maxThread = 1 no = 0 reqsLeft = len(g_searchIndexList) startTime = time.time() while reqsLeft > 0: if g_useThreads: th = RandomTestThread(no) no += 1 th.start() else: runBrowseTest(no) no += 1 reqsLeft -= 1 while threading.activeCount() > maxThread: time.sleep(2) # wait for thread close while threading.activeCount() > 1: secsToSleep = 0 stopTime = time.time() timeTaken = float(stopTime - startTime) print "Ran in %.3fs" % (timeTaken) print "Exceptions count: %d" % g_exceptionsCount
def start_pool(self): self.log.info("Starting Thread Pool") self.pool = ThreadPool(self.pool_size) if threading.activeCount() > MAX_ALLOWED_THREADS: self.log.error("Thread count ({0}) exceeds maximum ({1})".format(threading.activeCount(), MAX_ALLOWED_THREADS)) self.running_jobs = set()
def run(self): self.load() threadcount = config.search.threadcount for i in range(threadcount): h = threading.Thread(target = self.startOne, name='thread%d' % i) h.setDaemon(True) h.start() t = 0 while mysignal.ALIVE and threading.activeCount() > 1: try: time.sleep(1) except IOError: mysignal.ALIVE = False break self.saveNews() if self.users.qsize() < 100: self.load() while True: a = threading.activeCount() print '»î¶¯Ïß³Ì', a if a <= 1: break try: time.sleep(1) except: pass self.saveNews() print 'all complete', self.total, self.total2
def run_modules(): x = FeedModules() threads = [] for i in dir(x): if i.endswith('_update'): mod = getattr(x, i) threads.append(Thread(target=mod, name='{}'.format(i))) else: pass for t in threads: t.start() print('Initialized: {}'.format(t.name)) sleep(3) stat = 0.0 total = float(len(threads)) tcount = activeCount() while tcount > 1: stat = total - float(activeCount()) + 1.0 prog = stat/total update_progress(prog) tcount = activeCount() sleep(1) print((Fore.GREEN + '\n[+]' + Fore.RESET + ' Feed collection finished!'))
def run(self): self.__sockd = socket.socket(socket.AF_INET,socket.SOCK_STREAM,0) log.debug("Binding to port: " + str(self.__port)) self.__sockd.bind((self.__bind_addr, self.__port)) log.info("Listening on " + self.__bind_addr + ":" + str(self.__port)) self.__sockd.listen(10) self.__sockd.settimeout(2) # 2 second timeouts log.debug("Creating SystemDaemon") self.__systemcontrol = SystemControl() log.info("Starting SystemDaemon") self.__systemcontrol.start() while self.__systemcontrol.isAlive(): if self.__systemcontrol.acceptingConnections(): # allow timeout of accept() to avoid blocking a shutdown try: (s,a) = self.__sockd.accept() log.debug("New connection established from " + str(a)) self.__systemcontrol.addSocket(s) except: pass else: log.warning("SystemDaemon still alive, but not accepting connections") time.sleep(2) log.debug("Closing socket.") self.__sockd.shutdown(socket.SHUT_RDWR) self.__sockd.close() while threading.activeCount() > 1: cThreads = threading.enumerate() log.warning("Waiting on " + str(threading.activeCount()) + \ " active threads...") time.sleep(2) log.info("SystemDaemon Finished.") # Now everything is dead, we can exit. sys.exit(0)
def range_scan(ip_range): for ip in ip_range: threading.Thread(target=ssh_bruteforce, args=(ip)).start() while threading.activeCount() >= max_threads: time.sleep(1) while threading.activeCount() >= 2: time.sleep(1)
def mainCall(times=5,func=None,taskLst=[]): """ 线程调度方法 times:cpu核心倍数 func:多线程将要调度的方法 taskLst:任务列表 """ cpus = times*cpu_count()#同时开启的最大线程数量 activeCnt = threading.activeCount()#当前活动的线程数 taskCnt = len(taskLst)#任务总数 if taskCnt > 0:#确保任务总数大于0 if taskCnt < cpus:#任务总数小于最大线程数量 for e in taskLst: t = threading.Thread(target=func,args=(e,)) t.start() elif taskCnt > cpus: while taskCnt > 0: needCnt = cpus - threading.activeCount() + activeCnt#计算需要启动的线程数量 if needCnt == 0: taskCnt = len(taskLst) time.sleep(1) elif needCnt > 0: for e in range(0,needCnt): taskCnt = len(taskLst) if taskCnt > 0: t2 = threading.Thread(target=func,args=(taskLst.pop(),)) t2.start()
def XPluginStart(self): self.Name = "XTCPgps" self.Sig = "Timor.Python.XTCPgps" self.Desc = "A plugin to send NMEA sentences to mapping software over TCP." # For possible debugging use: # Open a file to write to, located in the same directory as this plugin. self.OutputFile = OutputFile self.LineCount = 0 self.ser = SocketPlugin() # test if self.ser is writable test_thread = threading.Thread(target=self.ser.write, args=("HELLO?",)) before_count = threading.activeCount() test_thread.start() time.sleep(0.1) after_count = threading.activeCount() self.CannotWrite = after_count - before_count self.OutputFile.write(str(before_count) + ' ' + str(after_count) + ' ' + str(self.CannotWrite) + '\n') self.LineCount = self.LineCount + 1 # Locate data references for all communicated variables. # time and date self.drZulu_time = XPLMFindDataRef("sim/time/zulu_time_sec") self.drDate = XPLMFindDataRef("sim/time/local_date_days") # probably ok to set fixed date from system, not x-plane self.n_date = date.today().strftime("%d%m%y") # ground speed self.drVgnd_kts = XPLMFindDataRef("sim/flightmodel/position/groundspeed") # magnetic heading and variation self.drHding_mag = XPLMFindDataRef("sim/flightmodel/position/magpsi") self.drMag_var = XPLMFindDataRef("sim/flightmodel/position/magnetic_variation") # latitude, longitude, and altitude self.drLat_deg = XPLMFindDataRef("sim/flightmodel/position/latitude") self.drLon_deg = XPLMFindDataRef("sim/flightmodel/position/longitude") self.drAlt_ind = XPLMFindDataRef("sim/flightmodel/position/elevation") # indicated airspeed self.drIAS_ind = XPLMFindDataRef("sim/flightmodel/position/indicated_airspeed") # wind vector currently acting on the plane in KTS self.drWind_dir = XPLMFindDataRef("sim/cockpit2/gauges/indicators/wind_heading_deg_mag") self.drWind_speed = XPLMFindDataRef("sim/cockpit2/gauges/indicators/wind_speed_kts") # barometric pressure self.drBaro_alt = XPLMFindDataRef("sim/flightmodel/misc/h_ind") self.drVario_fpm = XPLMFindDataRef("sim/cockpit2/gauges/indicators/total_energy_fpm") # Register our callback for once per 1-second. Positive intervals # are in seconds, negative are the negative of sim frames. Zero # registers but does not schedule a callback for time. self.FlightLoopCB = self.FlightLoopCallback XPLMRegisterFlightLoopCallback(self, self.FlightLoopCB, 1.0, 0) return self.Name, self.Sig, self.Desc
def CheckAppStatus(self): """ControlNotEnalbed or ControlNotVisible""" try: t=threading.Timer(config.apptimeout,timerhandler) t.start() nth=threading.activeCount() logger.debug("a number of thread is %d" % nth) while self.buttonconnect.IsVisible() != True: self.app_form.TypeKeys("\e") time.sleep(1) if nth!=threading.activeCount(): raise AlarmException("Timeout") t.cancel() self.__checkconnection() except AlarmException as e: self.app_form.TypeKeys("\e") self.app.kill_() time.sleep(3) self.Connect() raise e except controls.HwndWrapper.ControlNotEnabled as e : raise e except controls.HwndWrapper.ControlNotVisible as e : raise e finally: del t
def stopThread(self,event): print "Number of Active Threads",threading.activeCount() if threading.activeCount()>1: self.t1.kill() # Autocallbox Thread self.t2.kill() # Read_Status Thread print "Number of Active Threads",threading.activeCount() print "Number of Active Threads",threading.activeCount()
def process_handler(ipaddresses): if args.threads > 1: threads = [] for ip in ipaddresses: queue.put(ip) progress_lock = Lock() while not queue.empty() and not sigint: if args.threads >= activeCount() and not sigint: ip = queue.get() try: # setup thread to run process t = Thread(target=run_process,args=(ip,)) t.daemon = True threads.append(t) t.start() finally: progress_lock.acquire() try: #run a progress bar here pass finally: progress_lock.release() queue.task_done() while activeCount() > 1: time.sleep(0.1) for thread in threads: thread.join() queue.join() else: for ip in ipaddresses: run_process(ip) return
def main(host, portFrom, portTo, timeout, connections, outputFile = 'ports_open.txt'): """ Reads the dictionary and attempts to connect with the username and password on each line Keyword arguments: host -- Host to scan for open ports portFrom -- Port number to scan from (default 1) portTo -- Port number to scan to (default 10000) timeout -- Timeout in seconds (default 30) outputFile -- Output filename to write open ports found to """ print "Starting\n" portTo += 1 for port in range(portFrom,portTo): try: #make sure we only create connections when we have theads available #print threading.activeCount() while threading.activeCount() > connections: t.join(1) boom = int(threading.activeCount()/((connections+1)/2)*2) t = threading.Timer(boom,attempt, args=(host, port,timeout, outputFile)) t.start() except Exception, e: try: e[1] code, reason = e.args print "[ERROR] %s (%d)" %(reason,code) except IndexError: print "[ERROR] %s " %(e.args[0]) except (KeyboardInterrupt, SystemExit): t.cancel() exit(0)
def handle_connection(self, connection): ''' Steps: 1- Receive clients id and replay name (id;replay_name) 2- Pass id;replay_name to logger 3- Send the port mapping to client 4- Wait for results request 5- Send results to client 6- Close connection ''' print threading.activeCount() data = self.receive_object(connection) self.logger_q.put(data) if Configs().get('original_ports'): self.send_object(connection, '') else: self.send_object(connection, self.server_port_mapping_pickle) data = self.receive_object(connection) if data == 'GiveMeResults': self.send_reults(connection) connection.shutdown(socket.SHUT_RDWR) connection.close()
def error_cleanup(): global vm global schds for schd_job in schd_jobs: thread = threading.Thread(target=delete_scheduler_job, args=(schd_job.uuid, )) while threading.active_count() > 10: time.sleep(0.5) exc = sys.exc_info() thread.start() while threading.activeCount() > 1: exc = sys.exc_info() time.sleep(0.1) for schd_trigger in schd_triggers: thread = threading.Thread(target=delete_scheduler_trigger, args=(schd_trigger.uuid, )) while threading.active_count() > 10: time.sleep(0.5) exc = sys.exc_info() thread.start() while threading.activeCount() > 1: exc = sys.exc_info() time.sleep(0.1) if vm: try: vm.destroy() except: test_util.test_logger('expected exception when destroy VM since too many queued task')
def download_images(keyword, number): response = google_images_download.googleimagesdownload( ) #class instantiation arguments = { "keywords": keyword, "limit": number } #creating list of arguments paths = response.download(arguments) return startTime = time.time() activeThreads = threading.activeCount() print("Active Threads = ", activeThreads) numberOfCores = multiprocessing.cpu_count() print("Number Of Cores = ", numberOfCores) N = 2 print("Number Of Threds per Cores = ", N, "\n") print("Program Started....") print("Enter the name:") keyword = input() print("Enter the number:") number = input() k = int(number) for i in range(1, k + 1):
def start(target, users, passwds, ports, timeout_sec, thread_number, num, total, log_in_file, time_sleep, language, verbose_level, socks_proxy, retries, methods_args, scan_id, scan_cmd): # Main function if target_type(target) != 'SINGLE_IPv4' or target_type( target) != 'DOMAIN' or target_type( target) != 'HTTP' or target_type(target) != 'SINGLE_IPv6': # requirements check new_extra_requirements = extra_requirements_dict() if methods_args is not None: for extra_requirement in extra_requirements_dict(): if extra_requirement in methods_args: new_extra_requirements[extra_requirement] = methods_args[ extra_requirement] extra_requirements = new_extra_requirements if ports is None: ports = extra_requirements["port_scan_ports"] try: if extra_requirements["port_scan_stealth"][0].lower() == "true": stealth_flag = True udp_flag = False elif extra_requirements["udp_scan"][0].lower() == "true": stealth_flag = False udp_flag = True else: stealth_flag = False udp_flag = False except Exception: udp_flag = False stealth_flag = False if target_type(target) == 'HTTP': target = target_to_host(target) threads = [] total_req = len(ports) thread_tmp_filename = '{}/tmp/thread_tmp_'.format( load_file_path()) + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) __log_into_file(thread_tmp_filename, 'w', '1', language) trying = 0 keyboard_interrupt_flag = False for port in ports: t = threading.Thread(target=stealth if stealth_flag else (__udp if udp_flag else connect), args=(target, int(port), timeout_sec, log_in_file, language, time_sleep, thread_tmp_filename, socks_proxy, scan_id, scan_cmd, stealth_flag)) threads.append(t) t.start() time.sleep(time_sleep) trying += 1 if verbose_level > 3: info( messages(language, "trying_message").format(trying, total_req, num, total, target, port, 'port_scan')) while 1: try: if threading.activeCount() >= thread_number: time.sleep(0.01) else: break except KeyboardInterrupt: keyboard_interrupt_flag = True break if keyboard_interrupt_flag: break # wait for threads kill_switch = 0 kill_time = int(timeout_sec / 0.1) * (5 + timeout_sec) * 10 if int( timeout_sec / 0.1) is not 0 else 2 while 1: time.sleep(0.1) kill_switch += 1 try: if threading.activeCount() is 1 or kill_switch is kill_time: break except KeyboardInterrupt: break thread_write = int(open(thread_tmp_filename).read().rsplit()[0]) if thread_write is 1 and verbose_level is not 0: data = json.dumps( { 'HOST': target, 'USERNAME': '', 'PASSWORD': '', 'PORT': '', 'TYPE': 'port_scan', 'DESCRIPTION': messages(language, "no_open_ports"), 'TIME': now(), 'CATEGORY': "scan", 'SCAN_ID': scan_id, 'SCAN_CMD': scan_cmd }) + "\n" __log_into_file(log_in_file, 'a', data, language) os.remove(thread_tmp_filename) else: warn( messages(language, "input_target_error").format('port_scan', target))
def start(target, users, passwds, ports, timeout_sec, thread_number, num, total, log_in_file, time_sleep, language, verbose_level, show_version, check_update, socks_proxy, retries, ping_flag, methods_args, scan_id, scan_cmd): # Main function if target_type(target) != 'SINGLE_IPv4' or target_type( target) != 'DOMAIN' or target_type(target) != 'HTTP': # requirements check new_extra_requirements = extra_requirements_dict() if methods_args is not None: for extra_requirement in extra_requirements_dict(): if extra_requirement in methods_args: new_extra_requirements[extra_requirement] = methods_args[ extra_requirement] extra_requirements = new_extra_requirements if users is None: users = extra_requirements["ftp_brute_users"] if passwds is None: passwds = extra_requirements["ftp_brute_passwds"] if ports is None: ports = extra_requirements["ftp_brute_ports"] if target_type(target) == 'HTTP': target = target_to_host(target) if ping_flag and do_one_ping(target, timeout_sec, 8) is None: if socks_proxy is not None: socks_version = socks.SOCKS5 if socks_proxy.startswith( 'socks5://') else socks.SOCKS4 socks_proxy = socks_proxy.rsplit('://')[1] if '@' in socks_proxy: socks_username = socks_proxy.rsplit(':')[0] socks_password = socks_proxy.rsplit(':')[1].rsplit('@')[0] socks.set_default_proxy( socks_version, str(socks_proxy.rsplit('@')[1].rsplit(':')[0]), int(socks_proxy.rsplit(':')[-1]), username=socks_username, password=socks_password) socket.socket = socks.socksocket socket.getaddrinfo = getaddrinfo else: socks.set_default_proxy(socks_version, str(socks_proxy.rsplit(':')[0]), int(socks_proxy.rsplit(':')[1])) socket.socket = socks.socksocket socket.getaddrinfo = getaddrinfo warn(messages(language, 100).format(target, 'ftp_brute')) return None threads = [] max = thread_number total_req = len(users) * len(passwds) * len(ports) thread_tmp_filename = 'tmp/thread_tmp_' + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) ports_tmp_filename = 'tmp/ports_tmp_' + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) thread_write = open(thread_tmp_filename, 'w') thread_write.write('1') thread_write.close() ports_write = open(ports_tmp_filename, 'w') ports_write.write('') ports_write.close() trying = 0 ports = test_ports(ports, timeout_sec, target, retries, language, num, total, time_sleep, ports_tmp_filename, thread_number, total_req, verbose_level, socks_proxy) for port in ports: for user in users: for passwd in passwds: t = threading.Thread( target=login, args=(user, passwd, target, port, timeout_sec, log_in_file, language, retries, time_sleep, thread_tmp_filename, socks_proxy)) threads.append(t) t.start() trying += 1 if verbose_level is not 0: info( messages(language, 72).format(trying, total_req, num, total, target, port, 'ftp_brute')) while 1: try: if threading.activeCount() >= max: time.sleep(0.01) else: break except KeyboardInterrupt: break break # wait for threads kill_switch = 0 kill_time = int(timeout_sec / 0.1) if int(timeout_sec / 0.1) is not 0 else 1 while 1: time.sleep(0.1) kill_switch += 1 try: if threading.activeCount() is 1 or kill_switch is kill_time: break except KeyboardInterrupt: break thread_write = int(open(thread_tmp_filename).read().rsplit()[0]) if thread_write is 1 and verbose_level is not 0: save = open(log_in_file, 'a') save.write( json.dumps({ 'HOST': target, 'USERNAME': '', 'PASSWORD': '', 'PORT': '', 'TYPE': 'ftp_brute', 'DESCRIPTION': messages(language, 95), 'TIME': now(), 'CATEGORY': "brute", 'SCAN_ID': scan_id, 'SCAN_CMD': scan_cmd }) + '\n') save.close() os.remove(thread_tmp_filename) else: warn(messages(language, 69).format('ftp_brute', target))
G4MaterialTable.ListMaterial = _list_material # ------------------------------------------------------------------ # termination # ------------------------------------------------------------------ def gTerminate(): gGeometryManager.OpenGeometry() # ------------------------------------------------------------------ # signal handler # ------------------------------------------------------------------ import signal import threading def _run_abort(signum, frame): state = gStateManager.GetCurrentState() if (state == G4ApplicationState.G4State_GeomClosed or state == G4ApplicationState.G4State_EventProc): print "aborting Run ..." gRunManager.AbortRun(True) else: raise KeyboardInterrupt if (threading.activeCount() == 1): signal.signal(signal.SIGINT, _run_abort)
def start(target, users, passwds, ports, timeout_sec, thread_number, num, total, log_in_file, time_sleep, language, verbose_level, socks_proxy, retries, methods_args, scan_id, scan_cmd): # Main function if target_type(target) != 'SINGLE_IPv4' or target_type(target) != 'DOMAIN' or target_type( target) != 'HTTP' or target_type(target) != 'SINGLE_IPv6': # rand useragent user_agent_list = [ "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.0.5) Gecko/20060719 Firefox/1.5.0.5", "Googlebot/2.1 ( http://www.googlebot.com/bot.html)", "Mozilla/5.0 (X11; U; Linux x86_64; en-US) AppleWebKit/534.13 (KHTML, like Gecko) Ubuntu/10.04" " Chromium/9.0.595.0 Chrome/9.0.595.0 Safari/534.13", "Mozilla/5.0 (compatible; MSIE 7.0; Windows NT 5.2; WOW64; .NET CLR 2.0.50727)", "Opera/9.80 (Windows NT 5.2; U; ru) Presto/2.5.22 Version/10.51", "Mozilla/5.0 (compatible; 008/0.83; http://www.80legs.com/webcrawler.html) Gecko/2008032620", "Debian APT-HTTP/1.3 (0.8.10.3)", "Mozilla/5.0 (compatible; Googlebot/2.1; +http://www.google.com/bot.html)", "Googlebot/2.1 (+http://www.googlebot.com/bot.html)", "Mozilla/5.0 (compatible; Yahoo! Slurp; http://help.yahoo.com/help/us/ysearch/slurp)", "YahooSeeker/1.2 (compatible; Mozilla 4.0; MSIE 5.5; yahooseeker at yahoo-inc dot com ; " "http://help.yahoo.com/help/us/shop/merchant/)", "Mozilla/5.0 (compatible; YandexBot/3.0; +http://yandex.com/bots)", "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)", "msnbot/1.1 (+http://search.msn.com/msnbot.htm)" ] headers = {'User-agent': random.choice(user_agent_list), 'Content-Type': 'text/xml'} # requirements check new_extra_requirements = extra_requirements_dict() if methods_args is not None: for extra_requirement in extra_requirements_dict(): if extra_requirement in methods_args: new_extra_requirements[ extra_requirement] = methods_args[extra_requirement] extra_requirements = new_extra_requirements threads = [] if users is None: users = extra_requirements["wp_users"] if passwds is None: passwds = extra_requirements["wp_passwds"] if ports is None: ports = extra_requirements["wp_xmlrpc_brute_ports"] if verbose_level > 3: total_req = len(users) * len(passwds) * len (ports) else: total_req = len(users) * len(passwds) * len(ports) thread_tmp_filename = '{}/tmp/thread_tmp_'.format(load_file_path()) + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) __log_into_file(thread_tmp_filename, 'w', '1', language) trying = 0 if target_type(target) != "HTTP": target = 'https://' + target for port in ports: if test(str(target), port, retries, timeout_sec, headers, socks_proxy, verbose_level, trying, total_req, total, num, language) is True: keyboard_interrupt_flag = False for user in users: for passwd in passwds: #print(user + " " + passwd) t = threading.Thread(target=check, args=( user, passwd, target, port, headers, timeout_sec, log_in_file, language, retries, time_sleep, thread_tmp_filename, socks_proxy, scan_id, scan_cmd)) threads.append(t) t.start() trying += 1 if verbose_level > 3: info(messages(language, "trying_message").format(trying, total_req, num, total, target_to_host(target), port, 'wp_xmlrpc_brute')) while 1: try: if threading.activeCount() >= thread_number: time.sleep(0.01) else: break except KeyboardInterrupt: keyboard_interrupt_flag = True break if keyboard_interrupt_flag: break else: warn(messages(language, "open_error").format(target)) # wait for threads kill_switch = 0 kill_time = int( timeout_sec / 0.1) if int(timeout_sec / 0.1) is not 0 else 1 while 1: time.sleep(0.1) kill_switch += 1 try: if threading.activeCount() is 1 or kill_switch is kill_time: break except KeyboardInterrupt: break thread_write = int(open(thread_tmp_filename).read().rsplit()[0]) os.remove(thread_tmp_filename) else: warn(messages(language, "input_target_error").format( 'wp_xmlrpc_brute', target))
def crack0(data, sandi, p): tampil('\rh[*]MengCrack \rk%d Akun \rhdengan sandi \rm[\rk%s\rm]' % (len(data), sandi)) print('\033[32;1m[*]Cracking \033[31;1m[\033[36;1m0%\033[31;1m]\033[0m', end='') os.sys.stdout.flush() akun_jml = [] akun_sukses = [] akun_cekpoint = [] akun_error = [] akun_gagal = [] jml0, jml1 = 0, 0 th = [] for i in data: i = i.replace(' ', '') if len(i) != 0: th.append(mt(i, sandi)) for i in th: jml1 += 1 i.daemon = True try: i.start() except KeyboardInterrupt: exit() while 1: try: for i in th: a = i.update() if a[0] != 3 and a[1] not in akun_jml: jml0 += 1 if a[0] == 2: akun_cekpoint.append(a[1]) elif a[0] == 1: akun_sukses.append(a[1]) elif a[0] == 0: akun_gagal.append(a[1]) elif a[0] == 8: akun_error.append(a[1]) print( '\r\033[32;1m[*]Cracking \033[31;1m[\033[36;1m%0.2f%s\033[31;1m]\033[0m' % (float((float(jml0) / float(jml1)) * 100), '%'), end='') os.sys.stdout.flush() akun_jml.append(a[1]) except KeyboardInterrupt: os.sys.exit() try: if threading.activeCount() == 1: break except KeyboardInterrupt: keluar() print( '\r\033[32;1m[*]Cracking \033[31;1m[\033[36;1m100%\033[31;1m]\033[0m ' ) if len(akun_sukses) != 0: tampil('\rh[*]Daftar akun sukses') for i in akun_sukses: tampil('\rh==>\rk%s \rm[\rp%s\rm]' % (i, sandi)) tampil('\rh[*]Jumlah akun berhasil\rp %d' % len(akun_sukses)) tampil('\rm[*]Jumlah akun gagal\rp %d' % len(akun_gagal)) tampil('\rk[*]Jumlah akun cekpoint\rp %d' % len(akun_cekpoint)) tampil('\rc[*]Jumlah akun error\rp %d' % len(akun_error)) if p: i = inputD('[?]Tidak Puas dengan Hasil,Mau coba lagi (y/t)', ['Y', 'T']) if i.upper() == 'Y': return crack(data) else: return menu() else: return 0
i, time.strftime("%H:%M:%S", time.gmtime()))) for i in range(10): # Each time through the loop a Thread object is created # You pass it the function to execute and any # arguments to pass to that method # The arguments passed must be a sequence which # is why we need the comma with 1 argument thread = threading.Thread(target=execute_thread, args=(i, )) thread.start() # Display active threads # The extra 1 is this for loop executing in the main # thread print("Active Threads :", threading.activeCount()) # Returns a list of all active thread objects print("Thread Objects :", threading.enumerate()) # Subclassing Threads # You can subclass the Thread object and then define what happens each time a new thread is executed or run class CustThread(threading.Thread): def __init__(self, name): threading.Thread.__init__(self) self.name = name def run(self):
while alive: alive = False for thread in threads: if thread.isAlive(): alive = True time.sleep(0.1) except KeyboardInterrupt: print kb.threadContinue = False kb.threadException = True if numThreads > 1: logger.info("waiting for threads to finish (Ctrl+C was pressed)") try: while (threading.activeCount() > 1): pass except KeyboardInterrupt: raise SqlmapThreadException( "user aborted (Ctrl+C was pressed multiple times)") if forwardException: raise except (SqlmapConnectionException, SqlmapValueException), errMsg: print kb.threadException = True logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg))
def start(self): if self.finalized: self.bus.log('Already deamonized.') # forking has issues with threads: # http://www.opengroup.org/onlinepubs/000095399/functions/fork.html # "The general problem with making fork() work in a multi-threaded # world is what to do with all of the threads..." # So we check for active threads: if threading.activeCount() != 1: self.bus.log('There are %r active threads. ' 'Daemonizing now may cause strange failures.' % threading.enumerate(), level=30) # See http://www.erlenstar.demon.co.uk/unix/faq_2.html#SEC16 # (or http://www.faqs.org/faqs/unix-faq/programmer/faq/ section 1.7) # and http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/66012 # Finish up with the current stdout/stderr sys.stdout.flush() sys.stderr.flush() # Do first fork. try: pid = os.fork() if pid == 0: # This is the child process. Continue. pass else: # This is the first parent. Exit, now that we've forked. self.bus.log('Forking once.') os._exit(0) except OSError: # Python raises OSError rather than returning negative numbers. exc = sys.exc_info()[1] sys.exit('%s: fork #1 failed: (%d) %s\n' % (sys.argv[0], exc.errno, exc.strerror)) os.setsid() # Do second fork try: pid = os.fork() if pid > 0: self.bus.log('Forking twice.') os._exit(0) # Exit second parent except OSError: exc = sys.exc_info()[1] sys.exit('%s: fork #2 failed: (%d) %s\n' % (sys.argv[0], exc.errno, exc.strerror)) os.chdir('/') os.umask(0) si = open(self.stdin, 'r') so = open(self.stdout, 'a+') se = open(self.stderr, 'a+') # os.dup2(fd, fd2) will close fd2 if necessary, # so we don't explicitly close stdin/out/err. # See http://docs.python.org/lib/os-fd-ops.html os.dup2(si.fileno(), sys.stdin.fileno()) os.dup2(so.fileno(), sys.stdout.fileno()) os.dup2(se.fileno(), sys.stderr.fileno()) self.bus.log('Daemonized to PID: %s' % os.getpid()) self.finalized = True
elif command == "i": foosball.activity.setAllwaysOff( not foosball.activity.allwaysOff) elif command == "a": foosball.activity.setAllwaysOn(not foosball.activity.allwaysOn) elif command == "r": foosball.resetScore() elif command.isdigit(): foosball.setScore(int(command), foosball.team[2].score) else: log.info("Unknown command %s recieved" % command) # If exception was caught from select due to signal, then ignore except select.error: pass if threading.activeCount() < 3: log.debug( "Number of threads is lower than 3. Seems strange. Quitting") break if foosball.signalbreak: log.debug("Shutdown order given - Exiting main loop") break mainbeat += 1 if mainbeat % 100 == 0: q = time.time() log.info("Main loop %d loops." % mainbeat) # If we stop - for any reason - stop foosball instance. finally: log.debug("Running finally cleanup code")
def task(n): print("线程名:%s----%s" % (currentThread().name, n)) time.sleep(3) print("数量:%s" % activeCount())
def scan(self, folder=None, files=None, release_download=None, simple=False, newer_than=0, return_ignored=True, on_found=None): folder = sp(folder) if not folder or not os.path.isdir(folder): log.error('Folder doesn\'t exists: %s', folder) return {} # Get movie "master" files movie_files = {} leftovers = [] # Scan all files of the folder if no files are set if not files: check_file_date = True try: files = [] for root, dirs, walk_files in os.walk(folder): files.extend([ sp(os.path.join(root, filename)) for filename in walk_files ]) # Break if CP wants to shut down if self.shuttingDown(): break except: log.error('Failed getting files from %s: %s', (folder, traceback.format_exc())) log.debug('Found %s files to scan and group in %s', (len(files), folder)) else: check_file_date = False files = [sp(x) for x in files] for file_path in files: if not os.path.exists(file_path): continue # Remove ignored files if self.isSampleFile(file_path): leftovers.append(file_path) continue elif not self.keepFile(file_path): continue is_dvd_file = self.isDVDFile(file_path) if self.filesizeBetween( file_path, self.file_sizes['movie'] ) or is_dvd_file: # Minimal 300MB files or is DVD file # Normal identifier identifier = self.createStringIdentifier( file_path, folder, exclude_filename=is_dvd_file) identifiers = [identifier] # Identifier with quality quality = fireEvent('quality.guess', [file_path], single=True) if not is_dvd_file else { 'identifier': 'dvdr' } if quality: identifier_with_quality = '%s %s' % ( identifier, quality.get('identifier', '')) identifiers = [identifier_with_quality, identifier] if not movie_files.get(identifier): movie_files[identifier] = { 'unsorted_files': [], 'identifiers': identifiers, 'is_dvd': is_dvd_file, } movie_files[identifier]['unsorted_files'].append(file_path) else: leftovers.append(file_path) # Break if CP wants to shut down if self.shuttingDown(): break # Cleanup del files # Sort reverse, this prevents "Iron man 2" from getting grouped with "Iron man" as the "Iron Man 2" # files will be grouped first. leftovers = set(sorted(leftovers, reverse=True)) # Group files minus extension ignored_identifiers = [] for identifier, group in movie_files.items(): if identifier not in group['identifiers'] and len(identifier) > 0: group['identifiers'].append(identifier) log.debug('Grouping files: %s', identifier) has_ignored = 0 for file_path in list(group['unsorted_files']): ext = getExt(file_path) wo_ext = file_path[:-(len(ext) + 1)] found_files = set([i for i in leftovers if wo_ext in i]) group['unsorted_files'].extend(found_files) leftovers = leftovers - found_files has_ignored += 1 if ext == 'ignore' else 0 if has_ignored == 0: for file_path in list(group['unsorted_files']): ext = getExt(file_path) has_ignored += 1 if ext == 'ignore' else 0 if has_ignored > 0: ignored_identifiers.append(identifier) # Break if CP wants to shut down if self.shuttingDown(): break # Create identifiers for all leftover files path_identifiers = {} for file_path in leftovers: identifier = self.createStringIdentifier(file_path, folder) if not path_identifiers.get(identifier): path_identifiers[identifier] = [] path_identifiers[identifier].append(file_path) # Group the files based on the identifier delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on identifier: %s', identifier) group = movie_files.get(identifier) if group: group['unsorted_files'].extend(found_files) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers = leftovers - set(found_files) # Break if CP wants to shut down if self.shuttingDown(): break # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Group based on folder delete_identifiers = [] for identifier, found_files in path_identifiers.items(): log.debug('Grouping files on foldername: %s', identifier) for ff in found_files: new_identifier = self.createStringIdentifier( os.path.dirname(ff), folder) group = movie_files.get(new_identifier) if group: group['unsorted_files'].extend([ff]) delete_identifiers.append(identifier) # Remove the found files from the leftover stack leftovers -= leftovers - set([ff]) # Break if CP wants to shut down if self.shuttingDown(): break # leftovers should be empty if leftovers: log.debug('Some files are still left over: %s', leftovers) # Cleaning up used for identifier in delete_identifiers: if path_identifiers.get(identifier): del path_identifiers[identifier] del delete_identifiers # Make sure we remove older / still extracting files valid_files = {} while True and not self.shuttingDown(): try: identifier, group = movie_files.popitem() except: break # Check if movie is fresh and maybe still unpacking, ignore files newer than 1 minute if check_file_date: files_too_new, time_string = self.checkFilesChanged( group['unsorted_files']) if files_too_new: log.info( 'Files seem to be still unpacking or just unpacked (created on %s), ignoring for now: %s', (time_string, identifier)) # Delete the unsorted list del group['unsorted_files'] continue # Only process movies newer than x if newer_than and newer_than > 0: has_new_files = False for cur_file in group['unsorted_files']: file_time = self.getFileTimes(cur_file) if file_time[0] > newer_than or file_time[1] > newer_than: has_new_files = True break if not has_new_files: log.debug( 'None of the files have changed since %s for %s, skipping.', (time.ctime(newer_than), identifier)) # Delete the unsorted list del group['unsorted_files'] continue valid_files[identifier] = group del movie_files total_found = len(valid_files) # Make sure only one movie was found if a download ID is provided if release_download and total_found == 0: log.info( 'Download ID provided (%s), but no groups found! Make sure the download contains valid media files (fully extracted).', release_download.get('imdb_id')) elif release_download and total_found > 1: log.info( 'Download ID provided (%s), but more than one group found (%s). Ignoring Download ID...', (release_download.get('imdb_id'), len(valid_files))) release_download = None # Determine file types db = get_session() processed_movies = {} while True and not self.shuttingDown(): try: identifier, group = valid_files.popitem() except: break if return_ignored is False and identifier in ignored_identifiers: log.debug('Ignore file found, ignoring release: %s', identifier) continue # Group extra (and easy) files first group['files'] = { 'movie_extra': self.getMovieExtras(group['unsorted_files']), 'subtitle': self.getSubtitles(group['unsorted_files']), 'subtitle_extra': self.getSubtitlesExtras(group['unsorted_files']), 'nfo': self.getNfo(group['unsorted_files']), 'trailer': self.getTrailers(group['unsorted_files']), 'leftover': set(group['unsorted_files']), } # Media files if group['is_dvd']: group['files']['movie'] = self.getDVDFiles( group['unsorted_files']) else: group['files']['movie'] = self.getMediaFiles( group['unsorted_files']) if len(group['files']['movie']) == 0: log.error('Couldn\'t find any movie files for %s', identifier) continue log.debug('Getting metadata for %s', identifier) group['meta_data'] = self.getMetaData( group, folder=folder, release_download=release_download) # Subtitle meta group['subtitle_language'] = self.getSubtitleLanguage( group) if not simple else {} # Get parent dir from movie files for movie_file in group['files']['movie']: group['parentdir'] = os.path.dirname(movie_file) group['dirname'] = None folder_names = group['parentdir'].replace(folder, '').split( os.path.sep) folder_names.reverse() # Try and get a proper dirname, so no "A", "Movie", "Download" etc for folder_name in folder_names: if folder_name.lower( ) not in self.ignore_names and len(folder_name) > 2: group['dirname'] = folder_name break break # Leftover "sorted" files for file_type in group['files']: if not file_type is 'leftover': group['files']['leftover'] -= set( group['files'][file_type]) group['files'][file_type] = list(group['files'][file_type]) group['files']['leftover'] = list(group['files']['leftover']) # Delete the unsorted list del group['unsorted_files'] # Determine movie group['library'] = self.determineMovie( group, release_download=release_download) if not group['library']: log.error('Unable to determine movie: %s', group['identifiers']) else: movie = db.query(Media).filter_by( library_id=group['library']['id']).first() group['movie_id'] = None if not movie else movie.id db.expire_all() processed_movies[identifier] = group # Notify parent & progress on something found if on_found: on_found(group, total_found, total_found - len(processed_movies)) # Wait for all the async events calm down a bit while threading.activeCount() > 100 and not self.shuttingDown(): log.debug('Too many threads active, waiting a few seconds') time.sleep(10) if len(processed_movies) > 0: log.info('Found %s movies in the folder %s', (len(processed_movies), folder)) else: log.debug('Found no movies in the folder %s', folder) return processed_movies
while threading.activeCount() > 1 and ( time.time() - _) > THREAD_FINALIZATION_TIMEOUT: time.sleep(0.01) if cmdLineOptions.get("sqlmapShell"): cmdLineOptions.clear() conf.clear() kb.clear() conf.disableBanner = True main() if __name__ == "__main__": try: main() except KeyboardInterrupt: pass except SystemExit: raise except: traceback.print_exc() finally: # Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program if threading.activeCount() > 1: os._exit(getattr(os, "_exitcode", 0)) else: sys.exit(getattr(os, "_exitcode", 0)) else: # cancelling postponed imports (because of Travis CI checks) __import__("lib.controller.controller")
if __name__ == '__main__': frames = 200 count = 0 poolA = ActivePool() jobs = [] for i in range(frames): frame = i + 1 jobs.append( threading.Thread(target=worker, name='Frame_{0}'.format(frame), args=(poolA, str(i), frame, frame))) for i in range(8): jobs[i].daemon = True jobs[i].start() count += 1 while threading.activeCount() > 1: if threading.activeCount() < 8: if count < frames: jobs[count].daemon = True jobs[count].start() count += 1 for j in jobs: try: j.join(1) print 'Pool-threads active: {0}'.format(poolA.numActive()) except: pass
def _curl_setup_request(curl, request, buffer, headers): curl.setopt(pycurl.URL, utf8(request.url)) # libcurl's magic "Expect: 100-continue" behavior causes delays # with servers that don't support it (which include, among others, # Google's OpenID endpoint). Additionally, this behavior has # a bug in conjunction with the curl_multi_socket_action API # (https://sourceforge.net/tracker/?func=detail&atid=100976&aid=3039744&group_id=976), # which increases the delays. It's more trouble than it's worth, # so just turn off the feature (yes, setting Expect: to an empty # value is the official way to disable this) if "Expect" not in request.headers: request.headers["Expect"] = "" # libcurl adds Pragma: no-cache by default; disable that too if "Pragma" not in request.headers: request.headers["Pragma"] = "" # Request headers may be either a regular dict or HTTPHeaders object if isinstance(request.headers, httputil.HTTPHeaders): curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.get_all()]) else: curl.setopt(pycurl.HTTPHEADER, [utf8("%s: %s" % i) for i in request.headers.iteritems()]) if request.header_callback: curl.setopt(pycurl.HEADERFUNCTION, request.header_callback) else: curl.setopt(pycurl.HEADERFUNCTION, lambda line: _curl_header_callback(headers, line)) if request.streaming_callback: curl.setopt(pycurl.WRITEFUNCTION, request.streaming_callback) else: curl.setopt(pycurl.WRITEFUNCTION, buffer.write) curl.setopt(pycurl.FOLLOWLOCATION, request.follow_redirects) curl.setopt(pycurl.MAXREDIRS, request.max_redirects) curl.setopt(pycurl.CONNECTTIMEOUT_MS, int(1000 * request.connect_timeout)) curl.setopt(pycurl.TIMEOUT_MS, int(1000 * request.request_timeout)) if request.user_agent: curl.setopt(pycurl.USERAGENT, utf8(request.user_agent)) else: curl.setopt(pycurl.USERAGENT, "Mozilla/5.0 (compatible; pycurl)") if request.network_interface: curl.setopt(pycurl.INTERFACE, request.network_interface) if request.use_gzip: curl.setopt(pycurl.ENCODING, "gzip,deflate") else: curl.setopt(pycurl.ENCODING, "none") if request.proxy_host and request.proxy_port: curl.setopt(pycurl.PROXY, request.proxy_host) curl.setopt(pycurl.PROXYPORT, request.proxy_port) if request.proxy_username: credentials = '%s:%s' % (request.proxy_username, request.proxy_password) curl.setopt(pycurl.PROXYUSERPWD, credentials) else: curl.setopt(pycurl.PROXY, '') if request.validate_cert: curl.setopt(pycurl.SSL_VERIFYPEER, 1) curl.setopt(pycurl.SSL_VERIFYHOST, 2) else: curl.setopt(pycurl.SSL_VERIFYPEER, 0) curl.setopt(pycurl.SSL_VERIFYHOST, 0) if request.ca_certs is not None: curl.setopt(pycurl.CAINFO, request.ca_certs) else: # There is no way to restore pycurl.CAINFO to its default value # (Using unsetopt makes it reject all certificates). # I don't see any way to read the default value from python so it # can be restored later. We'll have to just leave CAINFO untouched # if no ca_certs file was specified, and require that if any # request uses a custom ca_certs file, they all must. pass if request.allow_ipv6 is False: # Curl behaves reasonably when DNS resolution gives an ipv6 address # that we can't reach, so allow ipv6 unless the user asks to disable. # (but see version check in _process_queue above) curl.setopt(pycurl.IPRESOLVE, pycurl.IPRESOLVE_V4) # Set the request method through curl's irritating interface which makes # up names for almost every single method curl_options = { "GET": pycurl.HTTPGET, "POST": pycurl.POST, "PUT": pycurl.UPLOAD, "HEAD": pycurl.NOBODY, } custom_methods = set(["DELETE"]) for o in curl_options.values(): curl.setopt(o, False) if request.method in curl_options: curl.unsetopt(pycurl.CUSTOMREQUEST) curl.setopt(curl_options[request.method], True) elif request.allow_nonstandard_methods or request.method in custom_methods: curl.setopt(pycurl.CUSTOMREQUEST, request.method) else: raise KeyError('unknown method ' + request.method) # Handle curl's cryptic options for every individual HTTP method if request.method in ("POST", "PUT"): request_buffer = cStringIO.StringIO(utf8(request.body)) curl.setopt(pycurl.READFUNCTION, request_buffer.read) if request.method == "POST": def ioctl(cmd): if cmd == curl.IOCMD_RESTARTREAD: request_buffer.seek(0) curl.setopt(pycurl.IOCTLFUNCTION, ioctl) curl.setopt(pycurl.POSTFIELDSIZE, len(request.body)) else: curl.setopt(pycurl.INFILESIZE, len(request.body)) if request.auth_username is not None: userpwd = "%s:%s" % (request.auth_username, request.auth_password or '') curl.setopt(pycurl.HTTPAUTH, pycurl.HTTPAUTH_BASIC) curl.setopt(pycurl.USERPWD, utf8(userpwd)) gen_log.debug("%s %s (username: %r)", request.method, request.url, request.auth_username) else: curl.unsetopt(pycurl.USERPWD) gen_log.debug("%s %s", request.method, request.url) if request.client_cert is not None: curl.setopt(pycurl.SSLCERT, request.client_cert) if request.client_key is not None: curl.setopt(pycurl.SSLKEY, request.client_key) if threading.activeCount() > 1: # libcurl/pycurl is not thread-safe by default. When multiple threads # are used, signals should be disabled. This has the side effect # of disabling DNS timeouts in some environments (when libcurl is # not linked against ares), so we don't do it when there is only one # thread. Applications that use many short-lived threads may need # to set NOSIGNAL manually in a prepare_curl_callback since # there may not be any other threads running at the time we call # threading.activeCount. curl.setopt(pycurl.NOSIGNAL, 1) if request.prepare_curl_callback is not None: request.prepare_curl_callback(curl)
def main(): """ Main function of sqlmap when running from command line. """ try: dirtyPatches() resolveCrossReferences() checkEnvironment() setPaths(modulePath()) banner() # Store original command line options for possible later restoration args = cmdLineParser() cmdLineOptions.update( args.__dict__ if hasattr(args, "__dict__") else args) initOptions(cmdLineOptions) if checkPipedInput(): conf.batch = True if conf.get("api"): # heavy imports from lib.utils.api import StdDbOut from lib.utils.api import setRestAPILog # Overwrite system standard output and standard error to write # to an IPC database sys.stdout = StdDbOut(conf.taskid, messagetype="stdout") sys.stderr = StdDbOut(conf.taskid, messagetype="stderr") setRestAPILog() conf.showTime = True dataToStdout("[!] legal disclaimer: %s\n\n" % LEGAL_DISCLAIMER, forceOutput=True) dataToStdout("[*] starting @ %s\n\n" % time.strftime("%X /%Y-%m-%d/"), forceOutput=True) init() if not conf.updateAll: # Postponed imports (faster start) if conf.smokeTest: from lib.core.testing import smokeTest os._exitcode = 1 - (smokeTest() or 0) elif conf.vulnTest: from lib.core.testing import vulnTest os._exitcode = 1 - (vulnTest() or 0) elif conf.bedTest: from lib.core.testing import bedTest os._exitcode = 1 - (bedTest() or 0) elif conf.fuzzTest: from lib.core.testing import fuzzTest fuzzTest() else: from lib.controller.controller import start if conf.profile and six.PY2: from lib.core.profiling import profile globals()["start"] = start profile() else: try: if conf.crawlDepth and conf.bulkFile: targets = getFileItems(conf.bulkFile) for i in xrange(len(targets)): try: kb.targets.clear() target = targets[i] if not re.search(r"(?i)\Ahttp[s]*://", target): target = "http://%s" % target infoMsg = "starting crawler for target URL '%s' (%d/%d)" % ( target, i + 1, len(targets)) logger.info(infoMsg) crawl(target) except Exception as ex: if not isinstance(ex, SqlmapUserQuitException): errMsg = "problem occurred while crawling '%s' ('%s')" % ( target, getSafeExString(ex)) logger.error(errMsg) else: raise else: if kb.targets: start() else: start() except Exception as ex: os._exitcode = 1 if "can't start new thread" in getSafeExString(ex): errMsg = "unable to start new threads. Please check OS (u)limits" logger.critical(errMsg) raise SystemExit else: raise except SqlmapUserQuitException: if not conf.batch: errMsg = "user quit" logger.error(errMsg) except (SqlmapSilentQuitException, bdb.BdbQuit): pass except SqlmapShellQuitException: cmdLineOptions.sqlmapShell = False except SqlmapBaseException as ex: errMsg = getSafeExString(ex) logger.critical(errMsg) os._exitcode = 1 raise SystemExit except KeyboardInterrupt: print() except EOFError: print() errMsg = "exit" logger.error(errMsg) except SystemExit as ex: os._exitcode = ex.code or 0 except: print() errMsg = unhandledExceptionMessage() excMsg = traceback.format_exc() valid = checkIntegrity() os._exitcode = 255 if any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")): errMsg = "memory exhaustion detected" logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded", "Disk full while accessing")): errMsg = "no space left on output device" logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("The paging file is too small", )): errMsg = "no space left for paging file" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("Access is denied", "subprocess", "metasploit")): errMsg = "permission error occurred while running Metasploit" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("Permission denied", "metasploit")): errMsg = "permission error occurred while using Metasploit" logger.critical(errMsg) raise SystemExit elif "Read-only file system" in excMsg: errMsg = "output device is mounted as read-only" logger.critical(errMsg) raise SystemExit elif "Insufficient system resources" in excMsg: errMsg = "resource exhaustion detected" logger.critical(errMsg) raise SystemExit elif "OperationalError: disk I/O error" in excMsg: errMsg = "I/O error on output device" logger.critical(errMsg) raise SystemExit elif "Violation of BIDI" in excMsg: errMsg = "invalid URL (violation of Bidi IDNA rule - RFC 5893)" logger.critical(errMsg) raise SystemExit elif "Invalid IPv6 URL" in excMsg: errMsg = "invalid URL ('%s')" % excMsg.strip().split('\n')[-1] logger.critical(errMsg) raise SystemExit elif "_mkstemp_inner" in excMsg: errMsg = "there has been a problem while accessing temporary files" logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("tempfile.mkdtemp", "tempfile.mkstemp", "tempfile.py")): errMsg = "unable to write to the temporary directory '%s'. " % tempfile.gettempdir( ) errMsg += "Please make sure that your disk is not full and " errMsg += "that you have sufficient write permissions to " errMsg += "create temporary files and/or directories" logger.critical(errMsg) raise SystemExit elif "Permission denied: '" in excMsg: match = re.search(r"Permission denied: '([^']*)", excMsg) errMsg = "permission error occurred while accessing file '%s'" % match.group( 1) logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("twophase", "sqlalchemy")): errMsg = "please update the 'sqlalchemy' package (>= 1.1.11) " errMsg += "(Reference: 'https://qiita.com/tkprof/items/7d7b2d00df9c5f16fffe')" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("scramble_caching_sha2", "TypeError")): errMsg = "please downgrade the 'PyMySQL' package (=< 0.8.1) " errMsg += "(Reference: 'https://github.com/PyMySQL/PyMySQL/issues/700')" logger.critical(errMsg) raise SystemExit elif "must be pinned buffer, not bytearray" in excMsg: errMsg = "error occurred at Python interpreter which " errMsg += "is fixed in 2.7. Please update accordingly " errMsg += "(Reference: 'https://bugs.python.org/issue8104')" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("Resource temporarily unavailable", "os.fork()", "dictionaryAttack")): errMsg = "there has been a problem while running the multiprocessing hash cracking. " errMsg += "Please rerun with option '--threads=1'" logger.critical(errMsg) raise SystemExit elif "can't start new thread" in excMsg: errMsg = "there has been a problem while creating new thread instance. " errMsg += "Please make sure that you are not running too many processes" if not IS_WIN: errMsg += " (or increase the 'ulimit -u' value)" logger.critical(errMsg) raise SystemExit elif "can't allocate read lock" in excMsg: errMsg = "there has been a problem in regular socket operation " errMsg += "('%s')" % excMsg.strip().split('\n')[-1] logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("pymysql", "configparser")): errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("ntlm", "socket.error, err", "SyntaxError")): errMsg = "wrong initialization of python-ntlm detected (using Python2 syntax)" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("drda", "to_bytes")): errMsg = "wrong initialization of drda detected (using Python3 syntax)" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("window = tkinter.Tk()", )): errMsg = "there has been a problem in initialization of GUI interface " errMsg += "('%s')" % excMsg.strip().split('\n')[-1] logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("unable to access item 'liveTest'", )): errMsg = "detected usage of files from different versions of sqlmap" logger.critical(errMsg) raise SystemExit elif kb.get("dumpKeyboardInterrupt"): raise SystemExit elif any(_ in excMsg for _ in ("Broken pipe", )): raise SystemExit elif valid is False: errMsg = "code integrity check failed (turning off automatic issue creation). " errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.critical(errMsg) print() dataToStdout(excMsg) raise SystemExit elif any(_ in excMsg for _ in ("tamper/", "waf/")): logger.critical(errMsg) print() dataToStdout(excMsg) raise SystemExit elif any(_ in excMsg for _ in ("ImportError", "ModuleNotFoundError", "Can't find file for module", "SAXReaderNotAvailable", "source code string cannot contain null bytes", "No module named", "tp_name field")): errMsg = "invalid runtime environment ('%s')" % excMsg.split( "Error: ")[-1].strip() logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("SyntaxError: Non-ASCII character", ".py on line", "but no encoding declared")): errMsg = "invalid runtime environment ('%s')" % excMsg.split( "Error: ")[-1].strip() logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("No such file", "_'")): errMsg = "corrupted installation detected ('%s'). " % excMsg.strip( ).split('\n')[-1] errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("HTTPNtlmAuthHandler", "'str' object has no attribute 'decode'")): errMsg = "package 'python-ntlm' has a known compatibility issue with the " errMsg += "Python 3 (Reference: 'https://github.com/mullender/python-ntlm/pull/61')" logger.critical(errMsg) raise SystemExit elif "'DictObject' object has no attribute '" in excMsg and all( _ in errMsg for _ in ("(fingerprinted)", "(identified)")): errMsg = "there has been a problem in enumeration. " errMsg += "Because of a considerable chance of false-positive case " errMsg += "you are advised to rerun with switch '--flush-session'" logger.critical(errMsg) raise SystemExit elif "bad marshal data (unknown type code)" in excMsg: match = re.search(r"\s*(.+)\s+ValueError", excMsg) errMsg = "one of your .pyc files are corrupted%s" % ( " ('%s')" % match.group(1) if match else "") errMsg += ". Please delete .pyc files on your system to fix the problem" logger.critical(errMsg) raise SystemExit for match in re.finditer(r'File "(.+?)", line', excMsg): file_ = match.group(1) try: file_ = os.path.relpath(file_, os.path.dirname(__file__)) except ValueError: pass file_ = file_.replace("\\", '/') if "../" in file_: file_ = re.sub(r"(\.\./)+", '/', file_) else: file_ = file_.lstrip('/') file_ = re.sub(r"/{2,}", '/', file_) excMsg = excMsg.replace(match.group(1), file_) errMsg = maskSensitiveData(errMsg) excMsg = maskSensitiveData(excMsg) if conf.get("api") or not valid: logger.critical("%s\n%s" % (errMsg, excMsg)) else: logger.critical(errMsg) dataToStdout("%s\n" % setColor(excMsg.strip(), level=logging.CRITICAL)) createGithubIssue(errMsg, excMsg) finally: kb.threadContinue = False if getDaysFromLastUpdate() > LAST_UPDATE_NAGGING_DAYS: warnMsg = "your sqlmap version is outdated" logger.warn(warnMsg) if conf.get("showTime"): dataToStdout("\n[*] ending @ %s\n\n" % time.strftime("%X /%Y-%m-%d/"), forceOutput=True) kb.threadException = True if kb.get("tempDir"): for prefix in (MKSTEMP_PREFIX.IPC, MKSTEMP_PREFIX.TESTING, MKSTEMP_PREFIX.COOKIE_JAR, MKSTEMP_PREFIX.BIG_ARRAY): for filepath in glob.glob( os.path.join(kb.tempDir, "%s*" % prefix)): try: os.remove(filepath) except OSError: pass if not filterNone( filepath for filepath in glob.glob(os.path.join(kb.tempDir, '*')) if not any( filepath.endswith(_) for _ in (".lock", ".exe", ".so", '_'))): # ignore junk files try: shutil.rmtree(kb.tempDir, ignore_errors=True) except OSError: pass if conf.get("hashDB"): conf.hashDB.flush(True) if conf.get("harFile"): try: with openFile(conf.harFile, "w+b") as f: json.dump(conf.httpCollector.obtain(), fp=f, indent=4, separators=(',', ': ')) except SqlmapBaseException as ex: errMsg = getSafeExString(ex) logger.critical(errMsg) if conf.get("api"): conf.databaseCursor.disconnect() if conf.get("dumper"): conf.dumper.flush() # short delay for thread finalization _ = time.time() while threading.activeCount() > 1 and ( time.time() - _) > THREAD_FINALIZATION_TIMEOUT: time.sleep(0.01) if cmdLineOptions.get("sqlmapShell"): cmdLineOptions.clear() conf.clear() kb.clear() conf.disableBanner = True main()
def test_readThreadSafe(self): """ Tests for race conditions. Reading n_threads (currently 30) times the same waveform file in parallel and compare the results which must be all the same. """ data = np.arange(0, 500) start = UTCDateTime(2009, 1, 13, 12, 1, 2, 999000) formats = _getEntryPoints('obspy.plugin.waveform', 'writeFormat') for format in formats: # XXX: skip SEGY and SU formats for now as they need some special # headers. if format in ['SEGY', 'SU', 'SEG2']: continue dt = np.dtype("int") if format in ('MSEED', 'GSE2'): dt = "int32" tr = Trace(data=data.astype(dt)) tr.stats.network = "BW" tr.stats.station = "MANZ1" tr.stats.location = "00" tr.stats.channel = "EHE" tr.stats.calib = 0.999999 tr.stats.delta = 0.005 tr.stats.starttime = start # create waveform file with given format and byte order with NamedTemporaryFile() as tf: outfile = tf.name tr.write(outfile, format=format) if format == 'Q': outfile += '.QHD' n_threads = 30 streams = [] def testFunction(streams): st = read(outfile, format=format) streams.append(st) # Read the ten files at one and save the output in the just # created class. for _i in xrange(n_threads): thread = threading.Thread(target=testFunction, args=(streams, )) thread.start() # Loop until all threads are finished. start = time.time() while True: if threading.activeCount() == 1: break # Avoid infinite loop and leave after 120 seconds # such a long time is needed for debugging with valgrind elif time.time() - start >= 120: # pragma: no cover msg = 'Not all threads finished!' raise Warning(msg) # Compare all values which should be identical and clean up # files #for data in : # np.testing.assert_array_equal(values, original) if format == 'Q': os.remove(outfile[:-4] + '.QBN') os.remove(outfile[:-4] + '.QHD')
import threading import time def func(): print('ran') time.sleep(3) print("done") time.sleep(0.85) print("now done") x = threading.Thread(target=func) x.start() print(threading.activeCount()) time.sleep(1) print("finally")
def start(target, users, passwds, ports, timeout_sec, thread_number, num, total, log_in_file, time_sleep, language, verbose_level, socks_proxy, retries, methods_args, scan_id, scan_cmd): # Main function if target_type(target) != 'SINGLE_IPv4' or target_type(target) != 'DOMAIN' or target_type(target) != 'HTTP': # requirements check new_extra_requirements = extra_requirements_dict() if methods_args is not None: for extra_requirement in extra_requirements_dict(): if extra_requirement in methods_args: new_extra_requirements[ extra_requirement] = methods_args[extra_requirement] extra_requirements = new_extra_requirements if ports is None: ports = extra_requirements["Proftpd_vuln_ports"] if target_type(target) == 'HTTP': target = target_to_host(target) threads = [] total_req = len(ports) thread_tmp_filename = '{}/tmp/thread_tmp_'.format(load_file_path()) + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) __log_into_file(thread_tmp_filename, 'w', '1', language) trying = 0 keyboard_interrupt_flag = False for port in ports: port = int(port) t = threading.Thread(target=__cpu_consumption, args=(target, int(port), timeout_sec, log_in_file, language, time_sleep, thread_tmp_filename, socks_proxy, scan_id, scan_cmd)) threads.append(t) t.start() trying += 1 if verbose_level > 3: info( messages(language, "trying_message").format(trying, total_req, num, total, target, port, 'Proftpd_cpu_consumption_vuln')) while 1: try: if threading.activeCount() >= thread_number: time.sleep(0.01) else: break except KeyboardInterrupt: keyboard_interrupt_flag = True break if keyboard_interrupt_flag: break # wait for threads kill_switch = 0 kill_time = int( timeout_sec / 0.1) if int(timeout_sec / 0.1) != 0 else 1 while 1: time.sleep(0.1) kill_switch += 1 try: if threading.activeCount() == 1 or kill_switch == kill_time: break except KeyboardInterrupt: break thread_write = int(open(thread_tmp_filename).read().rsplit()[0]) if thread_write == 1 and verbose_level != 0: info(messages(language, "no_vulnerability_found").format( 'ProFTPd CPU consumption CVE-2008-7265')) data = json.dumps({'HOST': target, 'USERNAME': '', 'PASSWORD': '', 'PORT': '', 'TYPE': 'Proftpd_cpu_consumption_vuln', 'DESCRIPTION': messages(language, "no_vulnerability_found").format('ProFTPd CPU consumption CVE-2008-7265'), 'TIME': now(), 'CATEGORY': "scan", 'SCAN_ID': scan_id, 'SCAN_CMD': scan_cmd}) __log_into_file(log_in_file, 'a', data, language) os.remove(thread_tmp_filename) else: warn(messages(language, "input_target_error").format( 'Proftpd_cpu_consumption_vuln', target))
def start(target, users, passwds, ports, timeout_sec, thread_number, num, total, log_in_file, time_sleep, language, verbose_level, socks_proxy, retries, methods_args, scan_id, scan_cmd): # Main function if target_type(target) != 'SINGLE_IPv4' or target_type(target) != 'DOMAIN' or target_type(target) == 'HTTP': # requirements check new_extra_requirements = extra_requirements_dict() if methods_args is not None: for extra_requirement in extra_requirements_dict(): if extra_requirement in methods_args: new_extra_requirements[ extra_requirement] = methods_args[extra_requirement] extra_requirements = new_extra_requirements if users is None: users = extra_requirements["ssh_brute_users"] if passwds is None: passwds = extra_requirements["ssh_brute_passwds"] if ports is None: ports = extra_requirements["ssh_brute_ports"] if target_type(target) == 'HTTP': target = target_to_host(target) threads = [] total_req = len(users) * len(passwds) thread_tmp_filename = '{}/tmp/thread_tmp_'.format(load_file_path()) + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) ports_tmp_filename = '{}/tmp/ports_tmp_'.format(load_file_path()) + ''.join( random.choice(string.ascii_letters + string.digits) for _ in range(20)) __log_into_file(thread_tmp_filename, 'w', '1', language) __log_into_file(ports_tmp_filename, 'w', '', language) trying = 0 ports = test_ports(ports, timeout_sec, target, retries, language, num, total, time_sleep, ports_tmp_filename, thread_number, total_req, verbose_level, socks_proxy) keyboard_interrupt_flag = False for port in ports: # test ssh portflag = True exit = 0 port = int(port) for user in users: for passwd in passwds: t = threading.Thread(target=login, args=(user, passwd, target, port, timeout_sec, log_in_file, language, retries, time_sleep, thread_tmp_filename, socks_proxy, scan_id, scan_cmd)) threads.append(t) t.start() trying += 1 if verbose_level > 3: info(messages(language, "trying_message").format( trying, total_req, num, total, target, port, 'ssh_brute')) while 1: try: if threading.activeCount() >= thread_number: time.sleep(0.01) else: break except KeyboardInterrupt: keyboard_interrupt_flag = True break if keyboard_interrupt_flag: break else: break else: break # wait for threads kill_switch = 0 kill_time = int( timeout_sec / 0.1) if int(timeout_sec / 0.1) is not 0 else 1 while 1: time.sleep(0.1) kill_switch += 1 try: if threading.activeCount() is 1 or kill_switch is kill_time: break except KeyboardInterrupt: break thread_write = int(open(thread_tmp_filename).read().rsplit()[0]) if thread_write is 1 and verbose_level is not 0: data = json.dumps({'HOST': target, 'USERNAME': '', 'PASSWORD': '', 'PORT': '', 'TYPE': 'ssh_brute', 'DESCRIPTION': messages(language, "no_user_passwords"), 'TIME': now(), 'CATEGORY': "brute", 'SCAN_ID': scan_id, 'SCAN_CMD': scan_cmd}) + "\n" __log_into_file(log_in_file, 'a', data, language) os.remove(thread_tmp_filename) else: warn(messages(language, "input_target_error").format('ssh_brute', target))
def runThreads(numThreads, threadFunction, cleanupFunction=None, forwardException=True, threadChoice=False, startThreadMsg=True): threads = [] kb.multipleCtrlC = False kb.threadContinue = True kb.threadException = False kb.technique = ThreadData.technique if threadChoice and conf.threads == numThreads == 1 and not ( kb.injection.data and not any( _ not in (PAYLOAD.TECHNIQUE.TIME, PAYLOAD.TECHNIQUE.STACKED) for _ in kb.injection.data)): while True: message = "please enter number of threads? [Enter for %d (current)] " % numThreads choice = readInput(message, default=str(numThreads)) if choice: skipThreadCheck = False if choice.endswith('!'): choice = choice[:-1] skipThreadCheck = True if isDigit(choice): if int(choice ) > MAX_NUMBER_OF_THREADS and not skipThreadCheck: errMsg = "maximum number of used threads is %d avoiding potential connection issues" % MAX_NUMBER_OF_THREADS logger.critical(errMsg) else: conf.threads = numThreads = int(choice) break if numThreads == 1: warnMsg = "running in a single-thread mode. This could take a while" logger.warn(warnMsg) try: if numThreads > 1: if startThreadMsg: infoMsg = "starting %d threads" % numThreads logger.info(infoMsg) else: threadFunction() return # Start the threads for numThread in xrange(numThreads): thread = threading.Thread(target=exceptionHandledFunction, name=str(numThread), args=[threadFunction]) setDaemon(thread) try: thread.start() except Exception as ex: errMsg = "error occurred while starting new thread ('%s')" % ex logger.critical(errMsg) break threads.append(thread) # And wait for them to all finish alive = True while alive: alive = False for thread in threads: if thread.is_alive(): alive = True time.sleep(0.1) except (KeyboardInterrupt, SqlmapUserQuitException) as ex: print() kb.prependFlag = False kb.threadContinue = False kb.threadException = True if kb.lastCtrlCTime and (time.time() - kb.lastCtrlCTime < 1): kb.multipleCtrlC = True raise SqlmapUserQuitException( "user aborted (Ctrl+C was pressed multiple times)") kb.lastCtrlCTime = time.time() if numThreads > 1: logger.info("waiting for threads to finish%s" % (" (Ctrl+C was pressed)" if isinstance( ex, KeyboardInterrupt) else "")) try: while (threading.activeCount() > 1): pass except KeyboardInterrupt: kb.multipleCtrlC = True raise SqlmapThreadException( "user aborted (Ctrl+C was pressed multiple times)") if forwardException: raise except (SqlmapConnectionException, SqlmapValueException) as ex: print() kb.threadException = True logger.error("thread %s: '%s'" % (threading.currentThread().getName(), ex)) if conf.get("verbose") > 1 and isinstance(ex, SqlmapValueException): traceback.print_exc() except: print() if not kb.multipleCtrlC: from lib.core.common import unhandledExceptionMessage kb.threadException = True errMsg = unhandledExceptionMessage() logger.error("thread %s: %s" % (threading.currentThread().getName(), errMsg)) traceback.print_exc() finally: kb.threadContinue = True kb.threadException = False kb.technique = None for lock in kb.locks.values(): if lock.locked(): try: lock.release() except: pass if conf.get("hashDB"): conf.hashDB.flush(True) if cleanupFunction: cleanupFunction()
def daemonize(self, chdir='/', umask=0): '''Daemonize a process using a double fork This method will fork the current process to create a daemon process. It will perform a double fork(2), chdir(2) to the given folder (or not chdir at all if the C{chdir} argument is C{None}), and set the new process umask(2) to the value of the C{umask} argument, or not reset it if this argument is -1. While forking, a setsid(2) call will be done to become session leader and detach from the controlling TTY. In the child process, all existing file descriptors will be closed, including stdin, stdout and stderr, which will be re-opened to /dev/null. The method returns a tuple<bool, number>. If the first item is True, the current process is the daemonized process. If it is False, the current process is the process which called the C{daemonize} method, which can most likely be closed now. The second item is the PID of the current process. @attention: Make sure you know really well what fork(2) does before using this method @param chdir: Path to chdir(2) to after forking. Set to None to disable chdir'ing @type chdir: string or None @param umask: Umask to set after forking. Set to -1 not to set umask @type umask: number @returns: Daemon status and PID @rtype: tuple<bool, number> @raise RuntimeError: System does not support fork(2) ''' # We display a warning here when threads are discovered in the current # process, because forking a threaded application is a pretty bad idea. # This is not an in-depth check, since it only checks whether any # threads were created using threading.Thread. On HPUX and maybe some # other UNIXes we could use pthread_is_multithreaded_np, but this is # not available on Linux at least. if not hasattr(os, 'fork'): raise j.exceptions.RuntimeError( 'os.fork not found, daemon mode not supported on your system') import threading if threading.activeCount() > 1: j.errorhandler.raiseWarning( 'You application got running threads, this can cause issues when using fork') pid = os.fork() if pid == 0: # First child # Become session leader... os.setsid() # Double fork pid = os.fork() if pid == 0: # Second child if umask >= 0: os.umask(umask) if chdir: os.chdir(chdir) else: # First child is useless now os._exit(0) else: return False, os.getpid() # Close all FDs import resource maxfd = resource.getrlimit(resource.RLIMIT_NOFILE)[1] if maxfd == resource.RLIM_INFINITY: maxfd = 1024 sys.stdin.close() sys.stdout.close() sys.stderr.close() for fd in range(maxfd): try: os.close(fd) except OSError: pass # Open fd0 to /dev/null redirect = getattr(os, 'devnull', '/dev/null') os.open(redirect, os.O_RDWR) # dup to stdout and stderr os.dup2(0, 1) os.dup2(0, 2) return True, os.getpid()
now_time = int(time.time()) connection.send(data) # 向客户端发送一个字符串信息 connection.close() os.mkdir(str(now_time)) inputfile = str(now_time) + '/' + str(now_time) + '.' + url_1.split( '.')[-1] outfile = str(now_time) + '/' + str(now_time) + '.webp' with urllib.request.urlopen(url_1, timeout=30) as response, open( inputfile, 'wb') as f_save: f_save.write(response.read()) f_save.flush() f_save.close() myqueue = queue.Queue() myqueue.put(inputfile) # while not myqueue.empty(): if threading.activeCount() < 3: input_file = myqueue.get() if '.pdf' in input_file: outfile = str(now_time) th_1 = threading.Thread(target=pdftowebp, args=(input_file, outfile, get_data)) th_1.start() elif '.doc' in input_file or '.docx' in input_file: path_1 = os.getcwd() th_1 = threading.Thread( target=wordtowebp, args=(path_1 + '\\' + str(now_time) + '\\' + str(now_time) + '.' + input_file.split('.')[-1], path_1 + '\\' + str(now_time) + '\\' + str(now_time) + '.pdf', get_data)) th_1.start()
puerto_cliente = direccion[1] print('Se acepto una conexion desde {}:{}'.format( direccion[0], direccion[1])) ##Esto inicia el threading de la comunicacion para un solo cliente thread_cliente = threading.Thread( target=manejador_conexion, args=( socket__conexion_servidor_cliente, nombre_cliente, puerto_cliente, ) #con la coma!! ) thread_cliente.start() #Se espera 15 segundos para ver actividad en el thread, o se cierra thread_cliente.join(15) if thread_cliente.is_alive(): e.set() thread_cliente.join() ########################################################## # Ahora si empieza el programa ########################################################## #Se crea el thread para manejar clientes thread_generador_clientes = threading.Thread(target=manejador_clientes) thread_generador_clientes.start() #Se mantiene vivo un conteo de threads. while True: print('Numero de threads activos: {}'.format(threading.activeCount())) time.sleep(3)
pubnub.unsubscribe({'channel': 'efgh1'}) def cb4(): pubnub.unsubscribe({'channel': 'abcd1'}) def subscribe(channel): pubnub.subscribe({ 'channel': channel, 'connect': connected, 'callback': message_received }) print threading.activeCount() pubnub.timeout(15, cb1) pubnub.timeout(30, cb2) pubnub.timeout(45, cb3) pubnub.timeout(60, cb4) #''' for x in range(1, 1000): #print x def y(t): subscribe('channel-' + str(t))
def thg_check(self, args): """Execute module check""" if not self.module_instance: raise ModuleNotUseException() [validate_result, validate_message] = self.module_instance.options.validate() if not validate_result: for error in validate_message: self._print_item(error, Fore.RED) return False if self.module_instance.multi_target: target_type = self.module_instance.target_type target_field = None if target_type == "tcp": target_field = "HOST" elif target_type == "http": target_field = "URL" elif target_type == "query": target_field = "QUERY" elif target_type == "crawler": target_type = "CRAWLER" elif target_type == "url": target_field = "URL" target_filename = self.module_instance.options.get_option( target_field) try: target_file = open(target_filename, 'r') self.module_instance.targets = [] for line in target_file.readlines(): self.module_instance.targets.append(line.strip()) self.module_instance.multi_target = True except IOError as e: self._print_item(e, color=Fore.RED) return False targets = self.module_instance.targets targets_queue = Queue() for target in targets: targets_queue.put(target) if not targets_queue.empty(): thread_count = int( self.module_instance.options.get_option("THREADS")) thread_queue = Queue(maxsize=thread_count) try: while not targets_queue.empty(): while thread_queue.full(): time.sleep(0.1) target = targets_queue.get() thread_queue.put(1) _thread = threading.Thread(target=self.check_thread, args=(target, target_type, thread_queue)) _thread.start() while not thread_queue.empty(): time.sleep(0.1) except KeyboardInterrupt: self._print_item( "Wait for existing process to exit...", color=Fore.RED) while threading.activeCount() > 1: time.sleep(0.5) return None self.poutput("{style}[*]{style_end} module execution completed".format( style=Fore.BLUE + Style.BRIGHT, style_end=Style.RESET_ALL )) return None exploit_result = self.module_instance.check() if exploit_result is None: self._print_item("Check Error: check function no results returned") return None if exploit_result.status: self._print_item("Check success!") self._print_item(exploit_result.success_message) else: self._print_item("Exploit failure!", color=Fore.RED) self._print_item(exploit_result.error_message, color=Fore.RED) self.poutput("{style}[*]{style_end} module execution completed".format( style=Fore.BLUE + Style.BRIGHT, style_end=Style.RESET_ALL ))
def main(): """ Main function of sqlmap when running from command line. """ try: checkEnvironment() setPaths(modulePath()) banner() # Store original command line options for possible later restoration cmdLineOptions.update(cmdLineParser().__dict__) initOptions(cmdLineOptions) if conf.get("api"): # heavy imports from lib.utils.api import StdDbOut from lib.utils.api import setRestAPILog # Overwrite system standard output and standard error to write # to an IPC database sys.stdout = StdDbOut(conf.taskid, messagetype="stdout") sys.stderr = StdDbOut(conf.taskid, messagetype="stderr") setRestAPILog() conf.showTime = True dataToStdout("[!] legal disclaimer: %s\n\n" % LEGAL_DISCLAIMER, forceOutput=True) dataToStdout("[*] starting at %s\n\n" % time.strftime("%X"), forceOutput=True) init() if conf.profile: profile() elif conf.smokeTest: smokeTest() elif conf.liveTest: liveTest() else: try: start() except thread.error as ex: if "can't start new thread" in getSafeExString(ex): errMsg = "unable to start new threads. Please check OS (u)limits" logger.critical(errMsg) raise SystemExit else: raise except SqlmapUserQuitException: errMsg = "user quit" try: logger.error(errMsg) except KeyboardInterrupt: pass except (SqlmapSilentQuitException, bdb.BdbQuit): pass except SqlmapShellQuitException: cmdLineOptions.sqlmapShell = False except SqlmapBaseException as ex: errMsg = getSafeExString(ex) try: logger.critical(errMsg) except KeyboardInterrupt: pass raise SystemExit except KeyboardInterrupt: print errMsg = "user aborted" try: logger.error(errMsg) except KeyboardInterrupt: pass except EOFError: print errMsg = "exit" try: logger.error(errMsg) except KeyboardInterrupt: pass except SystemExit: pass except: print errMsg = unhandledExceptionMessage() excMsg = traceback.format_exc() valid = checkIntegrity() try: if valid is False: errMsg = "code integrity check failed (turning off automatic issue creation). " errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.critical(errMsg) print dataToStdout(excMsg) raise SystemExit elif "tamper/" in excMsg: logger.critical(errMsg) print dataToStdout(excMsg) raise SystemExit elif "MemoryError" in excMsg: errMsg = "memory exhaustion detected" logger.error(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded")): errMsg = "no space left on output device" logger.error(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("No such file", "_'", "self.get_prog_name()")): errMsg = "corrupted installation detected ('%s'). " % excMsg.strip( ).split('\n')[-1] errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.error(errMsg) raise SystemExit elif "Read-only file system" in excMsg: errMsg = "output device is mounted as read-only" logger.error(errMsg) raise SystemExit elif "OperationalError: disk I/O error" in excMsg: errMsg = "I/O error on output device" logger.error(errMsg) raise SystemExit elif "_mkstemp_inner" in excMsg: errMsg = "there has been a problem while accessing temporary files" logger.error(errMsg) raise SystemExit elif "can't start new thread" in excMsg: errMsg = "there has been a problem while creating new thread instance. " errMsg += "Please make sure that you are not running too many processes" if not IS_WIN: errMsg += " (or increase the 'ulimit -u' value)" logger.error(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("pymysql", "configparser")): errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)" logger.error(errMsg) raise SystemExit elif "bad marshal data (unknown type code)" in excMsg: match = re.search(r"\s*(.+)\s+ValueError", excMsg) errMsg = "one of your .pyc files are corrupted%s" % ( " ('%s')" % match.group(1) if match else "") errMsg += ". Please delete .pyc files on your system to fix the problem" logger.error(errMsg) raise SystemExit elif "valueStack.pop" in excMsg and kb.get( "dumpKeyboardInterrupt"): raise SystemExit for match in re.finditer(r'File "(.+?)", line', excMsg): file_ = match.group(1) file_ = os.path.relpath(file_, os.path.dirname(__file__)) file_ = file_.replace("\\", '/') file_ = re.sub(r"\.\./", '/', file_).lstrip('/') excMsg = excMsg.replace(match.group(1), file_) errMsg = maskSensitiveData(errMsg) excMsg = maskSensitiveData(excMsg) if conf.get("api") or not valid: logger.critical("%s\n%s" % (errMsg, excMsg)) else: logger.critical(errMsg) kb.stickyLevel = logging.CRITICAL dataToStdout(excMsg) createGithubIssue(errMsg, excMsg) except KeyboardInterrupt: pass finally: kb.threadContinue = False if conf.get("showTime"): dataToStdout("\n[*] shutting down at %s\n\n" % time.strftime("%X"), forceOutput=True) kb.threadException = True if kb.get("tempDir"): for prefix in (MKSTEMP_PREFIX.IPC, MKSTEMP_PREFIX.TESTING, MKSTEMP_PREFIX.COOKIE_JAR, MKSTEMP_PREFIX.BIG_ARRAY): for filepath in glob.glob( os.path.join(kb.tempDir, "%s*" % prefix)): try: os.remove(filepath) except OSError: pass if not filter( None, (filepath for filepath in glob.glob(os.path.join(kb.tempDir, '*')) if not any( filepath.endswith(_) for _ in ('.lock', '.exe', '_')))): shutil.rmtree(kb.tempDir, ignore_errors=True) if conf.get("hashDB"): try: conf.hashDB.flush(True) except KeyboardInterrupt: pass if cmdLineOptions.get("sqlmapShell"): cmdLineOptions.clear() conf.clear() kb.clear() main() if conf.get("api"): try: conf.databaseCursor.disconnect() except KeyboardInterrupt: pass if conf.get("dumper"): conf.dumper.flush() # short delay for thread finalization try: _ = time.time() while threading.activeCount() > 1 and ( time.time() - _) > THREAD_FINALIZATION_TIMEOUT: time.sleep(0.01) except KeyboardInterrupt: pass finally: # Reference: http://stackoverflow.com/questions/1635080/terminate-a-multi-thread-python-program if threading.activeCount() > 1: os._exit(0)
def t_join(t_num): while True: time.sleep(1) ac_num = threading.activeCount() if ac_num <= 1 or queue.empty(): break
__author__ = 'Administrator' import threading, time def run(n): semaphore.acquire() time.sleep(3) print("run the thead%s" % n) semaphore.release() if __name__ == "__main__": num = 0 semaphore = threading.BoundedSemaphore(5) #最多允许5个线程同时执行 for i in range(23): t = threading.Thread(target=run, args=(i, )) t.start() while threading.activeCount() != 1: pass else: print("all theads is done")
def main(): """ Main function of sqlmap when running from command line. """ try: dirtyPatches() checkEnvironment() setPaths(modulePath()) banner() # Store original command line options for possible later restoration cmdLineOptions.update(cmdLineParser().__dict__) initOptions(cmdLineOptions) if checkPipedInput(): conf.batch = True if conf.get("api"): # heavy imports from lib.utils.api import StdDbOut from lib.utils.api import setRestAPILog # Overwrite system standard output and standard error to write # to an IPC database sys.stdout = StdDbOut(conf.taskid, messagetype="stdout") sys.stderr = StdDbOut(conf.taskid, messagetype="stderr") setRestAPILog() conf.showTime = True dataToStdout("[!] legal disclaimer: %s\n\n" % LEGAL_DISCLAIMER, forceOutput=True) dataToStdout("[*] starting @ %s\n\n" % time.strftime("%X /%Y-%m-%d/"), forceOutput=True) init() if not conf.updateAll: # Postponed imports (faster start) if conf.smokeTest: from lib.core.testing import smokeTest smokeTest() elif conf.liveTest: from lib.core.testing import liveTest liveTest() else: from lib.controller.controller import start if conf.profile and PY2: from lib.core.profiling import profile globals()["start"] = start profile() else: try: start() except Exception as ex: if "can't start new thread" in getSafeExString(ex): errMsg = "unable to start new threads. Please check OS (u)limits" logger.critical(errMsg) raise SystemExit else: raise except SqlmapUserQuitException: if not conf.batch: errMsg = "user quit" logger.error(errMsg) except (SqlmapSilentQuitException, bdb.BdbQuit): pass except SqlmapShellQuitException: cmdLineOptions.sqlmapShell = False except SqlmapBaseException as ex: errMsg = getSafeExString(ex) logger.critical(errMsg) raise SystemExit except KeyboardInterrupt: print() except EOFError: print() errMsg = "exit" logger.error(errMsg) except SystemExit: pass except: print() errMsg = unhandledExceptionMessage() excMsg = traceback.format_exc() valid = checkIntegrity() if valid is False: errMsg = "code integrity check failed (turning off automatic issue creation). " errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.critical(errMsg) print() dataToStdout(excMsg) raise SystemExit elif any(_ in excMsg for _ in ("tamper/", "waf/")): logger.critical(errMsg) print() dataToStdout(excMsg) raise SystemExit elif any(_ in excMsg for _ in ("ImportError", "Can't find file for module")): errMsg = "invalid runtime environment ('%s')" % excMsg.split( "Error: ")[-1].strip() logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("MemoryError", "Cannot allocate memory")): errMsg = "memory exhaustion detected" logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("No space left", "Disk quota exceeded", "Disk full while accessing")): errMsg = "no space left on output device" logger.critical(errMsg) raise SystemExit elif any(_ in excMsg for _ in ("The paging file is too small", )): errMsg = "no space left for paging file" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("No such file", "_'", "self.get_prog_name()")): errMsg = "corrupted installation detected ('%s'). " % excMsg.strip( ).split('\n')[-1] errMsg += "You should retrieve the latest development version from official GitHub " errMsg += "repository at '%s'" % GIT_PAGE logger.critical(errMsg) raise SystemExit elif "Read-only file system" in excMsg: errMsg = "output device is mounted as read-only" logger.critical(errMsg) raise SystemExit elif "OperationalError: disk I/O error" in excMsg: errMsg = "I/O error on output device" logger.critical(errMsg) raise SystemExit elif "Violation of BIDI" in excMsg: errMsg = "invalid URL (violation of Bidi IDNA rule - RFC 5893)" logger.critical(errMsg) raise SystemExit elif "_mkstemp_inner" in excMsg: errMsg = "there has been a problem while accessing temporary files" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("twophase", "sqlalchemy")): errMsg = "please update the 'sqlalchemy' package (>= 1.1.11) " errMsg += "(Reference: https://qiita.com/tkprof/items/7d7b2d00df9c5f16fffe)" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("scramble_caching_sha2", "TypeError")): errMsg = "please downgrade the 'PyMySQL' package (=< 0.8.1) " errMsg += "(Reference: https://github.com/PyMySQL/PyMySQL/issues/700)" logger.critical(errMsg) raise SystemExit elif "must be pinned buffer, not bytearray" in excMsg: errMsg = "error occurred at Python interpreter which " errMsg += "is fixed in 2.7.x. Please update accordingly " errMsg += "(Reference: https://bugs.python.org/issue8104)" logger.critical(errMsg) raise SystemExit elif "can't start new thread" in excMsg: errMsg = "there has been a problem while creating new thread instance. " errMsg += "Please make sure that you are not running too many processes" if not IS_WIN: errMsg += " (or increase the 'ulimit -u' value)" logger.critical(errMsg) raise SystemExit elif "'DictObject' object has no attribute '" in excMsg and all( _ in errMsg for _ in ("(fingerprinted)", "(identified)")): errMsg = "there has been a problem in enumeration. " errMsg += "Because of a considerable chance of false-positive case " errMsg += "you are advised to rerun with switch '--flush-session'" logger.critical(errMsg) raise SystemExit elif all(_ in excMsg for _ in ("pymysql", "configparser")): errMsg = "wrong initialization of pymsql detected (using Python3 dependencies)" logger.critical(errMsg) raise SystemExit elif "bad marshal data (unknown type code)" in excMsg: match = re.search(r"\s*(.+)\s+ValueError", excMsg) errMsg = "one of your .pyc files are corrupted%s" % ( " ('%s')" % match.group(1) if match else "") errMsg += ". Please delete .pyc files on your system to fix the problem" logger.critical(errMsg) raise SystemExit elif kb.get("dumpKeyboardInterrupt"): raise SystemExit elif any(_ in excMsg for _ in ("Broken pipe", )): raise SystemExit for match in re.finditer(r'File "(.+?)", line', excMsg): file_ = match.group(1) file_ = os.path.relpath(file_, os.path.dirname(__file__)) file_ = file_.replace("\\", '/') if "../" in file_: file_ = re.sub(r"(\.\./)+", '/', file_) else: file_ = file_.lstrip('/') file_ = re.sub(r"/{2,}", '/', file_) excMsg = excMsg.replace(match.group(1), file_) errMsg = maskSensitiveData(errMsg) excMsg = maskSensitiveData(excMsg) if conf.get("api") or not valid: logger.critical("%s\n%s" % (errMsg, excMsg)) else: logger.critical(errMsg) dataToStdout("%s\n" % setColor(excMsg.strip(), level=logging.CRITICAL)) createGithubIssue(errMsg, excMsg) finally: kb.threadContinue = False if conf.get("showTime"): dataToStdout("\n[*] ending @ %s\n\n" % time.strftime("%X /%Y-%m-%d/"), forceOutput=True) kb.threadException = True if kb.get("tempDir"): for prefix in (MKSTEMP_PREFIX.IPC, MKSTEMP_PREFIX.TESTING, MKSTEMP_PREFIX.COOKIE_JAR, MKSTEMP_PREFIX.BIG_ARRAY): for filepath in glob.glob( os.path.join(kb.tempDir, "%s*" % prefix)): try: os.remove(filepath) except OSError: pass if not filterNone( filepath for filepath in glob.glob(os.path.join(kb.tempDir, '*')) if not any( filepath.endswith(_) for _ in ('.lock', '.exe', '_'))): shutil.rmtree(kb.tempDir, ignore_errors=True) if conf.get("hashDB"): conf.hashDB.flush(True) if conf.get("harFile"): with openFile(conf.harFile, "w+b") as f: json.dump(conf.httpCollector.obtain(), fp=f, indent=4, separators=(',', ': ')) if conf.get("api"): conf.databaseCursor.disconnect() if conf.get("dumper"): conf.dumper.flush() # short delay for thread finalization _ = time.time() while threading.activeCount() > 1 and ( time.time() - _) > THREAD_FINALIZATION_TIMEOUT: time.sleep(0.01) if cmdLineOptions.get("sqlmapShell"): cmdLineOptions.clear() conf.clear() kb.clear() conf.disableBanner = True main()