def semanticParse(source, api_url): """ Parse the source text quasi-semantically and query MediaWiki API (PHP) to check the existence of a certain article (including redirects) Use multiple threads to speed up API querying Return candidate titles which exist on MediaWiki sites """ length = len(source) # Number of characters candidates = [] # Candidate titles r = [] q = Queue() # Queue for multithreading for i in range(length): start_index = i end_index = length t = Thread(target=_wordParse, args=(start_index, end_index, api_url, source, q)) t.start() q.join() try: while True: candidates.append(q.get(True,2)) # Get all items in queue with timeout (1 second) except: pass #print candidates return candidates
def main(): """ Main function of the proxy scanner. """ global pl, output, q parser = ArgumentParser(description='Scans a list of proxies to determine which work for HTTPS.') parser.add_argument('--output', default='output/proxies.txt', type=str, help='The file in which to store the found proxies.') parser.add_argument('--threads', default=10, type=int, help='Number of threads to use.') args = parser.parse_args() output = args.output threads = args.threads q = Queue(threads * 3) print 'Starting threads.' for x in xrange(threads): t = Thread(target=check_proxies) t.daemon = True t.start() print 'Queueing proxies.' for proxy in proxies.proxies: q.put(proxy) q.join() save_proxies()
class Worker: def __init__(self): self.q = Queue() self.t = Thread(target=self._handle) self.t.setDaemon(True) self.t.start() def _handle(self): while True: reset_caches() fn = self.q.get() try: fn() self.q.task_done() except: import traceback print traceback.format_exc() def do(self, fn, *a, **kw): fn1 = lambda: fn(*a, **kw) self.q.put(fn1) def join(self): self.q.join()
class ThreadPool(object): def __init__(self, num): """num: 线程池中的线程数.""" self.tasks = Queue() #存储任务的队列 self.pool = [] for i in range(num): self.pool.append(ThreadPoolSlot(self.tasks)) #创建了线程池 def spawn(self, func, *args): """ 向队列中添加任务 """ self.tasks.put((func, args)) def joinall(self): """等待队列中所有的任务完成.""" self.tasks.join() """等待所有线程结束.""" for thread in self.pool: thread.join() def undone_tasks(self): """ 注意:非线程安全 """ r = 0 for thread in self.pool: if not thread.idle: r += 1 if r: r += self.tasks.qsize() #未完成的任务数等于(非空闲的线程数+任务队列中的任务数) return r
def main(): print "[Facebook Album Downloader v1]" start = timeit.default_timer() # hide images prefs = {"profile.managed_default_content_settings.images": 2} extensions = webdriver.ChromeOptions() extensions.add_experimental_option("prefs", prefs) browser = webdriver.Chrome(executable_path="chromedriver", chrome_options=extensions) findAlbum(browser) createAlbumPath() queue = Queue() for x in range(max_workers): worker = DownloadWorker(queue) worker.daemon = True worker.start() print "[Getting Image Links]" linkImages = getImageLinks(browser) print "[Found: " + str(len(linkImages)) + "]" for fullRes in linkImages: queue.put(fullRes) print "[Downloading...]" queue.join() browser.quit() stop = timeit.default_timer() print "[Time taken: %ss]" % str(stop - start) raw_input("Press any key to continue...")
class CommandQueue(object): """Asynchronous command queue that can be used to communicate with blip.pl in the background.""" def __init__(self): self.queue = Queue() self.worker = threading.Thread(target=self.__worker) self.worker.setDaemon(True) def __del__(self): self.Finish() def __worker(self): while True: item = self.queue.get(True) item() self.queue.task_done() def Finish(self): """Finishes all commands in the queue""" self.queue.join() def HasPendingCommands(self): """Returns True if the queue is busy""" return self.queue.qsize() > 0 def Enqueue(self, command): """Enqueues a command in the queue. Command must refer to a function without parameters.""" self.queue.put(command)
def main(): ts = time() start = input("Please input the start value: ") stop = input("Please input the stop value: ") # set the number of threads threads = input("Please input the download numbers every piece: ") k = stop-start # acquire the download links links = [l for l in get_link(start,stop)] # set the download storage directory down_dir = setup_dir() queue = Queue() # judge download numbers if greater than threads or not # if K< = threads ,set the k for threads,else set the threads for the number of thread if k <= threads: for x in range(k-1): print queue.qsize() worker = DownloadWorker(queue) worker.setDaemon(True) worker.start() else: for x in range(threads): worker = DownloadWorker(queue) worker.setDaemon(True) worker.start() # traverse the links and put the link to queue for link in links: queue.put((down_dir,link)) # the new queue joining queue.join() print 'Took {}'.format(time()-ts) print "The finished time:" + ctime()
class ThreadPool(object): """ Pool of threads consuming tasks from a queue """ def __init__(self, num_threads): if num_threads < 1: raise ValueError("ThreadPool must have 1 thread or greenlet at least") elif num_threads == 1: self.__class__ = SingleThreadedPool return self.tasks = Queue(num_threads) for x in range(num_threads): WorkerThread(self.tasks) def spawn_n(self, func, *args, **kargs): """ Add a task to the queue and asign a thread to do the work """ self.tasks.put((func, args, kargs)) def waitall(self): """ Wait for completion of all the tasks in the queue """ self.tasks.join()
class Content: def __init__(self): self.outputs = Queue() d = time.strftime('%y-%m-%d %H-%M',time.localtime()) self.f = open( 'content.txt','a') self.f.write('[') t = Thread(target=self.run) t.setDaemon(True) t.start() def __del__(self): self.outputs.join() self.f.write(']') self.f.close() def write(self,url,dom,targets): result = {} result['url'] = url if dom != None: for t in targets: m = dom.xpath(t['xpath']) if len(m) >= 1: print m[0].text result[t['name']] = m[0].text output = json.dumps(result) + ',\n' self.outputs.put(output) def run(self): while True: output = self.outputs.get() self.f.write(output) self.f.close()
def main(numthreads=10): t1 = time.time() queue = Queue() factory = TokenFactory() config = ConfigParser() config.read('vk_api.conf') url = API.get_url( app_id=config.get('api', 'id'), app_key=config.get('api', 'key'), permissions=PERMISSIONS, redirect_uri=URI, display=DISPLAY, api_version=VERSION) # TODO: check token expiration token_pair = factory.get_token_pair() if not token_pair: token_pair = factory.store_token_pair(url) api = API(token=token_pair[0],user_id=token_pair[1]) audio = api.audio data = audio.get if data: for item in data['response']['items']: queue.put(item) for i in range(numthreads): t = DownloadThread(queue, FILE_DIR) t.start() queue.join() t2 = time.time() print('Time: {0}'.format(t2-t1))
class MaximaTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True): SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate) self.daemon_threads = True # ensure that we can handle more than the usual number of pending requests self.request_queue_size = 30 # initialize the request queue self.ready = False self.request_queue = Queue() def calculate(self, calc): if not self.ready: return None # terminate the request if it hasn't been terminated already # otherwise maxima will wait forever for extra input if not calc.endswith(";"): calc += ";" # form a calcrequest and put it on the queue for processing calc_req = CalculateRequest(calc) self.request_queue.put(calc_req) # wait until its done self.request_queue.join() # use the result that was populated by MaximaHandler when # it pulled our request off the queue and sent it to maxima, # then stored the response in result return calc_req.result
class ThreadPool: """A basic pool of worker threads""" class Worker(Thread): def __init__(self, tasks): Thread.__init__(self) self.tasks = tasks self.daemon = True # threads will be killed on exit self.start() def run(self): while True: # block until a task is ready to be done function, args, kargs = self.tasks.get() try: function(*args, **kargs) except: print(sys.exc_info()[0]) self.tasks.errors = True self.tasks.task_done() def __init__(self, num_threads = multiprocessing.cpu_count() + 1): self.tasks = Queue(num_threads) self.tasks.errors = False # create num_threads Workers, by default the number of CPUs + 1 for _ in range(num_threads): self.Worker(self.tasks) def add_task(self, function, *args, **kargs): self.tasks.put((function, args, kargs)) def wait_completion(self): # wait for the queue to be empty self.tasks.join() return self.tasks.errors
def parallelize(tasks, ntasks): tasks_to_run = Queue() for task in tasks: tasks_to_run.put(tuple(task)) results = Queue() workers = [] for i in range(ntasks): worker = threading.Thread(target = run_task_from_queue, args = (tasks_to_run, results)) worker.daemon = True worker.start() workers.append(worker) ## Wait for all to finish try: tasks_to_run.join() except KeyboardInterrupt: log.error("Caught Ctrl-C, quitting") outputs = [] while not results.empty(): outputs.append(results.get()) print "Finished parallelize() call" ## Kill workers return outputs
class ThreadPool(object): def __init__(self, size=1): self.size = size self.jobs = Queue() self.results = Queue() self.threads = [] def start(self): """start all threads""" for i in range(self.size): self.threads.append(ThreadWorker(self)) for thread in self.threads: thread.start() def append_job(self, job, *args, **kwargs): self.jobs.put((job, args, kwargs)) def join(self): """waiting all jobs done""" self.jobs.join() def stop(self): """kill all threads""" for thread in self.threads: # stop all threads thread.stop() for thread in self.threads: # waiting completing if thread.isAlive(): thread.join() del self.threads[:]
def run_remote_commands(self, ips=None, command=None, ): command = command or self.command ips = ips or self.ips if not ips: self.logger.warning('No IPs provided to run_remote_commands!') return self.results command = command or "" iq = Queue() #if not ips: # raise ValueError('run_remote_commands: IP list was empty:"{0}"'.format(ips)) for ip in ips: ip = str(ip).strip().rstrip() iq.put(ip) tlock = Lock() threadcount = self.args.thread_count if threadcount > iq.qsize(): threadcount = iq.qsize() if not iq: return self.results = {} for i in range(threadcount): t = Thread(target=self.do_ssh, args=(iq, tlock, i, command)) t.daemon = True t.start() self.logger.debug('Threads started now waiting for join') iq.join() self.logger.debug('Done with join') time.sleep(self.maxwait + .1) return self.results
class Crawler(object): ''' A multithread crawler downloads large number of images from adult website and normal website. ''' def __init__(self , url): self.url = url self.numThread = 100 self.queIn = Queue(self.numThread) self.queOut = Queue(self.numThread) self.queOut.put(url) self.seen = [] self.dome = urlparse(url)[1] def walk(self): for i in range(self.numThread): t = PopQueThread(self.queIn , self.queOut , self.seen) t.setDaemon(True) t.start() for i in range(self.numThread): dt = GetPageThread(self.queIn , self.queOut , self.seen , self.dome) dt.setDaemon(True) dt.start() self.queIn.join() self.queOut.join()
class Runner(object): def __init__(self, controller): self.logger = logging.getLogger("Runner") self.logger.debug("initializing") self.controller = controller self.jobmanager = self.controller.getJobManager() self.nodemanager = self.controller.getNodeManager() def runAllJobs(self): self.logger.info("starting workers") joblist = self.jobmanager.getJobList() nodes = self.nodemanager.getNodes() # Create a queue of jobs self.queue = Queue() for job in joblist: self.queue.put(job) # Create workers for each node and start them self.workers = [] for node in nodes.values(): for i in range(node.info.getWorkload()): worker = Worker(i, node, self.queue, self.controller) self.workers.append(worker) worker.start() # Wait for all jobs to be processed self.logger.info("waiting for completion") self.queue.join() self.logger.info("all jobs completed")
class CertDBCertificateReport(reporter.CertificateReport): def __init__(self, cert_db, log_key, checks=all_checks.ALL_CHECKS): self._cert_db = cert_db self.log_key = log_key self._certs_queue = Queue(FLAGS.cert_db_writer_queue_size) self._writer = None super(CertDBCertificateReport, self).__init__(checks=checks) def report(self): super(CertDBCertificateReport, self).report() self._certs_queue.join() logging.info("Finished scheduled writing to CertDB") self._certs_queue.put(None) self.reset() def reset(self): if self._writer: self._writer.join() self._writer = None def _batch_scanned_callback(self, result): if not self._writer: self._writer = threading.Thread(target=_process_certs, args=(self._cert_db, self.log_key, self._certs_queue)) self._writer.start() self._certs_queue.put([(desc, index) for desc, index, _ in result])
def run(self): args = list(islice(self.reqs, self.requests)) if self.shuffle: random.shuffle(args) print "Total requests: %d" % len(args) print "Concurrency : %d" % self.concurrency starttime = time.time() q, p = Queue(), Queue() for _ in xrange(self.concurrency): t = Thread(target=worker, args=(self.host, q, p, self.verbose)) t.daemon = True t.start() for a in args: q.put(a) q.join() outputs = [] for _ in xrange(self.requests): outputs.append(p.get()) elapsed = time.time() - starttime print print "Total requests: %d" % len(args) print "Concurrency : %d" % self.concurrency print "Elapsed time : %.3fs" % elapsed print "Avg time p/req: %.3fs" % (elapsed/len(args)) print "Received (per status code or error):" for c, n in Counter(outputs).items(): print " %s: %d" % (c, n)
class ThreadPool(object): """Pool of threads consuming tasks from a queue""" def __init__(self, workers): self.tasks = Queue() self.workers = [Worker(self.tasks) for x in xrange(workers)] self.state = ThreadPoolState.IDLE def apply_async(self, func, args, **kargs): """Add a task to the queue""" if self.state != ThreadPoolState.IDLE: raise ThreadPoolError('ThreadPool cant accept any more tasks') self.tasks.put((func, args, kargs)) def close(self): self.state = ThreadPoolState.CLOSED while not self.tasks.empty(): self.tasks.get_nowait() self.tasks.task_done() for worker in self.workers: self.tasks.put((None, (), {})) def join(self): """Wait for completion of all the tasks in the queue""" self.state = ThreadPoolState.WAIT_JOIN self.tasks.join()
def main(repo_list_files): """ main runner for key commit """ worker_q = Queue(maxsize=0) results_q = Queue(maxsize=0) num_threads = 10 stop = 5 for repo_list in repo_list_files: with open(repo_list, "rb") as f: for l, line in enumerate(f.xreadlines()): username = line.split(',')[0].split('/')[0].rstrip() repo = line.split(',')[0].split('/')[1].rstrip() worker_q.put((username, repo, line)) print l, '\t', username, '\t', repo if l > stop: break writer_thread = Thread(target=writer, args=(results_q, )) writer_thread.setDaemon(True) writer_thread.start() for i in range(num_threads): t = Thread(target=worker, args=(results_q, worker_q)) t.setDaemon(True) t.start() worker_q.join() results_q.join()
def get_sshkey_deploy(hosts, user, password, key): config = {} config["username"] = user config["password"] = password config["key"] = key config["authorized_keys"] = "authorized_keys" config["ssh_dir"] = "~/.ssh" config["port"] = 22 config["timeout_seconds"] = 3 config["threads"] = 100 config["append"] = True global results results = [] global queue queue = Queue(maxsize=10 * config["threads"]) deployer_threads = [] for i in range(config["threads"]): deployer_thread = DeployKeyThread(config) deployer_thread.daemon = True deployer_thread.start() deployer_threads.append(deployer_thread) for host in hosts: queue.put(host) queue.join() return results
class WorkerThread(threading.Thread): def __init__(self, *args, **kwargs): threading.Thread.__init__(self, *args, **kwargs) self.input_queue = Queue() def send(self, item): self.input_queue.put(item) def close(self): self.input_queue.put(None) self.input_queue.join() def run(self): while True: item = self.input_queue.get() if item is None: break # Process the item: replace with useful work print item self.input_queue.task_done() # Done. Indicate that sentinel was received and return self.input_queue.task_done() return
class ThreadPool(object): """ 线程池的具体实现 """ def __init__(self, thread_count=10): """ thread_count: 线程池大小(创建的线程数) """ try: self.thread_count = int(thread_count) except: raise SystemExit('thread_count not is a number') self.work_list = [] self.init_threadpool() def init_threadpool(self): """ 初始化线程池,并启动线程 """ self.work_queue = Queue() for _ in range(self.thread_count): t = ThreadWork(self.work_queue) self.work_list.append(t) t.setDaemon(True) t.start() def putjob(self, work_func, func_args=None): """ 把任务放入线程池调动 """ self.work_queue.put((work_func, func_args)) def wait(self): self.work_queue.join()
def threadedExec( servers, maxThreads ) : ''' This implements the email account access part of the program using a threaded queue model maxThreads in an INTEGER that denotes the maximum number of parallel threads that the program is allowed to open. This is a global setting. ''' from Queue import Queue inQueue = Queue() # Initiate and populate input queue with list of tasks (data for each task) for account in servers : inQueue.put( servers[ account ] ) # Store the account dictionary in inQueue. This denotes the data required for a single task: polling a single server outQueue = Queue( maxsize = inQueue.qsize() ) # Prepare output queue for storing results # Create a number of threads to parallelize the tasks. Threads created using the Worker class inherited from the threading.Thread class: workers = [ Worker( pollAccount, inQueue, outQueue ) for ii in range( maxThreads ) ] # maxThreads is a global variable that determines the maximum number of open threads for worker in workers : worker.start() # Begun execution of the thread inQueue.join() # Pause program stepping forward here until all tasks in inQueue are completed while not outQueue.empty() : yield outQueue.get() # this wording makes test3() a generator and it should be used as such. test3 will return Buffer type objects from each server poll one at a time.
def get_channels(channels, **kwargs): """Multi-threaded channel query """ if len(channels) == 0: return [] # set up Queues inqueue = Queue() outqueue = Queue() # open threads for i in range(len(channels)): t = ThreadChannelQuery(inqueue, outqueue, **kwargs) t.setDaemon(True) t.start() # populate input queue for i, c in enumerate(channels): inqueue.put((i, c)) # block inqueue.join() outqueue.join() result = [] for i in range(len(channels)): c = outqueue.get() if isinstance(c, Exception): raise c else: result.append(c) return zip(*sorted(result, key=lambda (idx, chan): idx))[1]
class DownloadThreadPool(object): def __init__(self, size=3): self.queue = Queue() self.workers = [Thread(target=self._do_work) for _ in range(size)] self.initialized = False def init_threads(self): for worker in self.workers: worker.setDaemon(True) worker.start() self.initialized = True def _do_work(self): while True: url, target = self.queue.get() download(url, target, close_target=True, quiet=False) self.queue.task_done() def join(self): self.queue.join() def submit(self, url, target): if not self.initialized: self.init_threads() self.queue.put((url, target))
class ThreadPool(): def __init__(self): self.q = Queue() self.NUM = 3 self.JOBS = 10 def do_somthing_using(self,arguments): print arguments def working(self): while True: arguments = self.q.get() self.do_somthing_using(arguments+1) sleep(1) self.q.task_done() def PoolStart(self): for i in range(self.NUM): t = Thread(target=self.working) t.setDaemon(True) t.start() for i in range(self.JOBS): self.q.put(i) self.q.join()
class _BatchWriter(object): def __init__(self, url, start, auth, size, interval, qsize, content_encoding): self.url = url self.offset = start self._nextid = count(start) self.auth = auth self.size = size self.interval = interval self.content_encoding = content_encoding self.checkpoint = time.time() self.itemsq = Queue(size * 2 if qsize is None else qsize) self.closed = False self.flushme = False def write(self, item): assert not self.closed, 'attempting writes to a closed writer' self.itemsq.put(jsonencode(item)) return self._nextid.next() def flush(self): self.flushme = True self.itemsq.join() self.flushme = False def close(self, block=True): self.closed = True if block: self.itemsq.join() def __str__(self): return self.url
def mysql_main(ipdict,threads): printPink("crack mysql now...") print "[*] start crack mysql %s" % time.ctime() starttime=time.time() global sp sp=Queue() global lock lock = threading.Lock() global result result=[] for i in xrange(threads): t = Thread(target=mysql) t.setDaemon(True) t.start() for ip in ipdict['mysql']: sp.put((str(ip).split(':')[0],int(str(ip).split(':')[1]))) sp.join() print "[*] stop crack mysql %s" % time.ctime() print "[*] crack mysql done,it has Elapsed time:%s " % (time.time()-starttime) return result
def worker(): global queue while True: try: item = queue.get() print 'Proccessing ' + item['filename'] if item['type'] == 'pcap': parse_pcap.parse_pcap(get_data_path()+item['filename']) elif item['type'] == 'log': parse_log.parse_log(get_data_path()+item['filename']) except Exception, e: print e finally: queue.task_done() if __name__ == "__main__": queue = Queue() while True: if queue.empty(): time.sleep( config.check_interval ) check_file_change() #check every 3 sec else: for i in range(config.num_worker_threads): t = threading.Thread(target=worker) t.daemon = True t.start() queue.join() # block until all tasks are done
print('Localhost timeout') return False print '*** Testing SSH Creds ***' if ssh_test() != True: exit() print '*** SSH Creds Success! ***' for thread_num in range(max_threads): worker = Thread(target=ssh_session, args=(thread_num, working_queue)) worker.setDaemon(True) worker.start() with open(hosts_csv) as devices: deviceDict = csv.DictReader(devices) for row in deviceDict: with print_lock: print 'Putting {} in queue'.format(row['hostname']) working_queue.put(row['hostname']) with print_lock: print '*** Waiting for all threads to complete ***' working_queue.join() print '*** Finished SSH Sessions ***' if len(error_list) > 0: print '*** Errors Encountered ***' for i in error_list: print '{}, {}'.format(i[0], i[1])
class Timelapse(object): def __init__( self, settings, octoprint_printer, data_folder, timelapse_folder, on_print_started=None, on_print_start_failed=None, on_snapshot_start=None, on_snapshot_end=None, on_render_start=None, on_render_end=None, on_timelapse_stopping=None, on_timelapse_stopped=None, on_state_changed=None, on_timelapse_start=None, on_timelapse_end=None, on_snapshot_position_error=None, on_position_error=None, on_plugin_message_sent=None): # config variables - These don't change even after a reset self.DataFolder = data_folder self.Settings = settings # type: OctolapseSettings self.OctoprintPrinter = octoprint_printer self.DefaultTimelapseDirectory = timelapse_folder self.OnPrintStartCallback = on_print_started self.OnPrintStartFailedCallback = on_print_start_failed self.OnRenderStartCallback = on_render_start self.OnRenderEndCallback = on_render_end self.OnSnapshotStartCallback = on_snapshot_start self.OnSnapshotCompleteCallback = on_snapshot_end self.TimelapseStoppingCallback = on_timelapse_stopping self.TimelapseStoppedCallback = on_timelapse_stopped self.OnStateChangedCallback = on_state_changed self.OnTimelapseStartCallback = on_timelapse_start self.OnTimelapseEndCallback = on_timelapse_end self.OnSnapshotPositionErrorCallback = on_snapshot_position_error self.OnPositionErrorCallback = on_position_error self.OnPluginMessageSentCallback = on_plugin_message_sent self.Commands = Commands() # used to parse and generate gcode self.Triggers = None self.PrintEndStatus = "Unknown" self.LastStateChangeMessageTime = None self.StateChangeMessageThread = None # Settings that may be different after StartTimelapse is called self.OctoprintPrinterProfile = None self.PrintStartTime = None self.FfMpegPath = None self.Snapshot = None self.Gcode = None self.Printer = None self.CaptureSnapshot = None self.Position = None self.Rendering = None self.State = TimelapseState.Idle self.IsTestMode = False # State Tracking that should only be reset when starting a timelapse self.SnapshotCount = 0 self.HasBeenStopped = False self.TimelapseStopRequested = False self.SavedCommand = None self.SecondsAddedByOctolapse = 0 # State tracking variables self.RequiresLocationDetectionAfterHome = False # fetch position private variables self._position_payload = None self._position_timeout_long = 600.0 self._position_timeout_short = 10.0 self._position_signal = threading.Event() self._position_signal.set() # get snapshot async private variables self._snapshot_success = False # It shouldn't take more than 5 seconds to take a snapshot! self._snapshot_timeout = 5.0 self._snapshot_signal = threading.Event() self._snapshot_signal.set() self._most_recent_snapshot_payload = None self.CurrentProfiles = {} self.CurrentFileLine = 0 # snapshot thread queue self._snapshot_task_queue = Queue(maxsize=1) self._rendering_task_queue = Queue(maxsize=1) self._reset() def start_timelapse( self, settings, octoprint_printer_profile, ffmpeg_path, g90_influences_extruder): # we must supply the settings first! Else reset won't work properly. self._reset() # in case the settings have been destroyed and recreated self.Settings = settings # time tracking - how much time did we add to the print? self.SnapshotCount = 0 self.SecondsAddedByOctolapse = 0 self.RequiresLocationDetectionAfterHome = False self.OctoprintPrinterProfile = octoprint_printer_profile self.FfMpegPath = ffmpeg_path self.PrintStartTime = time.time() self.Snapshot = Snapshot(self.Settings.current_snapshot()) self.Gcode = SnapshotGcodeGenerator( self.Settings, octoprint_printer_profile) self.Printer = Printer(self.Settings.current_printer()) self.Rendering = Rendering(self.Settings.current_rendering()) self.CaptureSnapshot = CaptureSnapshot( self.Settings, self.DataFolder, print_start_time=self.PrintStartTime) self.Position = Position( self.Settings, octoprint_printer_profile, g90_influences_extruder) self.State = TimelapseState.WaitingForTrigger self.IsTestMode = self.Settings.current_debug_profile().is_test_mode self.Triggers = Triggers(self.Settings) self.Triggers.create() # take a snapshot of the current settings for use in the Octolapse Tab self.CurrentProfiles = self.Settings.get_profiles_dict() # send an initial state message self._on_timelapse_start() def on_position_received(self, payload): if self.State != TimelapseState.Idle: self._position_payload = payload self._position_signal.set() def send_snapshot_gcode_array(self, gcode_array): self.OctoprintPrinter.commands(gcode_array, tags={"snapshot_gcode"}) def get_position_async(self, start_gcode=None, timeout=None): if timeout is None: timeout = self._position_timeout_long self.Settings.current_debug_profile().log_print_state_change("Octolapse is requesting a position.") # Warning, we can only request one position at a time! if self._position_signal.is_set(): self._position_signal.clear() # build the staret commands commands_to_send = ["M400", "M114"] # send any code that is to be run before the position request if start_gcode is not None and len(start_gcode) > 0: commands_to_send = start_gcode + commands_to_send self.send_snapshot_gcode_array(commands_to_send) event_is_set = self._position_signal.wait(timeout) if not event_is_set: # we ran into a timeout while waiting for a fresh position # set the position signal self._snapshot_signal.set() self.Settings.current_debug_profile().log_warning( "Warning: A timeout occurred while requesting the current position!." ) return None return self._position_payload def _on_snapshot_success(self, *args, **kwargs): # Increment the number of snapshots received self.SnapshotCount += 1 self._snapshot_success = True self._snapshot_signal.set() def _on_snapshot_fail(self, *args, **kwargs): reason = args[0] message = "Failed to download the snapshot. Reason: {0}".format( reason) self.Settings.current_debug_profile().log_snapshot_download(message) self._snapshot_success = False self.SnapshotError = message self._snapshot_signal.set() def _on_snapshot_complete(self, *args, **kwargs): self.Settings.current_debug_profile().log_snapshot_download("Snapshot download complete.") def _take_snapshot_async(self): snapshot_async_payload = { "success": False, "error": "Waiting on thread to signal, aborting" } if self._snapshot_signal.is_set(): # only clear signal and send a new M114 if we haven't already done that from another thread self._snapshot_signal.clear() # start the snapshot self.Settings.current_debug_profile().log_snapshot_download("Taking a snapshot.") snapshot_guid = str(uuid.uuid4()) snapshot_job = self.CaptureSnapshot.create_snapshot_job( utility.get_currently_printing_filename(self.OctoprintPrinter), self.SnapshotCount, snapshot_guid, self._snapshot_task_queue, on_success=self._on_snapshot_success, on_fail=self._on_snapshot_fail, on_complete=self._on_snapshot_complete ) self._snapshot_task_queue.put(snapshot_guid) snapshot_thread = threading.Thread(target=snapshot_job) snapshot_thread.daemon = True snapshot_thread.start() event_is_set = self._snapshot_signal.wait(self._snapshot_timeout) if not event_is_set: # we ran into a timeout while waiting for a fresh position snapshot_async_payload["success"] = False snapshot_async_payload["error"] = \ "Snapshot timed out in {0} seconds.".format(self._snapshot_timeout) self._snapshot_signal.set() else: snapshot_async_payload["success"] = True return snapshot_async_payload def _take_timelapse_snapshot( self, trigger, command_string, cmd, parameters, triggering_command_position, triggering_extruder_position ): timelapse_snapshot_payload = { "snapshot_position": None, "return_position": None, "snapshot_gcode": None, "snapshot_payload": None, "current_snapshot_time": 0, "total_snapshot_time": 0, "success": False, "error": "" } try: show_real_snapshot_time = self.Settings.show_real_snapshot_time # create the GCode for the timelapse and store it snapshot_gcode = self.Gcode.create_snapshot_gcode( self.Position, trigger, command_string, cmd, parameters, triggering_command_position, triggering_extruder_position ) # save the gcode fo the payload timelapse_snapshot_payload["snapshot_gcode"] = snapshot_gcode if snapshot_gcode is None: self.Settings.current_debug_profile().log_warning( "No snapshot gcode was generated." ) return timelapse_snapshot_payload assert (isinstance(snapshot_gcode, SnapshotGcode)) if not show_real_snapshot_time: gcodes_to_send = snapshot_gcode.StartGcode + snapshot_gcode.SnapshotCommands if len(gcodes_to_send) > 0: self.Settings.current_debug_profile().log_snapshot_gcode( "Sending snapshot start gcode and snapshot commands.") snapshot_position = self.get_position_async( start_gcode=snapshot_gcode.StartGcode + snapshot_gcode.SnapshotCommands ) else: self.Settings.current_debug_profile().log_snapshot_gcode( "Sending snapshot start gcode.") # send start commands and zhop/retract if they exist if len(snapshot_gcode.StartGcode) > 0: start_position = self.get_position_async(start_gcode=snapshot_gcode.StartGcode) # Todo: Handle start_position = None # park the printhead in the snapshot position and wait for the movement to complete snapshot_start_time = time.time() if len(snapshot_gcode.SnapshotCommands) > 0: self.Settings.current_debug_profile().log_snapshot_gcode("Sending snapshot commands.") snapshot_position = self.get_position_async( start_gcode=snapshot_gcode.SnapshotCommands, timeout=self._position_timeout_short ) # record the snapshot position timelapse_snapshot_payload["snapshot_position"] = snapshot_position # by now we should be ready to take a snapshot snapshot_async_payload = self._take_snapshot_async() timelapse_snapshot_payload["snapshot_payload"] = snapshot_async_payload if not show_real_snapshot_time: # return the printhead to the start position gcode_to_send = snapshot_gcode.ReturnCommands + snapshot_gcode.EndGcode if len (gcode_to_send) > 0: self.Settings.current_debug_profile().log_snapshot_gcode("Sending snapshot return and end gcode.") self.send_snapshot_gcode_array(gcode_to_send) else: if len(snapshot_gcode.ReturnCommands) > 0: self.Settings.current_debug_profile().log_snapshot_gcode("Sending return gcode.") return_position = self.get_position_async( start_gcode=snapshot_gcode.ReturnCommands, timeout=self._position_timeout_short ) timelapse_snapshot_payload["return_position"] = return_position # calculate the total snapshot time snapshot_end_time = time.time() snapshot_time = snapshot_end_time - snapshot_start_time self.SecondsAddedByOctolapse += snapshot_time timelapse_snapshot_payload["current_snapshot_time"] = snapshot_time timelapse_snapshot_payload["total_snapshot_time"] = self.SecondsAddedByOctolapse if len(snapshot_gcode.EndGcode) > 0: self.Settings.current_debug_profile().log_snapshot_gcode("Sending end gcode.") self.send_snapshot_gcode_array(snapshot_gcode.EndGcode) # we've completed the procedure, set success timelapse_snapshot_payload["success"] = True except Exception as e: self.Settings.current_debug_profile().log_exception(e) timelapse_snapshot_payload["error"] = "An unexpected error was encountered while running the timelapse " \ "snapshot procedure. " return timelapse_snapshot_payload # public functions def to_state_dict(self): try: position_dict = None position_state_dict = None extruder_dict = None trigger_state = None if self.Settings is not None: if self.Settings.show_position_changes and self.Position is not None: position_dict = self.Position.to_position_dict() if self.Settings.show_position_state_changes and self.Position is not None: position_state_dict = self.Position.to_state_dict() if self.Settings.show_extruder_state_changes and self.Position is not None: extruder_dict = self.Position.Extruder.to_dict() if self.Settings.show_trigger_state_changes and self.Triggers is not None: trigger_state = { "Name": self.Triggers.Name, "Triggers": self.Triggers.state_to_list() } state_dict = { "Extruder": extruder_dict, "Position": position_dict, "PositionState": position_state_dict, "TriggerState": trigger_state } return state_dict except Exception as e: self.Settings.CurrentDebugProfile().log_exception(e) # if we're here, we've reached and logged an error. return { "Extruder": None, "Position": None, "PositionState": None, "TriggerState": None } def stop_snapshots(self, message=None, error=False): self.State = TimelapseState.WaitingToRender if self.TimelapseStoppedCallback is not None: timelapse_stopped_callback_thread = threading.Thread( target=self.TimelapseStoppedCallback, args=[message, error] ) timelapse_stopped_callback_thread.daemon = True timelapse_stopped_callback_thread.start() return True def on_print_failed(self): if self.State != TimelapseState.Idle: self.end_timelapse("FAILED") def on_print_disconnecting(self): if self.State != TimelapseState.Idle: self.end_timelapse("DISCONNECTING") def on_print_disconnected(self): if self.State != TimelapseState.Idle: self.end_timelapse("DISCONNECTED") def on_print_canceled(self): if self.State != TimelapseState.Idle: self.end_timelapse("CANCELED") def on_print_completed(self): if self.State != TimelapseState.Idle: self.end_timelapse("COMPLETED") def end_timelapse(self, print_status): self.PrintEndStatus = print_status try: if self.PrintStartTime is None: self._reset() elif self.PrintStartTime is not None and self.State in [ TimelapseState.WaitingForTrigger, TimelapseState.WaitingToRender, TimelapseState.WaitingToEndTimelapse ]: if not self._render_timelapse(self.PrintEndStatus): if self.OnRenderEndCallback is not None: payload = RenderingCallbackArgs( "Could not start timelapse job.", -1, "unknown", "unknown", "unknown", "unknown", "unknown", "unknown", False, 0, 0, True, "timelapse_start", "The render_start function returned false" ) render_end_callback_thread = threading.Thread( target=self.OnRenderEndCallback, args=[payload] ) render_end_callback_thread.daemon = True render_end_callback_thread.start() self._reset() if self.State != TimelapseState.Idle: self.State = TimelapseState.WaitingToEndTimelapse except Exception as e: self.Settings.current_debug_profile().log_exception(e) if self.OnTimelapseEndCallback is not None: self.OnTimelapseEndCallback() def on_print_paused(self): try: if self.State == TimelapseState.Idle: return elif self.State < TimelapseState.WaitingToRender: self.Settings.current_debug_profile().log_print_state_change("Print Paused.") self.Triggers.pause() except Exception as e: self.Settings.current_debug_profile().log_exception(e) def on_print_resumed(self): try: if self.State == TimelapseState.Idle: return elif self.State < TimelapseState.WaitingToRender: self.Triggers.resume() except Exception as e: self.Settings.current_debug_profile().log_exception(e) def is_timelapse_active(self): if ( self.Settings is None or self.State in [TimelapseState.Idle, TimelapseState.Initializing, TimelapseState.WaitingToRender] or self.OctoprintPrinter.get_state_id() == "CANCELLING" or self.Triggers is None or self.Triggers.count() < 1 ): return False return True def get_is_rendering(self): return self._rendering_task_queue.qsize() > 0 def on_print_start(self, tags): self.OnPrintStartCallback(tags) def on_print_start_failed(self, message): self.OnPrintStartFailedCallback(message) def on_gcode_queuing(self, command_string, cmd_type, gcode, tags): self.detect_timelapse_start(command_string, tags) # if the timelapse is not active, exit without changing any gcode if not self.is_timelapse_active(): return self.check_current_line_number(tags) # update the position tracker so that we know where all of the axis are. # We will need this later when generating snapshot gcode so that we can return to the previous # position is_snapshot_gcode_command = self._is_snapshot_command(command_string) try: self.Settings.current_debug_profile().log_gcode_queuing( "Queuing Command: Command Type:{0}, gcode:{1}, cmd: {2}, tags: {3}".format( cmd_type, gcode, command_string, tags ) ) try: cmd, parameters = Commands.parse(command_string) except ValueError as e: message = "An error was thrown by the gcode parser, stopping timelapse. Details: {0}".format(str(e)) self.Settings.current_debug_profile().log_warning( message ) self.stop_snapshots(message, True) return None # get the position state in case it has changed # if there has been a position or extruder state change, inform any listener if cmd is not None and not is_snapshot_gcode_command: # create our state change dictionaries self.Position.update(command_string, cmd, parameters) # if this code is snapshot gcode, simply return it to the printer. if {'plugin:octolapse', 'snapshot_gcode'}.issubset(tags): return None if not self.check_for_non_metric_errors(): if self.Position.has_position_error(0): # There are position errors, report them! self._on_position_error() elif (self.State == TimelapseState.WaitingForTrigger and (self.Position.requires_location_detection(1)) and self.OctoprintPrinter.is_printing()): if 'source:script' in tags: # warn user self._send_plugin_message_async( "warning", "Octolapse could not acquire a position while sending" " OctoPrint scripts (settings=>GCODE Scripts)." " A fix is in the works, but a modification to OctoPrint itself" " is required. For now please move your start gcode into the" " actual gcode file.") else: self.State = TimelapseState.AcquiringLocation if self.OctoprintPrinter.set_job_on_hold(True): thread = threading.Thread(target=self.acquire_position, args=[command_string, cmd, parameters]) thread.daemon = True thread.start() return None, elif (self.State == TimelapseState.WaitingForTrigger and self.OctoprintPrinter.is_printing() and not self.Position.has_position_error(0)): # update the triggers with the current position self.Triggers.update(self.Position, command_string) # see if at least one trigger is triggering _first_triggering = self.get_first_triggering() if _first_triggering: if 'source:script' in tags: # warn user self._send_plugin_message_async( "warning", "Octolapse could not take a snapshot while sending" " OctoLapse scripts (settings=>GCODE Scripts)." " A fix is in the works, but a modification to OctoPrint itself" " is required. For now please move your start gcode into the" " actual gcode file." ) else: # We are triggering, take a snapshot self.State = TimelapseState.TakingSnapshot # pause any timer triggers that are enabled self.Triggers.pause() # get the job lock if self.OctoprintPrinter.set_job_on_hold(True): # take the snapshot on a new thread thread = threading.Thread( target=self.acquire_snapshot, args=[command_string, cmd, parameters, _first_triggering] ) thread.daemon = True thread.start() # suppress the current command, we'll send it later return None, elif self.State == TimelapseState.TakingSnapshot: # Don't do anything further to any commands unless we are # taking a timelapse , or if octolapse paused the print. # suppress any commands we don't, under any cirumstances, # to execute while we're taking a snapshot if cmd in self.Commands.SuppressedSnapshotGcodeCommands: command_string = None, # suppress the command if is_snapshot_gcode_command: # in all cases do not return the snapshot command to the printer. # It is NOT a real gcode and could cause errors. command_string = None, except Exception as e: self.Settings.current_debug_profile().log_exception(e) raise # notify any callbacks self._send_state_changed_message() # do any post processing for test mode if command_string != (None,): command_string = self._get_command_for_octoprint(command_string, cmd,parameters) return command_string def detect_timelapse_start(self, cmd, tags): # detect print start if ( self.Settings.is_octolapse_enabled and self.State == TimelapseState.Idle and {'trigger:comm.start_print', 'trigger:comm.reset_line_numbers'} <= tags and # ({'trigger:comm.start_print', 'fileline:1'} <= tags or {'script:beforePrintStarted', 'trigger:comm.send_gcode_script'} <= tags) and self.OctoprintPrinter.is_printing() ): if self.OctoprintPrinter.set_job_on_hold(True): try: self.State = TimelapseState.Initializing self.Settings.current_debug_profile().log_print_state_change( "Print Start Detected. Command: {0}, Tags:{1}".format(cmd, tags) ) # call the synchronous callback on_print_start self.on_print_start(tags) if self.State == TimelapseState.WaitingForTrigger: # set the current line to 0 so that the plugin checks for line 1 below after startup. self.CurrentFileLine = 0 finally: self.OctoprintPrinter.set_job_on_hold(False) else: self.on_print_start_failed( "Unable to start timelapse, failed to acquire a job lock. Print start failed." ) def check_current_line_number(self, tags): # check the current line number if {'source:file'} in tags: # this line is from the file, advance! self.CurrentFileLine += 1 if "fileline:{0}".format(self.CurrentFileLine) not in tags: actual_file_line = "unknown" for tag in tags: if len(tag) > 9 and tag.startswith("fileline:"): actual_file_line = tag[9:] message = "File line number {0} was expected, but {1} was received!".format( self.CurrentFileLine + 1, actual_file_line ) self.Settings.current_debug_profile().log_error(message) self.stop_snapshots(message, True) def check_for_non_metric_errors(self): # make sure we're not using inches is_metric = self.Position.is_metric() has_error = False error_message = "" if is_metric is None and self.Position.has_position_error(): has_error = True error_message = "The printer profile requires an explicit G21 command before any position " \ "altering/setting commands, including any home commands. Stopping timelapse, " \ "but continuing the print. " elif not is_metric and self.Position.has_position_error(): has_error = True if self.Printer.units_default == "inches": error_message = "The printer profile uses 'inches' as the default unit of measurement. In order to" \ " use Octolapse, a G21 command must come before any position altering/setting commands, including" \ " any home commands. Stopping timelapse, but continuing the print. " else: error_message = "The gcode file contains a G20 command (set units to inches), which Octolapse " \ "does not support. Stopping timelapse, but continuing the print." if has_error: self.stop_snapshots(error_message,has_error) return has_error def get_first_triggering(self): try: # make sure we're in a state that could want to check for triggers if not self.State == TimelapseState.WaitingForTrigger: return False # see if the PREVIOUS command triggered (that means current gcode gets sent if the trigger[0] # is triggering first_trigger = self.Triggers.get_first_triggering(0, Triggers.TRIGGER_TYPE_IN_PATH) if first_trigger: self.Settings.current_debug_profile().log_triggering("An in-path snapshot is triggering") return first_trigger first_trigger = self.Triggers.get_first_triggering(1, Triggers.TRIGGER_TYPE_DEFAULT) if first_trigger: # We're triggering self.Settings.current_debug_profile().log_triggering("A snapshot is triggering") return first_trigger except Exception as e: self.Settings.current_debug_profile().log_exception(e) # no need to re-raise here, the trigger just won't happen return False def acquire_position(self, command_string, cmd, parameters): try: self.Settings.current_debug_profile().log_print_state_change( "A position altering command has been detected. Fetching and updating position. " "Position Command: {0}".format(cmd)) # Undo the last position update, we will be resending the command self.Position.undo_update() current_position = self.get_position_async() if current_position is None: self.PrintEndStatus = "POSITION_TIMEOUT" self.State = TimelapseState.WaitingToEndTimelapse self.Settings.current_debug_profile().log_print_state_change( "Unable to acquire a position.") else: # update position self.Position.update_position( x=current_position["x"], y=current_position["y"], z=current_position["z"], e=current_position["e"], force=True, calculate_changes=True) # adjust the triggering command if self.IsTestMode: gcode = self.Commands.alter_for_test_mode(command_string, cmd, parameters, return_string=True) else: gcode = command_string if gcode != "": self.Settings.current_debug_profile().log_print_state_change( "Sending triggering command for position acquisition - {0}.".format(gcode)) # send the triggering command self.send_snapshot_gcode_array([gcode]) # set the state if self.State == TimelapseState.AcquiringLocation: self.State = TimelapseState.WaitingForTrigger self.Settings.current_debug_profile().log_print_state_change("Position Acquired") finally: self.OctoprintPrinter.set_job_on_hold(False) def acquire_snapshot(self, command_string, cmd, parameters, trigger): try: self.Settings.current_debug_profile().log_snapshot_download( "About to take a snapshot. Triggering Command: {0}".format(cmd)) if self.OnSnapshotStartCallback is not None: snapshot_callback_thread = threading.Thread(target=self.OnSnapshotStartCallback) snapshot_callback_thread.daemon = True snapshot_callback_thread.start() # Capture and undo the last position update, we're not going to be using it! triggering_command_position, triggering_extruder_position = self.Position.undo_update() # take the snapshot # Todo: We probably don't need the payload here. self._most_recent_snapshot_payload = self._take_timelapse_snapshot( trigger, command_string, cmd, parameters, triggering_command_position, triggering_extruder_position ) self.Settings.current_debug_profile().log_snapshot_download("The snapshot has completed") finally: # set the state if self.State == TimelapseState.TakingSnapshot: self.State = TimelapseState.WaitingForTrigger self.Triggers.resume() self.OctoprintPrinter.set_job_on_hold(False) # notify that we're finished, but only if we haven't just stopped the timelapse. if self._most_recent_snapshot_payload is not None: # send a copy of the dict in case it gets changed by threads. self._on_trigger_snapshot_complete(self._most_recent_snapshot_payload.copy()) def on_gcode_sent(self, cmd, cmd_type, gcode, tags): self.Settings.current_debug_profile().log_gcode_sent( "Sent to printer: Command Type:{0}, gcode:{1}, cmd: {2}, tags: {3}".format(cmd_type, gcode, cmd, tags)) def on_gcode_received(self, comm, line, *args, **kwargs): self.Settings.current_debug_profile().log_gcode_received( "Received from printer: line:{0}".format(line) ) return line # internal functions #################### def _get_command_for_octoprint(self, command_string, cmd, parameters): if command_string is None or command_string == (None,): return command_string if self.IsTestMode and self.State >= TimelapseState.WaitingForTrigger: return self.Commands.alter_for_test_mode(command_string, cmd, parameters) # if we were given a list, return it. if isinstance(command_string, list): return command_string # if we were given a command return None (don't change the command at all) return None def _send_state_changed_message(self): """Notifies any callbacks about any changes contained in the dictionaries. If you send a dict here the client will get a message, so check the settings to see if they are subscribed to notifications before populating the dictinaries!""" delay_seconds = 0 # if another thread is trying to send the message, stop it if self.StateChangeMessageThread is not None and self.StateChangeMessageThread.isAlive(): self.StateChangeMessageThread.cancel() if self.LastStateChangeMessageTime is not None: # do not send more than 1 per second time_since_last_update = time.time() - self.LastStateChangeMessageTime if time_since_last_update < 1: delay_seconds = 1-time_since_last_update if delay_seconds < 0: delay_seconds = 0 try: # Notify any callbacks if self.OnStateChangedCallback is not None: def send_change_message(): trigger_change_list = None position_change_dict = None position_state_change_dict = None extruder_change_dict = None trigger_changes_dict = None # Get the changes if self.Settings.show_trigger_state_changes: trigger_change_list = self.Triggers.state_to_list() if self.Settings.show_position_changes: position_change_dict = self.Position.to_position_dict() if self.Settings.show_position_state_changes: position_state_change_dict = self.Position.to_state_dict() if self.Settings.show_extruder_state_changes: extruder_change_dict = self.Position.Extruder.to_dict() # if there are any state changes, send them if ( position_change_dict is not None or position_state_change_dict is not None or extruder_change_dict is not None or trigger_change_list is not None ): if trigger_change_list is not None and len(trigger_change_list) > 0: trigger_changes_dict = { "Name": self.Triggers.Name, "Triggers": trigger_change_list } change_dict = { "Extruder": extruder_change_dict, "Position": position_change_dict, "PositionState": position_state_change_dict, "TriggerState": trigger_changes_dict } if ( change_dict["Extruder"] is not None or change_dict["Position"] is not None or change_dict["PositionState"] is not None or change_dict["TriggerState"] is not None ): self.OnStateChangedCallback(change_dict) self.LastStateChangeMessageTime = time.time() # Send a delayed message self.StateChangeMessageThread = threading.Timer( delay_seconds, send_change_message ) self.StateChangeMessageThread.daemon = True self.StateChangeMessageThread.start() except Exception as e: # no need to re-raise, callbacks won't be notified, however. self.Settings.current_debug_profile().log_exception(e) def _send_plugin_message(self, message_type, message): self.OnPluginMessageSentCallback(message_type, message) def _send_plugin_message_async(self, message_type, message): warning_thread = threading.Thread(target=self._send_plugin_message, args=[message_type, message]) warning_thread.daemon = True warning_thread.start() def _is_snapshot_command(self, command_string): return command_string == self.Printer.snapshot_command def _is_trigger_waiting(self): # make sure we're in a state that could want to check for triggers if not self.State == TimelapseState.WaitingForTrigger: return None # Loop through all of the active currentTriggers waiting_trigger = self.Triggers.get_first_waiting() if waiting_trigger is not None: return True return False def _on_position_error(self): message = self.Position.position_error(0) self.Settings.current_debug_profile().log_error(message) if self.OnPositionErrorCallback is not None: position_error_callback_thread = threading.Thread( target=self.OnPositionErrorCallback, args=[message] ) position_error_callback_thread.daemon = True position_error_callback_thread.start() def _on_trigger_snapshot_complete(self, snapshot_payload): if self.OnSnapshotCompleteCallback is not None: payload = { "success": snapshot_payload["success"], "error": snapshot_payload["error"], "snapshot_count": self.SnapshotCount, "total_snapshot_time": snapshot_payload["total_snapshot_time"], "current_snapshot_time": snapshot_payload["total_snapshot_time"] } if self.OnSnapshotCompleteCallback is not None: snapshot_complete_callback_thread = threading.Thread( target=self.OnSnapshotCompleteCallback, args=[payload] ) snapshot_complete_callback_thread.daemon = True snapshot_complete_callback_thread.start() def _render_timelapse(self, print_end_state): def _render_timelapse_async(render_job_id, timelapse_render_job): try: snapshot_thread = threading.Thread(target=timelapse_render_job, args=[]) snapshot_thread.daemon = True num_snapshot_tasks = self._snapshot_task_queue.qsize() if num_snapshot_tasks > 0: self.Settings.current_debug_profile().log_render_start("Started Rendering Timelapse.") else: self.Settings.current_debug_profile().log_render_start( "Waiting for {0} snapshot threads to complete".format( self._snapshot_task_queue.qsize())) self._snapshot_task_queue.join() self.Settings.current_debug_profile().log_render_start( "All snapshot tasks have completed, rendering timelapse" ) # we are rendering, set the state before starting the rendering job. self._rendering_task_queue.put(render_job_id) snapshot_thread.start() except Exception as e: self.Settings.current_debug_profile().log_exception(e) self._rendering_task_queue.get() self._rendering_task_queue.task_done() # make sure we have a non null TimelapseSettings object. We may have terminated the timelapse for some reason if self.Rendering is not None and self.Rendering.enabled: job_id = "TimelapseRenderJob_{0}".format(str(uuid.uuid4())) job = Render.create_render_job( self.Settings, self.Snapshot, self.Rendering, self.DataFolder, self.DefaultTimelapseDirectory, self.FfMpegPath, 1, self._rendering_task_queue, job_id, utility.get_currently_printing_filename(self.OctoprintPrinter), self.PrintStartTime, time.time(), print_end_state, self.SecondsAddedByOctolapse, self._on_render_start, self._on_render_end ) rendering_thread = threading.Thread(target=_render_timelapse_async, args=[job_id, job]) rendering_thread.daemon = True rendering_thread.start() return True return False def _on_render_start(self, *args, **kwargs): job_id = args[0] self.Settings.current_debug_profile().log_render_start( "Started rendering/synchronizing the timelapse. JobId: {0}".format(job_id)) payload = args[1] # notify the caller if self.OnRenderStartCallback is not None: render_start_complete_callback_thread = threading.Thread( target=self.OnRenderStartCallback, args=[payload] ) render_start_complete_callback_thread.daemon = True render_start_complete_callback_thread.start() def _on_render_end(self, *args, **kwargs): job_id = args[0] payload = args[1] self.Settings.current_debug_profile().log_render_complete("Completed rendering. JobId: {0}".format(job_id)) assert (isinstance(payload, RenderingCallbackArgs)) if not payload.HasError and self.Snapshot.cleanup_after_render_fail: self.CaptureSnapshot.clean_snapshots(utility.get_snapshot_temp_directory(self.DataFolder)) elif self.Snapshot.cleanup_after_render_complete: self.CaptureSnapshot.clean_snapshots(utility.get_snapshot_temp_directory(self.DataFolder)) if self.OnRenderEndCallback is not None: render_end_complete_callback_thread = threading.Thread( target=self.OnRenderEndCallback, args=[payload] ) render_end_complete_callback_thread.daemon = True render_end_complete_callback_thread.start() def _on_timelapse_start(self): if self.OnTimelapseStartCallback is None: return self.OnTimelapseStartCallback() def _reset(self): self.State = TimelapseState.Idle self.CurrentFileLine = 0 if self.Triggers is not None: self.Triggers.reset() self.CommandIndex = -1 self.LastStateChangeMessageTime = None self.PrintStartTime = None self.SnapshotGcodes = None self.SavedCommand = None self.PositionRequestAttempts = 0 self.IsTestMode = False self.ReturnPositionReceivedTime = None # A list of callbacks who want to be informed when a timelapse ends self.TimelapseStopRequested = False self._snapshot_success = False self.SnapshotError = "" self.HasBeenStopped = False self.CurrentProfiles = { "printer": "", "stabilization": "", "snapshot": "", "rendering": "", "camera": "", "debug_profile": "" } # fetch position private variables self._position_payload = None self._position_signal.set() # get snapshot async private variables self._snapshot_signal.set() def _reset_snapshot(self): self.State = TimelapseState.WaitingForTrigger self.CommandIndex = -1 self.SnapshotGcodes = None self.SavedCommand = None self.PositionRequestAttempts = 0 self._snapshot_success = False self.SnapshotError = ""
threads = [NNThread(train, test, lock) for _ in range(THREAD_NUM)] for t in threads: t.setDaemon(True) t.start() print 'Q11' eta = 0.1 r = 0.1 M = [1, 6, 11, 16, 21] e_out = {1: [], 6: [], 11: [], 16: [], 21: []} tStart = time.time() for m in M: for i in range(experiment): threadPool.put([eta, r, [2, m, 1], e_out[m]]) threadPool.join() tEnd = time.time() print 'Exec time: {}'.format(tEnd - tStart) for m in M: print 'm:{}, eout-avg: {}'.format(m, avg(e_out[m])) print 'Q12' eta = 0.1 R = [0, 0.001, 0.1, 10.0, 1000.0] M = [2, 3, 1] e_out = {0: [], 0.001: [], 0.1: [], 10.0: [], 1000.0: []} tStart = time.time() for r in R: for i in range(experiment): threadPool.put([eta, r, M, e_out[r]]) threadPool.join()
class GraphiteReporter(threading.Thread): """A graphite reporter thread.""" def __init__(self, host, port, maxQueueSize=10000): """Connect to a Graphite server on host:port.""" threading.Thread.__init__(self) self.host, self.port = host, port self.sock = None self.queue = Queue() self.maxQueueSize = maxQueueSize self.daemon = True def run(self): """Run the thread.""" while True: try: try: name, value, valueType, stamp = self.queue.get() except TypeError: break self.log(name, value, valueType, stamp) finally: self.queue.task_done() def connect(self): """Connects to the Graphite server if not already connected.""" if self.sock is not None: return backoff = 0.01 while True: try: sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.settimeout(5) sock.connect((self.host, self.port)) self.sock = sock return except socket.error: time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) def disconnect(self): """Disconnect from the Graphite server if connected.""" if self.sock is not None: try: self.sock.close() except socket.error: pass finally: self.sock = None def _sendMsg(self, msg): """Send a line to graphite. Retry with exponential backoff.""" if not self.sock: self.connect() backoff = 0.001 while True: try: self.sock.sendall(msg) break except socket.error: self.disconnect() time.sleep(random.uniform(0, 2.0*backoff)) backoff = min(backoff*2.0, 5.0) self.connect() def _sanitizeName(self, name): """Sanitize a metric name.""" return name.replace(' ', '-') def log(self, name, value, valueType=None, stamp=None): """Log a named numeric value. The value type may be 'value', 'count', or None.""" if type(value) == float: form = "%s%s %2.2f %d\n" else: form = "%s%s %s %d\n" if valueType is not None and len(valueType) > 0 and valueType[0] != '.': valueType = '.' + valueType if not stamp: stamp = time.time() self._sendMsg(form % (self._sanitizeName(name), valueType or '', value, stamp)) def enqueue(self, name, value, valueType=None, stamp=None): """Enqueue a call to log.""" # If queue is too large, refuse to log. if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize: return # Stick arguments into the queue self.queue.put((name, value, valueType, stamp)) def flush(self): """Block until all stats have been sent to Graphite.""" self.queue.join() def shutdown(self): """Shut down the background thread.""" self.queue.put(None) self.flush()
while True: # get the song from queue song = self.queue.get() download_song(song) # mark the song as downloaded self.queue.task_done() if __name__ == '__main__': parser = argparse.ArgumentParser(description='fetch music from accuradio.com') parser.add_argument('genre', help='something like jazz or adultalternative') parser.add_argument('channel', help='Groove Jazz or Latin Jazz', nargs='?', default=None) args = parser.parse_args() channels = fetch_channels(args.genre) if args.channel and args.channel in channels: # create THREAD_AMOUNT of threads to download songs for i in range(THREAD_AMOUNT): thread = DownloadThread(queue) thread.setDaemon(True) thread.start() fetch(args.channel, channels[args.channel]) else: print '\n'.join(sorted(channels)) # wait for all songs to download queue.join()
class UnknownRSThreadedCPUKernel(UnknownRSKernel): def __init__(self): UnknownRSKernel.__init__(self) self.numthreads = None self.threads = None self.q = Queue() self.G_lock = Lock() self.sigma_lock = Lock() def setup(self,params,diagout,statout,ostream): UnknownRSKernel.setup(self, params, diagout, statout, ostream) numthreads = params.get('num_threads','auto') det_numthreads = multiprocessing.cpu_count() if numthreads == 'auto': numthreads = np.inf use_numthreads = min(det_numthreads,numthreads) print("Detected {0} cores, using {1} threads".format(det_numthreads, use_numthreads)) if self.threads == None: self.numthreads = use_numthreads self.threads = [Thread(target=self.worker) for i in range(self.numthreads)] for th in self.threads: th.daemon = True th.start() def precompute_projections(self,fM): if self.using_precomp_slicing: # precompute all possible slices fM = np.require(fM, dtype=np.float32) getslices_interp(fM, self.slice_ops, self.rad, self.beamstop_rad, res=self.slices.reshape(-1, self.N_T)) self.precomp_slices = self.slices.reshape((-1, self.N_T)) else: # do on-the-fly slicing self.precomp_slices = None def worker(self): g_tmp = None lcl_sigma2_est = None lcl_correlation = None lcl_power = None lcl_G = None workspace = None while True: idxs, fM, res, compute_grad = self.q.get() sigma2 = self.inlier_sigma2_trunc inlier_const = self.inlier_const - res['totallike_logscale'] if lcl_sigma2_est is None or lcl_sigma2_est.shape[0] != self.N_T: lcl_sigma2_est = np.zeros(self.N_T,dtype=np.float64) else: lcl_sigma2_est[:] = 0 if lcl_correlation is None or lcl_correlation.shape[0] != self.N_T: lcl_correlation = np.zeros(self.N_T,dtype=np.float64) else: lcl_correlation[:] = 0 if lcl_power is None or lcl_power.shape[0] != self.N_T: lcl_power = np.zeros(self.N_T,dtype=np.float64) else: lcl_power[:] = 0 # Result buffers like = res['like'] Evar_like = res['Evar_like'] if compute_grad: if lcl_G is None or lcl_G.shape != self.G.shape: lcl_G = np.zeros_like(self.G) else: lcl_G[:] = 0 for idx in idxs: tic = time.time() if self.sampler_S is not None: slice_ops, envelope, \ W_R_sampled, sampleinfo_R, slices_sampled, slice_inds, \ W_I_sampled, sampleinfo_I, rotd_sampled, rotc_sampled, \ W_S_sampled, sampleinfo_S, S_sampled = \ self.prep_operators(fM,idx,res=res) else: slice_ops, envelope, \ W_R_sampled, sampleinfo_R, slices_sampled, slice_inds, \ W_I_sampled, sampleinfo_I, rotd_sampled, rotc_sampled = \ self.prep_operators(fM,idx,res=res) N_slices = slices_sampled.shape[0] log_W_R = np.log(W_R_sampled) log_W_I = np.log(W_I_sampled) if self.sampler_S is not None: log_W_S = np.log(W_S_sampled) if compute_grad: if g_tmp is None or g_tmp.shape[0] < N_slices or g_tmp.shape[1] != self.N_T: g_tmp = np.empty((N_slices,self.N_T), dtype=self.G_datatype) else: g_tmp[:] = 0.0 g = g_tmp[0:N_slices] g[:] = 0 else: g = None res['kern_timing']['prep'][idx] = time.time() - tic # get angular correlation slices if self.use_angular_correlation: tic = time.time() # ac_slices_sampled, ac_data_sampled ac_indices = self.get_angular_correlation( slices_sampled, rotd_sampled, rotc_sampled, envelope, W_I_sampled) else: pass # add timing tic = time.time() if self.sampler_S is not None: raise NotImplementedError('no shifts') if self.use_angular_correlation: like[idx], (cphi_I, cphi_R), csigma2_est, ccorrelation, cpower, workspace = \ objective_kernels.doimage_ACRI(slices_sampled, envelope, \ rotc_sampled, rotd_sampled, \ # ac_slices_sampled, ac_data_sampled, \ ac_indices, log_W_I, log_W_R, \ sigma2, g, workspace) else: if len(W_I_sampled) == 1: like[idx], cphi_R, csigma2_est, ccorrelation, cpower, workspace = \ objective_kernels.doimage_R(slices_sampled, envelope, rotc_sampled.reshape((-1,)), rotd_sampled.reshape((-1,)), \ log_W_R, sigma2, g, workspace) cphi_I = np.array([0.0]) else: like[idx], (cphi_I, cphi_R), csigma2_est, ccorrelation, cpower, workspace = \ objective_kernels.doimage_RI(slices_sampled, envelope, \ rotc_sampled, rotd_sampled, \ log_W_I, log_W_R, \ sigma2, g, workspace) res['kern_timing']['work'][idx] = time.time() - tic tic = time.time() # like[idx] is the negative log likelihood of the image like[idx] += self.inlier_like_trunc[idx] # Evar_like[idx] is the expected error Evar_like[idx] = (csigma2_est.sum() + self.imgpower_trunc[idx]) / self.N**2 lcl_sigma2_est += csigma2_est lcl_correlation += ccorrelation lcl_power += cpower like[idx] += inlier_const if compute_grad: if self.using_precomp_slicing: lcl_G[slice_inds] += g else: lcl_G += merge_slices(g, slice_ops, self.N, self.rad, self.beamstop_rad).reshape(self.N, self.N, self.N) res['kern_timing']['proc'][idx] = time.time() - tic tic = time.time() if self.sampler_S is not None: self.store_results(idx, 1, \ cphi_R,sampleinfo_R, \ cphi_I,sampleinfo_I, \ cphi_S=cphi_S, sampleinfo_S=sampleinfo_S, res=res, \ logspace_phis = True) else: self.store_results(idx, 1, \ cphi_R,sampleinfo_R, \ cphi_I,sampleinfo_I, \ res = res, logspace_phis = True) res['kern_timing']['store'][idx] = time.time() - tic if compute_grad: self.G_lock.acquire() self.G += lcl_G self.G_lock.release() self.sigma_lock.acquire() res['sigma2_est'][self.truncmask] += lcl_sigma2_est/self.minibatch['N_M'] res['correlation'][self.truncmask] += lcl_correlation/self.minibatch['N_M'] res['power'][self.truncmask] += lcl_power/self.minibatch['N_M'] self.sigma_lock.release() self.q.task_done() def eval(self, fM, compute_gradient=True, M=None): tic = time.time() N_M = self.minibatch['N_M'] outputs = self.get_result_struct() outputs['like_timing'] = {} if compute_gradient: self.G[:] = 0 outputs['like_timing']['setup'] = time.time() - tic tic = time.time() self.precompute_projections(fM) outputs['like_timing']['slice'] = time.time() - tic tic = time.time() numJobs = min(N_M,3*self.numthreads) #numJobs = int(self.numthreads + self.numthreads/2) imsPerJob = int(np.ceil(float(N_M)/numJobs)) for jobId in xrange(numJobs): idxs = range(imsPerJob*jobId, \ min(imsPerJob*(jobId+1),N_M)) self.q.put(( idxs, fM, outputs, compute_gradient )) outputs['like_timing']['queue'] = time.time() - tic tic = time.time() self.q.join() outputs['like_timing']['join'] = time.time() - tic outputs['kern_timing'] = dict([(k,np.sum(v)/self.numthreads) for k,v in outputs['kern_timing'].items()]) # compute gradient of likelihood if compute_gradient: tic = time.time() if self.using_precomp_slicing: # dLdfM = self.slice_ops.T.dot(self.G.reshape((-1,))).reshape((self.N,self.N,self.N)) dLdfM = merge_slices(self.G, self.slice_ops, self.N, self.rad, self.beamstop_rad).reshape(self.N, self.N, self.N) dLdfM /= N_M else: dLdfM = self.G/N_M outputs['like_timing']['unslice'] = time.time() - tic outputs['N_R'] = self.N_R outputs['N_I'] = self.N_I if self.sampler_S is not None: outputs['N_S'] = self.N_S outputs['N_Total'] = self.N_R*self.N_I*self.N_S else: outputs['N_Total'] = self.N_R * self.N_I outputs['N_R_sampled_total'] = float(np.sum(outputs['N_R_sampled']))/self.N_R outputs['N_I_sampled_total'] = float(np.sum(outputs['N_I_sampled']))/self.N_I if self.sampler_S is not None: outputs['N_S_sampled_total'] = float(np.sum(outputs['N_S_sampled']))/self.N_S outputs['N_Total_sampled_total'] = float(np.sum(outputs['N_Total_sampled']))/(self.N_R*self.N_I*self.N_S) else: outputs['N_Total_sampled_total'] = float(np.sum(outputs['N_Total_sampled']))/(self.N_R * self.N_I) L = outputs['like'].sum(dtype=np.float64)/N_M outputs['L'] = L if compute_gradient: return L, dLdfM, outputs else: return L, outputs
class WorkerPool(object): """TODO:""" halt_command='halt command' def __init__(self,numWorkers=16,items=None): self.workers=[] self.should_stop=False self.work_queue=Queue() self.completed_queue=Queue() self.num_assigned=0 if items is not None: for item in items: self.work_queue.put(item) self.num_assigned += 1 for i in range(0,numWorkers): w = Worker("worker%d" % i,self) self.workers.append(w) w.start() self.numWorkers = numWorkers self.logger = logger ### def getNumWorkers(self): return self.numWorkers def getNextWorkItem(self): return self.work_queue.get(block=True) def addFinishedWorkItem(self,command): self.completed_queue.put(command) self.work_queue.task_done() def markTaskDone(self): self.work_queue.task_done() def addCommand(self,cmd): self.logger.debug("Adding cmd to work_queue: %s" % cmd.cmdStr) self.work_queue.put(cmd) self.num_assigned += 1 def wait_and_printdots(self,command_count,quiet=True): while self.completed_queue.qsize() < command_count: time.sleep(1) if not quiet: sys.stdout.write(".") sys.stdout.flush() if not quiet: print " " self.join() def print_progress(self, command_count): while True: num_completed = self.completed_queue.qsize() num_completed_percentage = 0 if command_count: num_completed_percentage = float(num_completed) / command_count logger.info('%0.2f%% of jobs completed' % (num_completed_percentage * 100)) if num_completed >= command_count: return time.sleep(10) def join(self): self.work_queue.join() return True def joinWorkers(self): for w in self.workers: w.join() def getCompletedItems(self): completedList=[] try: while True: item=self.completed_queue.get(False) if item is not None: completedList.append(item) except Empty: return completedList return completedList #just to be sure def check_results(self): """ goes through all items in the completed_queue and throws an exception at the first one that didn't execute successfully throws ExecutionError """ try: while True: item=self.completed_queue.get(False) if not item.get_results().wasSuccessful(): raise ExecutionError("Error Executing Command: ",item) except Empty: return def empty_completed_items(self): while not self.completed_queue.empty(): self.completed_queue.get(False) def isDone(self): #TODO: not sure that qsize() is safe return (self.num_assigned == self.completed_queue.qsize()) def haltWork(self): self.logger.debug("WorkerPool haltWork()") self.should_stop=True for w in self.workers: w.haltWork() self.work_queue.put(self.halt_command)
class Stats(object): def __request(self, path): r = self.__session.get(path) r.raise_for_status() return r.json() def __get_post_data(self): while not self.__ids_to_process.empty(): post_id = self.__ids_to_process.get() reactions = self.__request("https://graph.facebook.com/v2.10/" + post_id + "/reactions") while reactions["data"]: for reaction in reactions["data"]: user_id = reaction["id"] reaction_type = reaction["type"] self.__lock.acquire() self.__user_names[user_id] = reaction["name"] if user_id not in self.__users: self.__users[user_id] = {} self.__user_reactions[user_id] = 0 if reaction_type not in self.__users[user_id]: self.__users[user_id][reaction_type] = 0 if post_id not in self.__initial_top_posts: self.__users[user_id][reaction_type] += 1 self.__user_reactions[user_id] += 1 self.__total_reactions += 1 self.__posts[post_id] += 1 self.__lock.release() if "next" not in reactions["paging"]: break reactions = self.__request(reactions["paging"]["next"]) self.__ids_to_process.task_done() def __init__(self, access_token, post_ids, top_reactors=[], top_posts=[], total_posts=0, total_reactions=0, num_threads=64): self.__users = {} self.__user_names = {} self.__user_reactions = {} for user_info in top_reactors: user_id = user_info["id"] reactions = user_info["reactions"] self.__users[user_id] = reactions self.__user_names[user_id] = user_info["name"] self.__user_reactions[user_id] = sum(reactions.values()) self.__posts = {post_id: 0 for post_id in post_ids + top_posts} self.__total_posts = len(post_ids) + total_posts self.__total_reactions = total_reactions self.__initial_top_posts = set(top_posts) self.__session = Session() self.__session.params = {"access_token": access_token} self.__lock = threading.Lock() self.__ids_to_process = Queue() for post_id in post_ids + top_posts: self.__ids_to_process.put(post_id) for _ in range(num_threads): thread = threading.Thread(target=self.__get_post_data) thread.daemon = True thread.start() def __calculate_stats(self): self.__ids_to_process.join() try: self.__top_posts except AttributeError: self.__top_posts = sorted( [(post_id, reactions) for post_id, reactions in self.__posts.items()], key=lambda x: x[1], reverse=True) self.__top_reactors = [] for reactor_info in sorted( [(user_id, reactions) for user_id, reactions in self.__user_reactions.items()], key=lambda x: x[1], reverse=True): user_id = reactor_info[0] self.__top_reactors.append({ "id": user_id, "name": self.__user_names[user_id], "reactions": self.__users[user_id] }) def get_top_posts(self, num_posts=100): self.__calculate_stats() return self.__top_posts[0:num_posts] def get_top_post_message(self, num_posts=10): self.__calculate_stats() message = "***Top " + str(num_posts) + " Posts***\n" total_reactions = 0 ranking = 1 for post_id, reactions in self.get_top_posts(num_posts): message += str(ranking) + ". facebook.com/" + post_id + "\n" total_reactions += reactions ranking += 1 return message + "Average reactions per post: " + str( float(total_reactions) / num_posts) def get_top_reactors(self, num_reactors=100): self.__calculate_stats() return self.__top_reactors[0:num_reactors] def get_top_reactor_message(self, num_reactors=10): self.__calculate_stats() emoticons = { "LIKE": "\xF0\x9F\x91\x8D", "LOVE": "\xF0\x9F\x92\x9F", "HAHA": "\xF0\x9F\x98\x86", "WOW": "\xF0\x9F\x98\xAE", "SAD": "\xF0\x9F\x98\xA2", "ANGRY": "\xF0\x9F\x98\xA1", "THANKFUL": "\xF0\x9F\x8C\xB8", "PRIDE": "\xF0\x9F\x8C\x88" } message = "***Top " + str(num_reactors) + " Reactors***\n" ranking = 1 for reactor_info in self.get_top_reactors(num_reactors): user_id = reactor_info["id"] reactions = self.__user_reactions[user_id] reactions_breakdown = " ".join([ " ".join([emoticons[reaction_type], str(num)]) for (reaction_type, num) in sorted(self.__users[user_id].items(), key=lambda x: x[1], reverse=True) ]) message += str( ranking) + ". " + self.__user_names[user_id] + " - " + str( reactions) + ": " + reactions_breakdown.decode( "utf-8") + "\n" ranking += 1 return message + "Average reactions per post: " + str( float(self.__total_reactions) / float(self.__total_posts)) @property def total_posts(self): self.__calculate_stats() return self.__total_posts @property def total_reactions(self): self.__calculate_stats() return self.__total_reactions @property def reactors(self): self.__calculate_stats() return self.__user_names
accu = Thread(target=accumulate_checkins, name='AccumulateCheckins') accu.daemon = True accu.start() start = clock() end = start + ARGS.duration * 60 * 60 failures = th.Failures(initial_waiting_time=2.0) while clock() < end: try: read_twitter_stream(api, end) except (KeyboardInterrupt, SystemExit): CHECKINS_QUEUE.put_nowait(None) raise except: msg = 'Fail to read or enqueue tweet\n' cac.logging.exception(msg) waiting_time = failures.fail() if clock() + waiting_time > end or \ failures.recent_failures >= 5 or \ not accu.is_alive(): # We might as well quit right now, as stars are agains us CHECKINS_QUEUE.put_nowait(None) break msg = 'Will wait for {:.0f} seconds'.format(waiting_time) cac.logging.info(msg) failures.do_sleep() CHECKINS_QUEUE.join() report = 'insert {} valid checkins in {:.2f}s (out of {}).' cac.logging.info(report.format(NUM_VALID, clock() - start, NB_TWEETS)) sleep(5)
def check(testfile, indy_url, scan_dirs=False, vagrant_dir=None): with open(testfile) as f: build_config = yaml.safe_load(f) vagrant_dir = mb.vagrant.find_vagrant_dir(vagrant_dir) cwd = os.getcwd() try: project_dir = os.path.abspath(os.path.dirname(testfile)) reports_dir = os.path.join(project_dir, 'reports') if not os.path.isdir(reports_dir): os.makedirs(reports_dir) tid_base = "build_%s" % os.path.basename(project_dir) if build_config.get('vagrant') is not None: mb.vagrant.init_ssh_config(vagrant_dir) mb.vagrant.vagrant_env(build_config, 'pre-report', indy_url, vagrant_dir, project_dir, reports_dir) os.chdir(project_dir) report = build_config['report'] report_queue = Queue() try: if scan_dirs is True: task_ids = [] for builds_cand in os.listdir(project_dir): if builds_cand.startswith("builds"): for tid in os.listdir( os.path.join(project_dir, builds_cand)): if tid.startswith(tid_base): task_ids.append(tid) else: task_ids = mb.reporter.get_sealed_reports(indy_url) task_ids = [ tid for tid in task_ids if tid.startswith(tid_base) ] print "\n".join(task_ids) for t in range(int(report['threads'])): thread = mb.reporter.Reporter(report_queue) thread.daemon = True thread.start() for tid in task_ids: builddir = os.path.join(reports_dir, tid) if not os.path.isdir(builddir): os.makedirs(builddir) report_queue.put((builddir, indy_url, tid)) # mb.reporter.verify_report(builddir, args.indy_url, tid) report_queue.join() except (KeyboardInterrupt, SystemExit) as e: print e print "Quitting." if build_config.get('vagrant') is not None: mb.vagrant.vagrant_env(build_config, 'post-report', indy_url, vagrant_dir, project_dir, reports_dir) finally: os.chdir(cwd)
while True: sys.stdout.write( "Directories left / total: {0} / {1}, Time: {2} s \r".format( q.qsize(), s.qsize(), int(time.time() - start))) sys.stdout.flush() time.sleep(1) s = Queue() q = Queue() for i in range(args.workers): t = Thread(target=worker) t.daemon = True t.start() t = Thread(target=status) t.daemon = True t.start() if args.dirs: for file in open(args.dirs).read().splitlines(): q.put(file.rstrip()) else: q.put('/') q.join() print( "\nTotal time: {0} s, Total directories: {1}".format( int(time.time() - start)), s.qsize())
def add_data(h5_file, image_dir, prefix, args): # Make a list of all images in the source directory image_list = genlist(image_dir) # Shuffle to mix images for different styles np.random.shuffle(image_list) num_images = len(image_list) # Resize all images and copy them into the hdf5 file # We'll bravely try multithreading dset_name = os.path.join(prefix, 'images') dset_size = (num_images, 3, args.height, args.width) imgs_dset = h5_file.create_dataset(dset_name, dset_size, np.uint8) # input_queue stores (idx, filename) tuples, # output_queue stores (idx, resized_img) tuples input_queue = Queue() output_queue = Queue() # Read workers pull images off disk and resize them def read_worker(): while True: idx, filename = input_queue.get() img = imread(filename) try: # First crop the image so its size is a multiple of max_resize H, W = img.shape[0], img.shape[1] H_crop = H - H % args.max_resize W_crop = W - W % args.max_resize img = img[:H_crop, :W_crop] img = imresize(img, (args.height, args.width)) except (ValueError, IndexError) as e: print filename print img.shape, img.dtype print e input_queue.task_done() output_queue.put((idx, img)) # Write workers write resized images to the hdf5 file def write_worker(): num_written = 0 while True: idx, img = output_queue.get() if img.ndim == 3: #RGB image, transpose from H x W x C to C x H x W imgs_dset[idx] = img.transpose(2, 0, 1) #Throw gray images #elif img.ndim == 2: # Grayscale image; it is H x W so broadcasting to C x H x W will just copy # grayscale values into all channels. #imgs_dset[idx] = img output_queue.task_done() num_written = num_written + 1 if num_written % 100 == 0: print 'Copied %d / %d images' % (num_written, num_images) # Start the read workers. for i in xrange(args.num_workers): t = Thread(target=read_worker) t.daemon = True t.start() # h5py locks internally, so we can only use a single write worker =( t = Thread(target=write_worker) t.daemon = True t.start() for idx, filename in enumerate(image_list): if args.max_images > 0 and idx >= args.max_images: break input_queue.put((idx, filename)) input_queue.join() output_queue.join()
def build(testfile, indy_url, delay, vagrant_dir): with open(testfile) as f: build_config = yaml.safe_load(f) if vagrant_dir is not None: vagrant_dir = mb.vagrant.find_vagrant_dir(vagrant_dir) if delay is None: delay = 0 else: delay = int(delay) cwd = os.getcwd() try: project_dir = os.path.abspath(os.path.dirname(testfile)) builds_dir = "builds-%s" % dt.now().strftime("%Y%m%dT%H%M%S") tid_base = "build_%s" % os.path.basename(project_dir) build = build_config['build'] report = build_config.get('report') if build_config.get('vagrant') is not None: mb.vagrant.init_ssh_config(vagrant_dir) mb.vagrant.vagrant_env(build_config, 'pre-build', indy_url, vagrant_dir, project_dir, os.path.join(project_dir, builds_dir)) os.chdir(project_dir) project_src_dir = build.get('project-dir') or 'project' project_src_dir = os.path.join(os.getcwd(), project_src_dir) git_branch = build.get('git-branch') or 'master' build_queue = Queue() report_queue = Queue() try: for t in range(int(build['threads'])): thread = mb.builder.Builder(build_queue, report_queue) thread.daemon = True thread.start() for x in range(build['builds']): builddir = mb.util.setup_builddir(builds_dir, project_src_dir, git_branch, tid_base, x) build_queue.put( (builddir, indy_url, build_config.get('proxy-port'), (x % int(build['threads'])) * int(delay))) build_queue.join() if build_config.get('vagrant') is not None: mb.vagrant.vagrant_env(build_config, 'post-build', indy_url, vagrant_dir, project_dir, builds_dir) mb.vagrant.vagrant_env(build_config, 'pre-report', indy_url, vagrant_dir, project_dir, builds_dir) if report is not None: if build_config.get('vagrant') is not None: mb.vagrant.vagrant_env(build_config, 'pre-report', indy_url, vagrant_dir, project_dir, builds_dir) for t in range(int(report['threads'])): thread = mb.reporter.Reporter(report_queue) thread.daemon = True thread.start() report_queue.join() except Exception as e: print e print "Quitting." if build_config.get('vagrant') is not None: mb.vagrant.vagrant_env(build_config, 'post-report', indy_url, vagrant_dir, project_dir, builds_dir) finally: os.chdir(cwd)
class MultiThreadESSearchClient(): def __init__(self, host, query_template, search_uris, concurrency, threads_per_second, requests_queue=None): self.host = host self.query_template = query_template self.requests_queue = Queue(maxsize=concurrency) if not isinstance( requests_queue, Queue) else requests_queue self.sess = self.create_request_session() self.search_uris = search_uris self.threads_per_second = threads_per_second self.requests_count = 0 self.initial_requests_start_time = time() self.start_threads(concurrency) def start_threads(self, concurrency): # Define threads for i in range(concurrency): t = Thread(target=self.do_work) t.daemon = True t.start() def fire_requests(self): for uri in self.search_uris: es_query_keyword = self.get_query_word_from_url(uri) request_params = dict( search_uri=uri.replace('\r\n', ''), query_keyword=es_query_keyword ) self.requests_queue.put(request_params) # Block until all items in the queue have been handled self.requests_queue.join() # Output total execution time after all requests are fired total_time_elapsed = time() - self.initial_requests_start_time print("\nTotal request time in seconds: %s" % total_time_elapsed) def do_work(self): while True: # Initial start time if not hasattr(self, 'start_time'): self.start_time = time() self.elapsed_time = time() - self.start_time # If we have passed 1 seconds or had more than threads_per_seconds requestswithin a second if self.elapsed_time >= 1 or self.requests_count > self.threads_per_second: self.start_time = time() self.requests_count = 0 # If had more requests than threads_per_seconds then pause for the diff time if self.elapsed_time < 1: sleep(1 - self.elapsed_time) request_params = self.requests_queue.get() response = self.perform_es_request(**request_params) self.output_results(response, **request_params) self.requests_queue.task_done() self.requests_count += 1 def perform_es_request(self, query_keyword, search_uri): params = dict( url=self.host + '/_search', headers={"content-type": 'application/json'}, data=self.query_template.replace("{{queryKeyword}}", query_keyword) ) start_time = time() response = self.do_request(**params) end_time = time() response.elapsed_time = (end_time - start_time) * 1000 return response def do_request(self, **kwargs): try: response = self.sess.post( timeout=60, **kwargs ) return response except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e: print e.message raise e def output_results(self, response, query_keyword, search_uri): response_to_json = response.json() print('%s,%s,%s,%s' % ( response_to_json['took'], response.elapsed_time, search_uri, query_keyword)) def create_request_session(self): sess = requests.Session() adapter = requests.adapters.HTTPAdapter( pool_connections=500, pool_maxsize=100, max_retries=3 ) sess.mount('http://', adapter) return sess def get_query_word_from_url(self, url): url = unquote(url).decode('utf-8') return re.search('q=([^\r]+)&', url).group(1).split('&')[0]
class KafkaThreadedImageDownloader(KafkaImageDownloader): def __init__(self, global_conf_filename, prefix=default_prefix, pid=None): super(KafkaThreadedImageDownloader, self).__init__(global_conf_filename, prefix, pid) # Get number of threads self.nb_threads = self.get_required_param('nb_threads') def set_pp(self): self.pp = "KafkaThreadedImageDownloader" if self.pid: self.pp += "." + str(self.pid) def process_one(self, msg): self.print_stats(msg) msg_value = json.loads(msg.value) # From msg value get list_urls for image objects only list_urls = self.get_images_urls(msg_value) nb_img = len(list_urls) if self.verbose > 2: print_msg = "[{}.process_one: info] Got {} image urls from ad id {}" print print_msg.format(self.pp, nb_img, msg_value['_id']) # Initialize queues from Queue import Queue self.q_in = Queue(0) self.q_out = Queue(0) threads = [] if list_urls: # Fill input queue for item in list_urls: self.q_in.put(item) # Start threads, at most one per image for i in range(min(self.nb_threads, len(list_urls))): # should read (url, obj_pos) from self.q_in # and push (url, obj_pos, buffer, img_info, start_process, end_process) to self.q_out thread = ThreadedDownloader(self.q_in, self.q_out) thread.start() threads.append(thread) # Wait for all tasks to be marked as done if self.q_in.qsize() > 0: self.q_in.join() # Get images data and infos dict_imgs = dict() nb_dl_img = 0 nb_valid_dl_img = 0 while nb_dl_img < nb_img: url, obj_pos, img_buffer, img_info, start_process, end_process, inst = self.q_out.get( ) nb_dl_img += 1 if img_buffer is not None and img_info is not None: # TODO: should we just discard buffer here?... sha1, img_type, width, height = img_info dict_imgs[url] = { 'obj_pos': obj_pos, 'img_buffer': img_buffer, 'sha1': sha1, 'img_info': { 'format': img_type, 'width': width, 'height': height } } nb_valid_dl_img += 1 self.toc_process_ok(start_process, end_process) else: self.toc_process_failed(start_process, end_process) if inst is not None: if self.verbose > 0: print_msg = "[{}.process_one: error] Could not download image from: {} ({})" print print_msg.format(self.pp, url, inst) sys.stdout.flush() else: if self.verbose > 1: print_msg = "[{}.process_one: info] Could not download image from: {} (tried downloading for {}s)" print print_msg.format(self.pp, url, end_process - start_process) sys.stdout.flush() if self.verbose > 0: if nb_valid_dl_img < nb_dl_img: print_msg = "[{}.process_one: info] Found only {} valid images out of {} for ad id {}" print print_msg.format(self.pp, nb_valid_dl_img, nb_dl_img, msg_value['_id']) sys.stdout.flush() # Push to cdr_out_topic #self.producer.send(self.cdr_out_topic, self.build_cdr_msg(msg_value, dict_imgs)) if self.cdr_out_topic: self.producer.send(self.cdr_out_topic, self.build_cdr_msg(msg_value, dict_imgs)) else: print_msg = "[{}.process_one: warning] cdr_out_topic is not defined" print print_msg.format(self.pp) # Push to images_out_topic for img_out_msg in self.build_image_msg(dict_imgs): self.producer.send(self.images_out_topic, img_out_msg)
class CappedQueue(object): """ Capped queue with opiniatied methods """ def __init__(self, maxsize=None, name=None): if maxsize is None: maxsize = CONFIG["MAX_QUEUE_LENGTH"] self.maxsize = maxsize self.queue = Queue(self.maxsize) self.name = name or "queue" self._count_ingress = itertools.count() self._count_dropped = itertools.count() self._count_egress = itertools.count() self._last_count_ingress = 0 self._last_count_dropped = 0 self._last_count_egress = 0 def get(self, timeout, block=True): """ Wait for up to timeout for an item to return and block while waiting """ ret = self.queue.get(timeout=timeout, block=block) next(self._count_egress) return ret def get_nowait(self): """ Get without waiting, raise queue.Empty if nothing is present """ ret = self.queue.get_nowait() next(self._count_egress) return ret def put(self, item): """ Tries to put an item to the queue, if the queue is full, pop an item and try again """ while True: try: ret = self.queue.put_nowait(item) next(self._count_ingress) return ret except Full: try: self.queue.get_nowait() self.queue.task_done() next(self._count_dropped) except Empty: pass def half_full(self): """ Return True if the current queue size if at least half the maxsize """ return self.queue.qsize() > (self.maxsize / 2.) def get_metrics(self, at=None): """ Return the queue metrics, this method is *NOT* thread-safe. """ ingress_delta = next(self._count_ingress) - self._last_count_ingress dropped_delta = next(self._count_dropped) - self._last_count_dropped egress_delta = next(self._count_egress) - self._last_count_egress self._last_count_ingress += ingress_delta + 1 self._last_count_dropped += dropped_delta + 1 self._last_count_egress += egress_delta + 1 at = at or now() return [ ("event_manager", at, "{}_ingress".format(self.name), ingress_delta), ("event_manager", at, "{}_dropped".format(self.name), dropped_delta), ("event_manager", at, "{}_egress".format(self.name), egress_delta), ] def task_done(self): """ Inform the queue that an event was processed. """ self.queue.task_done() def join(self): """ Block until all events are consumed """ self.queue.join()
import threading import time import sys from Queue import Queue from utils import do_work pending_tasks = Queue() def do_work_from_queue(): while True: do_work(pending_tasks.get()) pending_tasks.task_done() if __name__ == "__main__": tasks = [sys.argv[1]] * int(sys.argv[2]) num_threads = int(sys.argv[3]) if len(sys.argv) > 3 else 4 [pending_tasks.put(task) for task in tasks] start = time.time() for i in range(num_threads): t = threading.Thread(target=do_work_from_queue) t.daemon = True t.start() pending_tasks.join() print time.time() - start
class VideoRecorder: def __init__(self): "docstring" try: self.camera = PiCamera() self.set_camera_params() # self.camera.start_preview() #self.camera.start_recording(self.my_stream, format="h264") channel = grpc.insecure_channel('200.126.23.95:50052') self.grpc_stub = FeatureExtractionApi_pb2_grpc.FeatureExtractionStub( channel) self.recording_stop = True self.stream = io.BytesIO() self.image_queue = Queue() self.count = 0 logger.debug("Camera and grpc started") except (PiCameraMMALError, PiCameraError) as error: raise ConnectionError("Camera not available") print(error) def set_camera_params(self): self.camera.resolution = CAMERA['resolution'] self.camera.framerate = CAMERA['framerate'] self.camera.brightness = CAMERA['brightness'] self.camera.saturation = CAMERA['saturation'] self.camera.contrast = CAMERA['contrast'] self.camera.hflip = True self.camera.vflip = True time.sleep(5) def capture_continuous(self, filename): print("capture continuos") try: self.count = 1 # Use the video-port for captures... for foo in self.camera.capture_continuous(self.stream, 'jpeg', use_video_port=True): print("New frame: #", self.count) self.stream.seek(0) self.image_queue.put( Image(source=self.stream.read(), file_name=filename, timestamp=str(datetime.now()))) if self.recording_stop: break self.stream.seek(0) self.stream.truncate() self.count += 1 finally: self.stream.seek(0) self.stream.truncate() def generate_videos_iterator(self): print("generate video iterator") count = 1 while not self.recording_stop or not self.image_queue.empty(): try: yield self.image_queue.get(block=True, timeout=1) self.image_queue.task_done() print "sent", count, "of", self.count, "captured" count += 1 except Empty as ex: print("No data in image queue") print("Done generating images") def start_recording(self, filename): print("Start recording") threading.Thread(target=self.capture_continuous, args=(filename, )).start() videos_iterator = self.generate_videos_iterator() response = self.grpc_stub.processVideo(videos_iterator) print(response) def record(self): filename = CONFIGS["session"] self.recording_stop = False threading.Thread(target=self.start_recording, args=(filename, )).start() def stop_record(self): self.recording_stop = True self.image_queue.join() def clean(self): self.camera.close() def convert_to_mp4(self): filename_mp4 = self.filename.split(".")[0] + ".mp4" print("file .h264 saved.. Transforming to mp4...") os.system("MP4Box -fps 30 -add " + self.filename + " " + filename_mp4) print("File converted to mp4")
#print "Starting thread [" + str(i)+"]\n" time.sleep(5) while True: proxy = q.get() # get proxy proxy_check(proxy) q.task_done() if (len(sys.argv) < 4): print( "\n\nUsage: programe.py proxy_list.txt output_proxies.txt LMH \n\nType\n L = Low\n M = Med\n H = High\n" ) else: num_threads = 300 q_queue = Queue() for i in range(num_threads): # number of threads to spin up worker = Thread(target=get_proxy, args=( i, q_queue, )) worker.setDaemon(True) worker.start() with open(sys.argv[1]) as fp: for line in fp: q_queue.put(line) # put list in queue q_queue.join() # wehen queue is empty exit print 'Finished'
class AsyncCamera: def __init__(self, driver): self._driver = driver # NOTE: command queue is not necessary anymore, PTZ commands # create a new thread, login/out could do the same self._cmd_queue = Queue() self._cmd_thread = Thread(target=self._control_fn) self._cmd_thread.daemon = True self._cmd_thread.start() self._cmd_queue.put((self._driver.login, None)) self._cmd_queue.join() # will block creator try: self._pan, self._tilt, self._zoom = self._driver.getptz() except: self._pan, self._tilt, self._zoom = 0, 0, 0 self._kick_thread = Thread(target=self._kick_fn) self._kick_thread.daemon = True self._kick_thread.start() self._jpg_queue = Queue() self._is_previewing = False self._preview_thread = Thread(target=self._preview_fn) self._preview_thread.daemon = True self._preview_thread.start() def _control_fn(self): '''Receives commands through queue and executes them in sequence''' while True: # TODO: use variable number of arguments fn, args = self._cmd_queue.get() if args is None: fn() else: fn(args[0], args[1]) self._cmd_queue.task_done() def _kick_fn(self): '''Keeps kicking the camera to keep session alive''' while True: sleep(10) if True != self._driver.kick(): self._driver.logout() self._driver.login() def _preview_fn(self): '''Keeps fetching preview images. Fetches JPEG images one-by-one, as fast as possible.''' while True: if not self._is_previewing: # reduce CPU load by avoiding busy looping sleep(.5) continue jpg = self._driver.getjpg() if bool(jpg): logging.info('preview: {0} bytes'.format(len(jpg))) self._jpg_queue.put(jpg) def close(self): self._cmd_queue.put((self._driver.logout, None)) self._cmd_queue.join() def pantilt(self, pan=0, tilt=0): '''Pan and tilt camera to the given angles (degrees)''' self._pan, self._tilt = pan, tilt # Create a thread for the potentially long-running command. # This means commands can run parallel (out-of-order). # If this is not desirable (not sure?) use the command queue. Thread(target=self._driver.pantilt, args=(pan, tilt)).start() def zoom(self, zoom=10, speed=1): '''Set zoom level using given speed''' self._zoom = zoom # Create a thread for the potentially long-running command. # This means commands can run parallel (out-of-order). # If this is not desirable (not sure?) use the command queue. Thread(target=self._driver.zoom, args=(zoom, speed)).start() def getjpg(self): ''' Return the most recent image received by the preview thread. ''' jpg = None while not self._jpg_queue.empty(): jpg = self._jpg_queue.get() return jpg def ptz(self): return self._pan, self._tilt, self._zoom def start_preview(self): self._is_previewing = True def stop_preview(self): self._is_previewing = False # TODO: use decorator def start_recording(self): Thread(target=self._driver.start_recording).start() def stop_recording(self): Thread(target=self._driver.stop_recording).start()
class ThreadTool: def __init__(self, isThread=1, needfinishqueue=0, deamon=True): self.isThread = isThread self.idletask = {} self.Threads = [] self.alivenum = 0 self.needfinishqueue = needfinishqueue self.running = 0 self.threads_num = 10 self.deamon = deamon self.job = None self.default_object = None if self.isThread == 1: # self.lock = Lock() #线程锁 self.lock = threading.Lock() self.q_request = Queue() #任务队列 if needfinishqueue > 0: self.q_finish = Queue() #完成队列 elif self.isThread == 0: self.lock = multiprocessing.Lock() self.q_request = multiprocessing.Queue() if self.needfinishqueue > 0: self.q_finish = multiprocessing.Queue() else: from gevent.queue import JoinableQueue as geventqueue from gevent.lock import Semaphore self.lock = Semaphore() self.q_request = geventqueue() if self.needfinishqueue > 0: self.q_finish = geventqueue() def __del__(self): #解构时需等待两个队列完成 time.sleep(0.5) if self.isThread == 1 or self.isThread == 2: self.q_request.join() if self.needfinishqueue > 0: self.q_finish.join() def getqueue_size(self): return self.q_request.qsize() def set_Thread_size(self, threads_num=10): self.threads_num = threads_num def init_add(self, add_init_object): self.default_object = add_init_object def add_task(self, job): self.job = job #获取当前剩余的任务,用于集群操做 def get_work(self): tmparray = [] if self.q_request.qsize() > 0: try: req = self.q_request.get(block=True, timeout=4) tmparray.append(req) return tmparray except: return tmparray else: return tmparray def start(self): sizenumber = min(self.threads_num, self.q_request.qsize()) if self.isThread == 1: for i in range(sizenumber): t = Thread(target=self.getTask) print '线程' + str(i + 1) + ' 正在启动' t.setDaemon(self.deamon) t.start() self.Threads.append(t) with self.lock: self.alivenum += 1 elif self.isThread == 0: for i in range(sizenumber): t = multiprocessing.Process(target=self.getTaskProcess) print '进程' + str(i + 1) + ' 正在启动' t.Daemon = self.deamon t.start() self.Threads.append(t) with self.lock: self.alivenum += 1 else: for i in range(sizenumber): t = gevent.spawn(self.getgeventTask) print '协程' + str(i + 1) + ' 正在启动' self.Threads.append(t) with self.lock: self.alivenum += 1 def get_running_size(self): return self.running def taskleft(self): if self.needfinishqueue > 0: return self.q_request.qsize() + self.q_finish.qsize( ) + self.running else: return self.q_request.qsize() + self.running def push(self, req): sizenum = len(req) for urls in req: self.q_request.put(urls) threadnownum = 0 threaddie = [] dienum = 0 if self.isThread == 1: tempnumb = 0 with self.lock: tempnumb = self.alivenum if tempnumb < self.threads_num: for item in self.Threads: if item.isAlive(): threadnownum = threadnownum + 1 with self.lock: print str(threadnownum) + '活着的线程数' self.Threads = filter(lambda x: x.isAlive() != False, self.Threads) print str(len(self.Threads)) + '清理后活着的进程数' else: threadnownum = self.threads_num elif self.isThread == 0: tempnumb = 0 with self.lock: tempnumb = self.alivenum if tempnumb < self.threads_num: for item in self.Threads: if item.is_alive(): threadnownum = threadnownum + 1 with self.lock: print str(threadnownum) + '活着的进程数' self.Threads = filter(lambda x: x.is_alive() != False, self.Threads) print str(len(self.Threads)) + '清理后活着的进程数' else: threadnownum = self.threads_num sizenumber = min(self.threads_num - threadnownum, sizenum) if self.isThread == 1: for i in range(sizenumber): t = Thread(target=self.getTask) t.Daemon = self.deamon t.start() self.Threads.append(t) with self.lock: self.alivenum += 1 elif self.isThread == 0: for i in range(sizenumber): t = multiprocessing.Process(target=self.getTaskProcess) t.Daemon = self.deamon t.start() self.Threads.append(t) with self.lock: self.alivenum += 1 else: for i in range(self.threads_num): print 'alive num', self.alivenum, self.threads_num if self.alivenum < self.threads_num: t = gevent.spawn(self.getgeventTask) print '协程' + str(self.alivenum) + ' 正在启动' self.Threads.append(t) with self.lock: self.alivenum += 1 else: break self.q_request.join() def pop(self): return self.q_finish.get() def do_job(self, job, req, threadname): return job(req, threadname) def getTaskProcess(self): while True: # if self.taskleft()>0: # try: # req = self.q_request.get(block=True,timeout=10000) # except: # continue # else: # threadname=multiprocessing.current_process().name # print threadname+'关闭' # with self.lock: # self.alivenum-=1 # break req = self.q_request.get() with self.lock: #要保证该操作的原子性,进入critical area self.running = self.running + 1 threadname = multiprocessing.current_process().name print '进程' + threadname + '发起请求: ' ans = self.do_job(self.job, req, threadname) # ans = self.connectpool.getConnect(req) # self.lock.release() if self.needfinishqueue > 0: self.q_finish.put((req, ans)) # self.lock.acquire() with self.lock: self.running = self.running - 1 threadname = multiprocessing.current_process().name print '进程' + threadname + '完成请求' # self.lock.release() #self.q_request.task_done() def getTask(self): while True: # if self.taskleft()>0: # try: # req = self.q_request.get(block=True,timeout=10000) # except: # continue # else: # threadname=threading.currentThread().getName() # with self.lock: # self.alivenum-=1 # print threadname+'关闭' # break req = self.q_request.get() with self.lock: #要保证该操作的原子性,进入critical area self.running = self.running + 1 threadname = threading.currentThread().getName() print '线程' + threadname + '发起请求: ' ans = self.do_job(self.job, req, threadname) # ans = self.connectpool.getConnect(req) # self.lock.release() if self.needfinishqueue > 0: self.q_finish.put((req, ans)) # self.lock.acquire() with self.lock: self.running -= 1 threadname = threading.currentThread().getName() print '线程' + threadname + '完成请求' # self.lock.release() self.q_request.task_done() def getgeventTask(self): while True: # if self.taskleft()>0: # try: # req = self.q_request.get(block=True,timeout=10000) # except: # continue # else: # threadname=threading.currentThread().getName() # with self.lock: # self.alivenum-=1 # print threadname+'关闭' # break req = self.q_request.get() with self.lock: #要保证该操作的原子性,进入critical area self.running = self.running + 1 threadname = gevent.getcurrent() print threadname, '协程发起请求: ' ans = self.do_job(self.job, req, threadname) # ans = self.connectpool.getConnect(req) # self.lock.release() if self.needfinishqueue > 0: self.q_finish.put((req, ans)) # self.lock.acquire() with self.lock: self.running -= 1 threadname = gevent.getcurrent() print threadname, '协程发起请求: ' # self.lock.release() self.q_request.task_done()
def main(): start_time = time.time() #Read in arguments parser = argparse.ArgumentParser(description='Chunk a file and then kick' ' off multiple SCP threads.' 'Speeds up transfers over ' 'high latency links') parser.add_argument('-c', '--cypher', help='cypher use with from transfer see: ssh', default=default_cypher, required=False) parser.add_argument('-s', '--size', help='size of chunks to transfer.', default='500M', required=False, type=human_sizes) parser.add_argument('-r', '--retries', help='number of times to retry transfer.', default=default_retries, required=False, type=int) parser.add_argument('-t', '--threads', help='number of threads (default ' + str(default_num_threads) + ')', default=default_num_threads, required=False, type=int) parser.add_argument('src', help='source file') parser.add_argument('srv', help='remote server and user if required' ' e.g [email protected]') parser.add_argument('dst', help='directory (if remote home dir then specify . )') args = parser.parse_args() ssh_crypto = args.cypher try: chunk_size = human2bytes(args.size) except ValueError as e: print 'Invalid chunk size ' + str(e) exit(1) num_threads = args.threads src_file = args.src dst_file = args.dst remote_server = args.srv retries = args.retries (dest_path, _) = os.path.split(dst_file) (_, src_filename) = os.path.split(src_file) remote_dest_file = os.path.join(dest_path, src_filename) remote_chunk_files = [] # Check args for errors + instantiate variables. if not os.path.exists(src_file): print 'Error: Source file does not exist', src_file exit(1) if not os.path.isfile(src_file): print 'Error: Source is not a file', src_file exit(1) src_file_size = os.stat(src_file).st_size # Split file and calc the file md5 local_chunk_start_time = time.time() print "spliting file" spinner = spinning_cursor() sys.stdout.write(spinner.next()) sys.stdout.flush() sys.stdout.write('\b') (src_file_info, chunk_infos) = split_file_and_md5(src_file, src_filename, chunk_size) src_file_md5 = src_file_info[1] local_chunk_end_time = time.time() print "uploading MD5 ({0!s}) checksum to remote site".format(src_file_md5) try: checksum_filename = src_file + '.md5' dest_checksum_filename = os.path.join(dest_path, src_filename + '.md5') with open(checksum_filename, 'w+') as checksum_file: checksum_file.write(src_file_md5 + ' ' + src_filename) print 'copying ' + src_file + ' to ' + dest_checksum_filename subprocess.check_call(['scp', '-c' + ssh_crypto, '-q', '-oBatchMode=yes', checksum_filename, remote_server + ':' + \ dest_checksum_filename]) except CalledProcessError as e: print(e.returncode) print "ERROR: Couldn't connect to remote server." exit(1) # Fill the queue of files to transfer q = Queue() chunk_num = 1 total_chunks = len(chunk_infos) for (src_chunk_filename, chunk_md5) in chunk_infos: # create destination path (_, src_filename) = os.path.split(src_chunk_filename) dest_chunk_filename = os.path.join(dest_path, src_filename) remote_chunk_files.append( (src_chunk_filename, dest_chunk_filename, chunk_md5)) q.put((src_chunk_filename, dest_chunk_filename, chunk_num, total_chunks, retries)) chunk_num = chunk_num + 1 # Kick off threads transfer_start_time = time.time() print "starting transfers" for i in range(num_threads): t = WorkerThread(q, dst_file, remote_server, ssh_crypto) t.daemon = True t.start() q.join() transfer_end_time = time.time() #join the chunks back together and check the md5 print "re-assembling file at remote end" remote_chunk_start_time = time.time() chunk_count = 0 for (chunk_filename, chunk_md5) in chunk_infos: (path, remote_chunk_filename) = os.path.split(chunk_filename) remote_chunk_file = os.path.join(dest_path, remote_chunk_filename) spin('processing ' + remote_chunk_filename) if chunk_count: cmd = remote_chunk_file + '>> ' + remote_dest_file else: #truncate if the first chunk cmd = remote_chunk_file + '> ' + remote_dest_file subprocess.call(['ssh', remote_server, 'cat', cmd]) chunk_count += 1 print print 're-assembled' remote_chunk_end_time = time.time() print "checking remote file checksum" remote_checksum_start_time = time.time() try: # use openssl to be cross platform (OSX,Linux) checksum = subprocess.check_output( ['ssh', remote_server, 'openssl', 'md5', remote_dest_file]) # MD5(2GB.mov)= d8ce4123aaacaec671a854f6ec74d8c0 if checksum.find(src_file_md5) != -1: print 'PASSED checksums match' else: print 'ERROR: MD5s do not match local(' + src_file_md5 + \ ') != (' + checksum.strip() + ')' print ' File uploaded with errors - MD5 did not match.' print ' local and remote chunks not cleared up' exit(1) except CalledProcessError as e: print(e.returncode) print 'ERROR: File uploaded with errors - MD5 did not match.' print ' local and remote chunks not cleared up' exit(1) remote_checksum_end_time = time.time() # clean up print "cleaning up" print "removing file chunks" for (local_chunk, remote_chunk, chunk_md5) in remote_chunk_files: spin("removing file chunk " + local_chunk) os.remove(local_chunk) try: subprocess.call(['ssh', remote_server, 'rm', remote_chunk]) except CalledProcessError as e: print(e.returncode) print 'ERROR: failed to remove remote chunk ' + remote_chunk print '' print "transfer complete" end_time = time.time() print "-" * 80 print "file size :" + bytes2human(src_file_size) + "B" print "transfer rate :" + bytes2human(src_file_size / int(transfer_end_time - \ transfer_start_time)) + "B/s" print " :" + bytes2human((src_file_size * 8) / int(transfer_end_time - \ transfer_start_time)) + "b/s" print "transfer time :" + str(humanize_time_to_string( \ humanize_time( \ int(transfer_end_time - \ transfer_start_time), "seconds"))) print "local chunking time :" + str(humanize_time_to_string( \ humanize_time( \ int(local_chunk_end_time - \ local_chunk_start_time), "seconds"))) print "remote reassembly time :" + str(humanize_time_to_string( \ humanize_time( \ int(remote_chunk_end_time - \ remote_chunk_start_time), "seconds"))) print "remote checksum time :" + str(humanize_time_to_string( humanize_time( \ int(remote_checksum_end_time - \ remote_checksum_start_time), \ "seconds"))) print "total transfer rate :" + bytes2human(src_file_size / \ int(end_time - \ start_time)) + "B/s" print " :" + bytes2human((src_file_size * 8) / \ int(end_time - \ start_time)) + "b/s" print "total time :" + str(humanize_time_to_string( \ humanize_time(int(end_time - \ start_time), "seconds"))) exit(0)
class GeoGroupTable: '''Group OSM nodes by their geographical coordinates. The coordinates of the globe are partitioned into disjoint areas. Each partition is named by the geohash code of its (n,w) corner. Grouping of nodes is implemented by restricting the length of the geohash codes used. ''' def __init__(self, config, options, db): '''Initialize the table. Keyword arguments: config - A ConfigParser instance. options - An optparse.OptionParser structure. db - A DB object supporting 'get()' and 'store()' methods. ''' self.geodb = collections.defaultdict(NodeGroup) self.db = db lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE) self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb) if options.nothreading: nthreads = 0 else: nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS) self.nthreads = max(0, nthreads) if self.nthreads: self.wrthreads = [] self.wrqueue = Queue(self.nthreads) self.wrcond = threading.Condition() self.wrpending = [] for n in range(self.nthreads): t = threading.Thread(target=self._worker) t.name = "GeoWB-%d" % n t.daemon = True self.wrthreads.append(t) t.start() db.register_threads(self.wrthreads) def _cb(self, key, value): "Callback called when an LRU item is ejected." nodeset = self.geodb.pop(key) if self.nthreads: # Defer processing to a worker thread. self.wrqueue.put((key, nodeset)) else: # Synchronous operation. self._write_geodoc(key, nodeset) def _worker(self): "Helper method, used by worker threads." while True: # Retrieve a work item. v = self.wrqueue.get() if v is None: # Exit the thread. self.wrqueue.task_done() return # Unpack the work item. key, nodeset = v # Mark the item as "I/O in progress". with self.wrcond: while key in self.wrpending: self.wrcond.wait() assert key not in self.wrpending self.wrpending.append(key) # Process this node set. self._write_geodoc(key, nodeset) # Remove the "I/O in progress" marker. with self.wrcond: assert key in self.wrpending self.wrpending.remove(key) self.wrcond.notifyAll() self.wrqueue.task_done() def _write_geodoc(self, key, nodegroup): "Merge in a group of nodes into a geodoc." assert isinstance(nodegroup, NodeGroup) geodoc = self.db.retrieve_element(C.GEODOC, key) if geodoc is None: # New document. geodoc = new_osm_element(C.GEODOC, key) nodegroup.update(geodoc[C.NODES]) geodoc[C.NODES] = nodegroup.aslist() self.db.store_element(C.GEODOC, key, geodoc) def add(self, elem): '''Add information about a node 'elem' to the geo table. Usage: >>> gt = GeoGroupTable() >>> gt = gt.add(elem) The node 'elem' should have a 'lat' and 'lon' fields that encode its latitude and longitude respectively. The 'id' field specifies the node's "id". ''' assert elem.namespace == C.NODE, "elem is not a node: %s" % str(elem) # Determine the geo-key for the node. ghkey = geohash_key_for_element(elem) # Retrieve the partition covering this location. ghdoc = self.geodb[ghkey] elemid = elem.id if elemid not in ghdoc: ghdoc.add(elem) self.lru[ghkey] = ghdoc def flush(self): "Wait pending I/Os" # Flush items from the LRU. self.lru.flush() if self.nthreads: # Wait for the work queue to drain. self.wrqueue.join()
def recdata(name): if name == "recipelist": foodtype = request.query.type recipeCount = request.query.number #print recipeCount foodtype.replace ("%20", "+") apicall = "https://spoonacular-recipe-food-nutrition-v1.p.mashape.com/recipes/searchComplex?addRecipeInformation=true&cuisine=mexican&diet=vegetarian&excludeIngredients=onions%2C+garlic&fillIngredients=true&includeIngredients=tomato%2C+cilantro%2C+pepper%2C+carrot%2C+Coriander%2C+Cumin%2C+Beans%2C+potatoes%2C+capsicum&intolerances=egg%2C+seafood%2C+shellfish&limitLicense=false&maxCalories=1500&maxCarbs=100&maxFat=100&maxProtein=100&minCalories=150&minCarbs=5&minFat=5&minProtein=5&number="+recipeCount+"&offset=0&query=<required>&ranking=1&type="+foodtype response = unirest.get(apicall, headers={ "X-Mashape-Key": "FM6rhmzOJ1mshE29nA3PzJNQSNjup1cziiQjsnXdRgxLo859D4", "Accept": "application/json" } ) #print response.body with io.open('recjsonone.json', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(response.body,ensure_ascii=False))) with open('recjsonone.json') as data_file: data = json.load(data_file) counter = 0; # Create a queue to communicate with the worker threads queue = Queue() # Create 10 worker threads for x in range(10): worker = RecipeRunner(queue) # Setting daemon to True will let the main thread exit even though the workers are blocking worker.daemon = True worker.start() for val in data: if val == 'results': listrecm = [] for recipe in data[val]: if len(recipe) <=13: continue queue.put((recipe,listrecm,recipe['pricePerServing'])) #recipelistgen(recipe,listrecm,recipe['pricePerServing']) queue.join() with io.open('recresponse.json', 'w', encoding='utf-8') as f: f.write(unicode(json.dumps(listrecm,ensure_ascii=False))) for item in listrecm: del item['cookingInstructions'] return dict(data=listrecm) elif name == "cookingsteps": with open('recresponse.json') as data_file: data = json.load(data_file) recipeid = request.query.id listrecm = [] for item in data: if str(item['id']).strip() != str(recipeid).strip(): continue listrecm = item return dict(data=listrecm)
class AsyncRequest: MIN_THREAD = 50 RETRY = 1 def __init__(self, request=None, retry=1, thread=50): self.request = request or Request() self.RETRY = retry self.MIN_THREAD = thread def __create_queue(self, urls): print("*********************** Start Queue %d" % len(urls)) self.length = len(urls) self.q = Queue(maxsize=self.length) self.num_theads = min(self.MIN_THREAD, self.length) self.dialog = xbmcgui.DialogProgress() self.dialog.create('Get URL', "Loading 0/%d urls" % self.length) self.results = [{} for x in urls] for i in range(len(urls)): self.q.put((i, urls[i])) def __start_thread(self, args): for i in range(self.num_theads): worker = Thread(target=self.__request, args=args) worker.setDaemon(True) worker.start() self.q.join() print("*********************** All %s thread done" % self.length) self.dialog.close() def __request(self, action, params=None, headers=None, json=None, redirect=False, parser=None, args=None): while not self.q.empty(): work = self.q.get() url = work[1] if type(url) is dict: params = 'params' in url and url['params'] or params headers = 'headers' in url and url['headers'] or headers redirect = 'redirect' in url and url['redirect'] or redirect parser = 'parser' in url and url['parser'] or parser args = 'args' in url and url['args'] or args json = 'json' in url and url['json'] or json required_response_header = 'responseHeader' in url and True or False url = work[1]['url'] retry = self.RETRY while retry > 0: try: if action is 'head': data = self.request.head(url, params=params, headers=headers, redirect=redirect) if action is 'get': data = self.request.get(url, params=params, headers=headers) if action is 'post': data = self.request.post(url, params=params, headers=headers, json=json) if parser: if required_response_header: response_headers = self.request.get_request( ).headers data = parser(data, args, response_headers) else: data = parser(data, args) # print('Requested %s' % work[1]) self.results[work[0]] = data retry = 0 except: print('Request %s fail retry %d' % (work[1], retry)) self.results[work[0]] = {} finally: retry -= 1 done = self.q.qsize() progress = 100 - (done * 100 / self.length) self.dialog.update( progress, 'Processing %d/%d urls' % (self.length - done, self.length)) self.q.task_done() return True def head(self, urls, params=None, headers=None, redirect=False, parser=None, args=None): self.__create_queue(urls) self.__start_thread(('head', params, headers, redirect, parser, args)) return self.results def get(self, urls, headers=None, params=None, redirect=False, parser=None, args=None): self.__create_queue(urls) self.__start_thread(('get', params, headers, redirect, parser, args)) return self.results def post(self, urls, params=None, headers=None, json=None, redirect=False, parser=None, args=None): self.__create_queue(urls) self.__start_thread( ('post', params, headers, json, redirect, parser, args)) return self.results
class Pool: """Pool of threads consuming tasks from a queue""" def __init__(self, thread_count, batch_mode=True, exception_handler=default_handler): # batch mode means block when adding tasks if no threads available to process self.queue = Queue(thread_count if batch_mode else 0) self.resultQueue = Queue(0) self.thread_count = thread_count self.exception_handler = exception_handler self.aborts = [] self.idles = [] self.threads = [] def __del__(self): """Tell my threads to quit""" self.abort() def run(self, block=False): """Start the threads, or restart them if you've aborted""" # either wait for them to finish or return false if some arent if block: while self.alive(): sleep(1) elif self.alive(): return False # go start them self.aborts = [] self.idles = [] self.threads = [] for n in range(self.thread_count): abort = Event() idle = Event() self.aborts.append(abort) self.idles.append(idle) self.threads.append( Worker('thread-%d' % n, self.queue, self.resultQueue, abort, idle, self.exception_handler)) return True def enqueue(self, func, *args, **kargs): """Add a task to the queue""" self.queue.put((func, args, kargs)) def join(self): """Wait for completion of all the tasks in the queue""" self.queue.join() def abort(self, block=False): """Tell each worker that its done working""" # tell the threads to stop after they are done with what they are currently doing for a in self.aborts: a.set() # wait for them to finish if requested while block and self.alive(): sleep(1) def alive(self): """Returns True if any threads are currently running""" return True in [t.is_alive() for t in self.threads] def idle(self): """Returns True if all threads are waiting for work""" return False not in [i.is_set() for i in self.idles] def done(self): """Returns True if not tasks are left to be completed""" return self.queue.empty() def results(self, sleep_time=0): """Get the set of results that have been processed, repeatedly call until done""" sleep(sleep_time) results = [] try: while True: # get a result, raises empty exception immediately if none available results.append(self.resultQueue.get(False)) self.resultQueue.task_done() except: return results return results
class ThreadPool(object): """ Pool of threads consuming tasks from a queue """ def __init__(self, num_threads, worker_class=None, max_queue_size=0, **kwargs): # task queue self.tasks = Queue(max_queue_size) # flags self._operating = True # params self.trace_logs = kwargs.pop('trace_logs', False) self.name = kwargs.pop('name', id(self)) # counters self._total_task_count = 0 self._tasks_ok_count = 0 self._tasks_nok_count = 0 self.worker_counters = {} # workers self._workers = {} self._worker_class = worker_class or Worker for i in range(num_threads): self._add_worker(i) def _add_worker(self, worker_id, raise_on_id_clash=False): """add a worker to the pool""" if worker_id in self._workers: if raise_on_id_clash: raise Exception('worker with that id already exists', worker_id) worker = self._worker_class(worker_id=worker_id, task_queue=self.tasks, parent_pool=self) self._workers[worker_id] = worker self.worker_counters[worker_id] = 0 @property def identification(self): return '{}({})'.format(self.__class__.__name__, self.name) def worker_notification(self, worker_id, log_func, message, **kwargs): msg = '{} {}({}) {}: {}'.format( self.identification, self._worker_class.__name__, worker_id, message, ' '.join( sorted(utils.convert_dict_params_to_list_of_string(kwargs)))) log_func(msg) @property def operating(self): return self._operating @property def count_completed(self): """gets the current count of all completed tasks""" return sum(self.worker_counters.values()) @property def count_remaining(self): """gets the current count of all remaining tasks""" return self.count_total - self.count_completed @property def count_total(self): """gets the total count of all tasks""" return self._total_task_count @property def count_ok(self): """gets the total count of all tasks completed okay""" return self._tasks_ok_count @property def count_nok(self): """gets the total count of all tasks completed not okay""" return self._tasks_nok_count @property def all_workers_started(self): """gets if all workers have started working""" return all(self.worker_counters.values()) @property def any_workers_started(self): """gets if any workers have started working""" return any(self.worker_counters.values()) def _increment_worker_count(self, worker_id): """count the number of tasks each worker completes""" self.worker_counters[worker_id] += 1 def task_ok(self, worker_id, func, args, kwargs): """count the number of tasks completed okay""" self._increment_worker_count(worker_id) self._tasks_ok_count += 1 def task_nok(self, worker_id, func, args, kwargs): """count the number of tasks completed not okay""" self._increment_worker_count(worker_id) self._tasks_nok_count += 1 def add_task(self, func, *args, **kwargs): """ Add a task to the queue """ self.tasks.put((func, args, kwargs)) self._total_task_count += 1 def map(self, func, args_list): """ Add a list of tasks to the queue """ # Add the jobs in bulk to the thread pool. Alternatively you could use # `add_task` to add single jobs. The code will block here, which # makes it possible to cancel the thread pool with an exception when # the currently running batch of workers is finished. for args in args_list: self.add_task(func, args) def stop(self): """stops the operation of this thread pool""" self._operating = False @property def finished(self): """are all tasks complete (based on count)""" return bool(self.count_remaining == 0) @property def processing(self): """are tasks still being processed""" return not self.finished def wait_completion(self): """ Wait for completion of all the tasks in the queue BLOCKING """ self.tasks.join() def notify_progress(self): """sends the current progress status to the log""" log.info('{} progress: ran={}/{} ok={} nok={}'.format( self.identification, self.count_completed, self.count_total, self.count_ok, self.count_nok)) def wait_with_progress(self, period=30, timeout=None): log.info('{} waiting with progress: threads={} tasks={}'.format( self.identification, len(self._workers), self.count_total)) while self.processing: self.notify_progress() time.sleep(period) log.info('{} finished: ran={}/{} ok={} nok={}'.format( self.identification, self.count_completed, self.count_total, self.count_ok, self.count_nok))
def render_tiles(bbox, mapfile, tile_dir, minZoom=1, maxZoom=18, name="unknown", num_threads=NUM_THREADS, tms_scheme=False): print("{} Generating tiles for {} bbox zoom levels [ {} - {} ] using {} mapnik mapfile".format( plus, bbox, minZoom, maxZoom, mapfile)) # Launch rendering threads queue = Queue(32) printLock = threading.Lock() renderers = {} for i in range(num_threads): renderer = RenderThread(tile_dir, mapfile, queue, printLock, maxZoom) render_thread = threading.Thread(target=renderer.loop) render_thread.start() print("{} Started render thread {}".format(plus, render_thread.getName())) renderers[i] = render_thread if not os.path.isdir(tile_dir): os.mkdir(tile_dir) gprj = GoogleProjection(maxZoom + 1) ll0 = (bbox[0], bbox[3]) ll1 = (bbox[2], bbox[1]) for z in range(minZoom, maxZoom + 1): px0 = gprj.fromLLtoPixel(ll0, z) px1 = gprj.fromLLtoPixel(ll1, z) # check if we have directories in place zoom = "%s" % z if not os.path.isdir(tile_dir + zoom): os.mkdir(tile_dir + zoom) for x in range(int(px0[0] / 256.0), int(px1[0] / 256.0) + 1): # Validate x co-ordinate if (x < 0) or (x >= 2 ** z): continue # check if we have directories in place str_x = "%s" % x if not os.path.isdir(tile_dir + zoom + '/' + str_x): os.mkdir(tile_dir + zoom + '/' + str_x) for y in range(int(px0[1] / 256.0), int(px1[1] / 256.0) + 1): # Validate x co-ordinate if (y < 0) or (y >= 2 ** z): continue # flip y to match OSGEO TMS spec if tms_scheme: str_y = "%s" % ((2 ** z - 1) - y) else: str_y = "%s" % y tile_uri = tile_dir + zoom + '/' + str_x + '/' + str_y + '.png' # Submit tile to be rendered into the queue t = (name, tile_uri, x, y, z) try: queue.put(t) except KeyboardInterrupt: raise SystemExit("Ctrl-c detected, exiting...") # Signal render threads to exit by sending empty request to queue for i in range(num_threads): queue.put(None) # wait for pending rendering jobs to complete queue.join() for i in range(num_threads): renderers[i].join()
class Worker(object): def __init__(self, work_dir, file_filter, input_service, output_service, threads_num=5, max_size=30): self._input_service = input_service self._output_service = output_service self._filter = file_filter self._work_dir = work_dir self._threads_num = threads_num self._threads_pool = [] self._queue = Queue(maxsize=max_size) self._stop = False self._succ = 0 self._fail = 0 self._lock = Lock() def __work_thread(self): while not self._stop: # logger.info("worker stop: " + str(self._stop)) try: # logger.debug("try to get task") task = self._queue.get_nowait() # logger.debug("get task succeefully") self._queue.task_done() except Empty: logger.debug("Empty queue" + str(self._stop)) if self._stop: break else: import time time.sleep(1) continue task_path = task.key if task_path.startswith('/'): task_path = task_path[1:] if isinstance(task_path, str): task_path = task_path.decode('utf-8') import uuid localpath = unicode(path.join(self._work_dir, uuid.uuid1().hex)) if task.size is None: logger.warn( "{file_path} task doesn't contains size attribute".format( file_path=task_path.encode('utf-8'))) self._fail += 1 fail_logger.error(task_path) continue try: try: makedirs(path.dirname(localpath)) except OSError as e: # directory is exists logger.debug(str(e)) try: ret = self._input_service.exists(task) if ret: logger.info("{file_path} exists".format( file_path=task_path.encode('utf-8'))) with self._lock: self._succ += 1 self._filter.add(task_path) continue except Exception as e: logger.exception("exists failed") try: self._output_service.download(task, localpath) except Exception as e: logger.exception("download failed") self._fail += 1 fail_logger.error(task_path) continue try: self._input_service.upload(task, localpath) except Exception: logger.exception("upload {} failed".format( task_path.encode('utf-8'))) self._fail += 1 fail_logger.error(task_path) continue try: import os if isinstance(localpath, unicode): localpath = localpath.encode('utf-8') os.remove(localpath) try: os.removedirs(path.dirname(localpath)) except OSError: pass except Exception as e: logger.exception(str(e)) continue try: ret = self._input_service.exists(task) if ret: if isinstance(task_path, unicode): logger.info("inc succ with {}".format( task_path.encode('utf-8'))) else: logger.info("inc succ with {}".format( task_path.encode('utf-8'))) with self._lock: self._succ += 1 self._filter.add(task_path) continue else: logger.exception( "there is not expected object in service, you need to re-run this application again" ) except Exception as e: logger.exception("exists failed") except Exception: logger.exception("try except for deleting file") finally: self._filter.del_doing_task(task_path) import os if isinstance(localpath, unicode): localpath = localpath.encode('utf-8') try: os.remove(localpath) os.removedirs(path.dirname(localpath)) except OSError: pass def add_task(self, task): # blocking self._queue.put(task) def start(self): self._threads_pool = [ Thread(target=self.__work_thread) for _ in range(self._threads_num) ] for t in self._threads_pool: t.start() def stop(self): self._queue.join() self.term() def term(self): self._stop = True logger.info("try to stop migrate process.") # while any([t.is_alive() for t in self._threads_pool]): # map(lambda i: i.join(5), filter(lambda j: j.is_alive(), self._threads_pool)) # print filter(lambda j: j.is_alive(), self._threads_pool) map(lambda i: i.join(), self._threads_pool) @property def success_num(self): return self._succ @property def failure_num(self): return self._fail