def semanticParse(source, api_url):
"""
Parse the source text quasi-semantically and query MediaWiki API (PHP)
to check the existence of a certain article (including redirects)
Use multiple threads to speed up API querying
Return candidate titles which exist on MediaWiki sites
"""
length = len(source) # Number of characters
candidates = [] # Candidate titles
r = []
q = Queue() # Queue for multithreading
for i in range(length):
start_index = i
end_index = length
t = Thread(target=_wordParse, args=(start_index, end_index, api_url, source, q))
t.start()
q.join()
try:
while True:
candidates.append(q.get(True,2)) # Get all items in queue with timeout (1 second)
except:
pass
#print candidates
return candidates
def main():
"""
Main function of the proxy scanner.
"""
global pl, output, q
parser = ArgumentParser(description='Scans a list of proxies to determine which work for HTTPS.')
parser.add_argument('--output', default='output/proxies.txt', type=str,
help='The file in which to store the found proxies.')
parser.add_argument('--threads', default=10, type=int,
help='Number of threads to use.')
args = parser.parse_args()
output = args.output
threads = args.threads
q = Queue(threads * 3)
print 'Starting threads.'
for x in xrange(threads):
t = Thread(target=check_proxies)
t.daemon = True
t.start()
print 'Queueing proxies.'
for proxy in proxies.proxies:
q.put(proxy)
q.join()
save_proxies()
class Worker:
def __init__(self):
self.q = Queue()
self.t = Thread(target=self._handle)
self.t.setDaemon(True)
self.t.start()
def _handle(self):
while True:
reset_caches()
fn = self.q.get()
try:
fn()
self.q.task_done()
except:
import traceback
print traceback.format_exc()
def do(self, fn, *a, **kw):
fn1 = lambda: fn(*a, **kw)
self.q.put(fn1)
def join(self):
self.q.join()
class ThreadPool(object):
def __init__(self, num):
"""num: 线程池中的线程数."""
self.tasks = Queue() #存储任务的队列
self.pool = []
for i in range(num):
self.pool.append(ThreadPoolSlot(self.tasks)) #创建了线程池
def spawn(self, func, *args):
"""
向队列中添加任务
"""
self.tasks.put((func, args))
def joinall(self):
"""等待队列中所有的任务完成."""
self.tasks.join()
"""等待所有线程结束."""
for thread in self.pool:
thread.join()
def undone_tasks(self):
"""
注意：非线程安全
"""
r = 0
for thread in self.pool:
if not thread.idle:
r += 1
if r:
r += self.tasks.qsize() #未完成的任务数等于（非空闲的线程数+任务队列中的任务数）
return r
def main():
print "[Facebook Album Downloader v1]"
start = timeit.default_timer()
# hide images
prefs = {"profile.managed_default_content_settings.images": 2}
extensions = webdriver.ChromeOptions()
extensions.add_experimental_option("prefs", prefs)
browser = webdriver.Chrome(executable_path="chromedriver", chrome_options=extensions)
findAlbum(browser)
createAlbumPath()
queue = Queue()
for x in range(max_workers):
worker = DownloadWorker(queue)
worker.daemon = True
worker.start()
print "[Getting Image Links]"
linkImages = getImageLinks(browser)
print "[Found: " + str(len(linkImages)) + "]"
for fullRes in linkImages:
queue.put(fullRes)
print "[Downloading...]"
queue.join()
browser.quit()
stop = timeit.default_timer()
print "[Time taken: %ss]" % str(stop - start)
raw_input("Press any key to continue...")
class CommandQueue(object):
"""Asynchronous command queue that can be used to communicate with blip.pl
in the background."""
def __init__(self):
self.queue = Queue()
self.worker = threading.Thread(target=self.__worker)
self.worker.setDaemon(True)
def __del__(self):
self.Finish()
def __worker(self):
while True:
item = self.queue.get(True)
item()
self.queue.task_done()
def Finish(self):
"""Finishes all commands in the queue"""
self.queue.join()
def HasPendingCommands(self):
"""Returns True if the queue is busy"""
return self.queue.qsize() > 0
def Enqueue(self, command):
"""Enqueues a command in the queue.
Command must refer to a function without parameters."""
self.queue.put(command)
def main():
ts = time()
start = input("Please input the start value: ")
stop = input("Please input the stop value: ")
# set the number of threads
threads = input("Please input the download numbers every piece: ")
k = stop-start
# acquire the download links
links = [l for l in get_link(start,stop)]
# set the download storage directory
down_dir = setup_dir()
queue = Queue()
# judge download numbers if greater than threads or not
# if K< = threads ,set the k for threads,else set the threads for the number of thread
if k <= threads:
for x in range(k-1):
print queue.qsize()
worker = DownloadWorker(queue)
worker.setDaemon(True)
worker.start()
else:
for x in range(threads):
worker = DownloadWorker(queue)
worker.setDaemon(True)
worker.start()
# traverse the links and put the link to queue
for link in links:
queue.put((down_dir,link))
# the new queue joining
queue.join()
print 'Took {}'.format(time()-ts)
print "The finished time:" + ctime()
class ThreadPool(object):
"""
Pool of threads consuming tasks from a queue
"""
def __init__(self, num_threads):
if num_threads < 1:
raise ValueError("ThreadPool must have 1 thread or greenlet at least")
elif num_threads == 1:
self.__class__ = SingleThreadedPool
return
self.tasks = Queue(num_threads)
for x in range(num_threads):
WorkerThread(self.tasks)
def spawn_n(self, func, *args, **kargs):
"""
Add a task to the queue and asign a thread to do the work
"""
self.tasks.put((func, args, kargs))
def waitall(self):
"""
Wait for completion of all the tasks in the queue
"""
self.tasks.join()
class Content:
def __init__(self):
self.outputs = Queue()
d = time.strftime('%y-%m-%d %H-%M',time.localtime())
self.f = open( 'content.txt','a')
self.f.write('[')
t = Thread(target=self.run)
t.setDaemon(True)
t.start()
def __del__(self):
self.outputs.join()
self.f.write(']')
self.f.close()
def write(self,url,dom,targets):
result = {}
result['url'] = url
if dom != None:
for t in targets:
m = dom.xpath(t['xpath'])
if len(m) >= 1:
print m[0].text
result[t['name']] = m[0].text
output = json.dumps(result) + ',\n'
self.outputs.put(output)
def run(self):
while True:
output = self.outputs.get()
self.f.write(output)
self.f.close()
def main(numthreads=10):
t1 = time.time()
queue = Queue()
factory = TokenFactory()
config = ConfigParser()
config.read('vk_api.conf')
url = API.get_url(
app_id=config.get('api', 'id'), app_key=config.get('api', 'key'),
permissions=PERMISSIONS, redirect_uri=URI, display=DISPLAY, api_version=VERSION)
# TODO: check token expiration
token_pair = factory.get_token_pair()
if not token_pair:
token_pair = factory.store_token_pair(url)
api = API(token=token_pair[0],user_id=token_pair[1])
audio = api.audio
data = audio.get
if data:
for item in data['response']['items']:
queue.put(item)
for i in range(numthreads):
t = DownloadThread(queue, FILE_DIR)
t.start()
queue.join()
t2 = time.time()
print('Time: {0}'.format(t2-t1))
class MaximaTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
def __init__(self, server_address, RequestHandlerClass, bind_and_activate=True):
SocketServer.TCPServer.__init__(self, server_address, RequestHandlerClass, bind_and_activate)
self.daemon_threads = True
# ensure that we can handle more than the usual number of pending requests
self.request_queue_size = 30
# initialize the request queue
self.ready = False
self.request_queue = Queue()
def calculate(self, calc):
if not self.ready:
return None
# terminate the request if it hasn't been terminated already
# otherwise maxima will wait forever for extra input
if not calc.endswith(";"):
calc += ";"
# form a calcrequest and put it on the queue for processing
calc_req = CalculateRequest(calc)
self.request_queue.put(calc_req)
# wait until its done
self.request_queue.join()
# use the result that was populated by MaximaHandler when
# it pulled our request off the queue and sent it to maxima,
# then stored the response in result
return calc_req.result
class ThreadPool:
"""A basic pool of worker threads"""
class Worker(Thread):
def __init__(self, tasks):
Thread.__init__(self)
self.tasks = tasks
self.daemon = True # threads will be killed on exit
self.start()
def run(self):
while True:
# block until a task is ready to be done
function, args, kargs = self.tasks.get()
try:
function(*args, **kargs)
except:
print(sys.exc_info()[0])
self.tasks.errors = True
self.tasks.task_done()
def __init__(self, num_threads = multiprocessing.cpu_count() + 1):
self.tasks = Queue(num_threads)
self.tasks.errors = False
# create num_threads Workers, by default the number of CPUs + 1
for _ in range(num_threads): self.Worker(self.tasks)
def add_task(self, function, *args, **kargs):
self.tasks.put((function, args, kargs))
def wait_completion(self):
# wait for the queue to be empty
self.tasks.join()
return self.tasks.errors
def parallelize(tasks, ntasks):
tasks_to_run = Queue()
for task in tasks:
tasks_to_run.put(tuple(task))
results = Queue()
workers = []
for i in range(ntasks):
worker = threading.Thread(target = run_task_from_queue,
args = (tasks_to_run, results))
worker.daemon = True
worker.start()
workers.append(worker)
## Wait for all to finish
try:
tasks_to_run.join()
except KeyboardInterrupt:
log.error("Caught Ctrl-C, quitting")
outputs = []
while not results.empty():
outputs.append(results.get())
print "Finished parallelize() call"
## Kill workers
return outputs
class ThreadPool(object):
def __init__(self, size=1):
self.size = size
self.jobs = Queue()
self.results = Queue()
self.threads = []
def start(self):
"""start all threads"""
for i in range(self.size):
self.threads.append(ThreadWorker(self))
for thread in self.threads:
thread.start()
def append_job(self, job, *args, **kwargs):
self.jobs.put((job, args, kwargs))
def join(self):
"""waiting all jobs done"""
self.jobs.join()
def stop(self):
"""kill all threads"""
for thread in self.threads: # stop all threads
thread.stop()
for thread in self.threads: # waiting completing
if thread.isAlive():
thread.join()
del self.threads[:]
def run_remote_commands(self, ips=None, command=None, ):
command = command or self.command
ips = ips or self.ips
if not ips:
self.logger.warning('No IPs provided to run_remote_commands!')
return self.results
command = command or ""
iq = Queue()
#if not ips:
# raise ValueError('run_remote_commands: IP list was empty:"{0}"'.format(ips))
for ip in ips:
ip = str(ip).strip().rstrip()
iq.put(ip)
tlock = Lock()
threadcount = self.args.thread_count
if threadcount > iq.qsize():
threadcount = iq.qsize()
if not iq:
return
self.results = {}
for i in range(threadcount):
t = Thread(target=self.do_ssh, args=(iq, tlock, i, command))
t.daemon = True
t.start()
self.logger.debug('Threads started now waiting for join')
iq.join()
self.logger.debug('Done with join')
time.sleep(self.maxwait + .1)
return self.results
class Crawler(object):
'''
A multithread crawler downloads large number of images from adult website
and normal website.
'''
def __init__(self , url):
self.url = url
self.numThread = 100
self.queIn = Queue(self.numThread)
self.queOut = Queue(self.numThread)
self.queOut.put(url)
self.seen = []
self.dome = urlparse(url)[1]
def walk(self):
for i in range(self.numThread):
t = PopQueThread(self.queIn , self.queOut , self.seen)
t.setDaemon(True)
t.start()
for i in range(self.numThread):
dt = GetPageThread(self.queIn , self.queOut , self.seen , self.dome)
dt.setDaemon(True)
dt.start()
self.queIn.join()
self.queOut.join()
class Runner(object):
def __init__(self, controller):
self.logger = logging.getLogger("Runner")
self.logger.debug("initializing")
self.controller = controller
self.jobmanager = self.controller.getJobManager()
self.nodemanager = self.controller.getNodeManager()
def runAllJobs(self):
self.logger.info("starting workers")
joblist = self.jobmanager.getJobList()
nodes = self.nodemanager.getNodes()
# Create a queue of jobs
self.queue = Queue()
for job in joblist:
self.queue.put(job)
# Create workers for each node and start them
self.workers = []
for node in nodes.values():
for i in range(node.info.getWorkload()):
worker = Worker(i, node, self.queue, self.controller)
self.workers.append(worker)
worker.start()
# Wait for all jobs to be processed
self.logger.info("waiting for completion")
self.queue.join()
self.logger.info("all jobs completed")
class CertDBCertificateReport(reporter.CertificateReport):
def __init__(self, cert_db, log_key, checks=all_checks.ALL_CHECKS):
self._cert_db = cert_db
self.log_key = log_key
self._certs_queue = Queue(FLAGS.cert_db_writer_queue_size)
self._writer = None
super(CertDBCertificateReport, self).__init__(checks=checks)
def report(self):
super(CertDBCertificateReport, self).report()
self._certs_queue.join()
logging.info("Finished scheduled writing to CertDB")
self._certs_queue.put(None)
self.reset()
def reset(self):
if self._writer:
self._writer.join()
self._writer = None
def _batch_scanned_callback(self, result):
if not self._writer:
self._writer = threading.Thread(target=_process_certs,
args=(self._cert_db, self.log_key,
self._certs_queue))
self._writer.start()
self._certs_queue.put([(desc, index) for desc, index, _ in result])
def run(self):
args = list(islice(self.reqs, self.requests))
if self.shuffle:
random.shuffle(args)
print "Total requests: %d" % len(args)
print "Concurrency : %d" % self.concurrency
starttime = time.time()
q, p = Queue(), Queue()
for _ in xrange(self.concurrency):
t = Thread(target=worker, args=(self.host, q, p, self.verbose))
t.daemon = True
t.start()
for a in args:
q.put(a)
q.join()
outputs = []
for _ in xrange(self.requests):
outputs.append(p.get())
elapsed = time.time() - starttime
print
print "Total requests: %d" % len(args)
print "Concurrency : %d" % self.concurrency
print "Elapsed time : %.3fs" % elapsed
print "Avg time p/req: %.3fs" % (elapsed/len(args))
print "Received (per status code or error):"
for c, n in Counter(outputs).items():
print " %s: %d" % (c, n)
class ThreadPool(object):
"""Pool of threads consuming tasks from a queue"""
def __init__(self, workers):
self.tasks = Queue()
self.workers = [Worker(self.tasks) for x in xrange(workers)]
self.state = ThreadPoolState.IDLE
def apply_async(self, func, args, **kargs):
"""Add a task to the queue"""
if self.state != ThreadPoolState.IDLE:
raise ThreadPoolError('ThreadPool cant accept any more tasks')
self.tasks.put((func, args, kargs))
def close(self):
self.state = ThreadPoolState.CLOSED
while not self.tasks.empty():
self.tasks.get_nowait()
self.tasks.task_done()
for worker in self.workers:
self.tasks.put((None, (), {}))
def join(self):
"""Wait for completion of all the tasks in the queue"""
self.state = ThreadPoolState.WAIT_JOIN
self.tasks.join()
def main(repo_list_files):
"""
main runner for key commit
"""
worker_q = Queue(maxsize=0)
results_q = Queue(maxsize=0)
num_threads = 10
stop = 5
for repo_list in repo_list_files:
with open(repo_list, "rb") as f:
for l, line in enumerate(f.xreadlines()):
username = line.split(',')[0].split('/')[0].rstrip()
repo = line.split(',')[0].split('/')[1].rstrip()
worker_q.put((username, repo, line))
print l, '\t', username, '\t', repo
if l > stop:
break
writer_thread = Thread(target=writer, args=(results_q, ))
writer_thread.setDaemon(True)
writer_thread.start()
for i in range(num_threads):
t = Thread(target=worker, args=(results_q, worker_q))
t.setDaemon(True)
t.start()
worker_q.join()
results_q.join()
def get_sshkey_deploy(hosts, user, password, key):
config = {}
config["username"] = user
config["password"] = password
config["key"] = key
config["authorized_keys"] = "authorized_keys"
config["ssh_dir"] = "~/.ssh"
config["port"] = 22
config["timeout_seconds"] = 3
config["threads"] = 100
config["append"] = True
global results
results = []
global queue
queue = Queue(maxsize=10 * config["threads"])
deployer_threads = []
for i in range(config["threads"]):
deployer_thread = DeployKeyThread(config)
deployer_thread.daemon = True
deployer_thread.start()
deployer_threads.append(deployer_thread)
for host in hosts:
queue.put(host)
queue.join()
return results
class WorkerThread(threading.Thread):
def __init__(self, *args, **kwargs):
threading.Thread.__init__(self, *args, **kwargs)
self.input_queue = Queue()
def send(self, item):
self.input_queue.put(item)
def close(self):
self.input_queue.put(None)
self.input_queue.join()
def run(self):
while True:
item = self.input_queue.get()
if item is None:
break
# Process the item: replace with useful work
print item
self.input_queue.task_done()
# Done. Indicate that sentinel was received and return
self.input_queue.task_done()
return
class ThreadPool(object):
""" 线程池的具体实现
"""
def __init__(self, thread_count=10):
""" thread_count: 线程池大小（创建的线程数）
"""
try:
self.thread_count = int(thread_count)
except:
raise SystemExit('thread_count not is a number')
self.work_list = []
self.init_threadpool()
def init_threadpool(self):
""" 初始化线程池，并启动线程
"""
self.work_queue = Queue()
for _ in range(self.thread_count):
t = ThreadWork(self.work_queue)
self.work_list.append(t)
t.setDaemon(True)
t.start()
def putjob(self, work_func, func_args=None):
""" 把任务放入线程池调动
"""
self.work_queue.put((work_func, func_args))
def wait(self):
self.work_queue.join()
def threadedExec( servers, maxThreads ) :
'''
This implements the email account access part of the program using a threaded queue model
maxThreads in an INTEGER that denotes the maximum number of parallel threads that the program is allowed to open. This is a global setting.
'''
from Queue import Queue
inQueue = Queue() # Initiate and populate input queue with list of tasks (data for each task)
for account in servers :
inQueue.put( servers[ account ] ) # Store the account dictionary in inQueue. This denotes the data required for a single task: polling a single server
outQueue = Queue( maxsize = inQueue.qsize() ) # Prepare output queue for storing results
# Create a number of threads to parallelize the tasks. Threads created using the Worker class inherited from the threading.Thread class:
workers = [ Worker( pollAccount, inQueue, outQueue ) for ii in range( maxThreads ) ] # maxThreads is a global variable that determines the maximum number of open threads
for worker in workers :
worker.start() # Begun execution of the thread
inQueue.join() # Pause program stepping forward here until all tasks in inQueue are completed
while not outQueue.empty() :
yield outQueue.get() # this wording makes test3() a generator and it should be used as such. test3 will return Buffer type objects from each server poll one at a time.
def get_channels(channels, **kwargs):
"""Multi-threaded channel query
"""
if len(channels) == 0:
return []
# set up Queues
inqueue = Queue()
outqueue = Queue()
# open threads
for i in range(len(channels)):
t = ThreadChannelQuery(inqueue, outqueue, **kwargs)
t.setDaemon(True)
t.start()
# populate input queue
for i, c in enumerate(channels):
inqueue.put((i, c))
# block
inqueue.join()
outqueue.join()
result = []
for i in range(len(channels)):
c = outqueue.get()
if isinstance(c, Exception):
raise c
else:
result.append(c)
return zip(*sorted(result, key=lambda (idx, chan): idx))[1]
class DownloadThreadPool(object):
def __init__(self, size=3):
self.queue = Queue()
self.workers = [Thread(target=self._do_work) for _ in range(size)]
self.initialized = False
def init_threads(self):
for worker in self.workers:
worker.setDaemon(True)
worker.start()
self.initialized = True
def _do_work(self):
while True:
url, target = self.queue.get()
download(url, target, close_target=True, quiet=False)
self.queue.task_done()
def join(self):
self.queue.join()
def submit(self, url, target):
if not self.initialized:
self.init_threads()
self.queue.put((url, target))
class ThreadPool():
def __init__(self):
self.q = Queue()
self.NUM = 3
self.JOBS = 10
def do_somthing_using(self,arguments):
print arguments
def working(self):
while True:
arguments = self.q.get()
self.do_somthing_using(arguments+1)
sleep(1)
self.q.task_done()
def PoolStart(self):
for i in range(self.NUM):
t = Thread(target=self.working)
t.setDaemon(True)
t.start()
for i in range(self.JOBS):
self.q.put(i)
self.q.join()
class _BatchWriter(object):
def __init__(self, url, start, auth, size, interval, qsize,
content_encoding):
self.url = url
self.offset = start
self._nextid = count(start)
self.auth = auth
self.size = size
self.interval = interval
self.content_encoding = content_encoding
self.checkpoint = time.time()
self.itemsq = Queue(size * 2 if qsize is None else qsize)
self.closed = False
self.flushme = False
def write(self, item):
assert not self.closed, 'attempting writes to a closed writer'
self.itemsq.put(jsonencode(item))
return self._nextid.next()
def flush(self):
self.flushme = True
self.itemsq.join()
self.flushme = False
def close(self, block=True):
self.closed = True
if block:
self.itemsq.join()
def __str__(self):
return self.url
def mysql_main(ipdict,threads):
printPink("crack mysql now...")
print "[*] start crack mysql %s" % time.ctime()
starttime=time.time()
global sp
sp=Queue()
global lock
lock = threading.Lock()
global result
result=[]
for i in xrange(threads):
t = Thread(target=mysql)
t.setDaemon(True)
t.start()
for ip in ipdict['mysql']:
sp.put((str(ip).split(':')[0],int(str(ip).split(':')[1])))
sp.join()
print "[*] stop crack mysql %s" % time.ctime()
print "[*] crack mysql done,it has Elapsed time:%s " % (time.time()-starttime)
return result
def worker(): global queue while True: try: item = queue.get() print 'Proccessing ' + item['filename'] if item['type'] == 'pcap': parse_pcap.parse_pcap(get_data_path()+item['filename']) elif item['type'] == 'log': parse_log.parse_log(get_data_path()+item['filename']) except Exception, e: print e finally: queue.task_done() if __name__ == "__main__": queue = Queue() while True: if queue.empty(): time.sleep( config.check_interval ) check_file_change() #check every 3 sec else: for i in range(config.num_worker_threads): t = threading.Thread(target=worker) t.daemon = True t.start() queue.join() # block until all tasks are done
print('Localhost timeout')
return False
print '*** Testing SSH Creds ***'
if ssh_test() != True:
exit()
print '*** SSH Creds Success! ***'
for thread_num in range(max_threads):
worker = Thread(target=ssh_session, args=(thread_num, working_queue))
worker.setDaemon(True)
worker.start()
with open(hosts_csv) as devices:
deviceDict = csv.DictReader(devices)
for row in deviceDict:
with print_lock:
print 'Putting {} in queue'.format(row['hostname'])
working_queue.put(row['hostname'])
with print_lock:
print '*** Waiting for all threads to complete ***'
working_queue.join()
print '*** Finished SSH Sessions ***'
if len(error_list) > 0:
print '*** Errors Encountered ***'
for i in error_list:
print '{}, {}'.format(i[0], i[1])
class Timelapse(object):
def __init__(
self, settings, octoprint_printer, data_folder, timelapse_folder,
on_print_started=None, on_print_start_failed=None,
on_snapshot_start=None, on_snapshot_end=None,
on_render_start=None, on_render_end=None,
on_timelapse_stopping=None, on_timelapse_stopped=None,
on_state_changed=None, on_timelapse_start=None, on_timelapse_end=None,
on_snapshot_position_error=None, on_position_error=None, on_plugin_message_sent=None):
# config variables - These don't change even after a reset
self.DataFolder = data_folder
self.Settings = settings # type: OctolapseSettings
self.OctoprintPrinter = octoprint_printer
self.DefaultTimelapseDirectory = timelapse_folder
self.OnPrintStartCallback = on_print_started
self.OnPrintStartFailedCallback = on_print_start_failed
self.OnRenderStartCallback = on_render_start
self.OnRenderEndCallback = on_render_end
self.OnSnapshotStartCallback = on_snapshot_start
self.OnSnapshotCompleteCallback = on_snapshot_end
self.TimelapseStoppingCallback = on_timelapse_stopping
self.TimelapseStoppedCallback = on_timelapse_stopped
self.OnStateChangedCallback = on_state_changed
self.OnTimelapseStartCallback = on_timelapse_start
self.OnTimelapseEndCallback = on_timelapse_end
self.OnSnapshotPositionErrorCallback = on_snapshot_position_error
self.OnPositionErrorCallback = on_position_error
self.OnPluginMessageSentCallback = on_plugin_message_sent
self.Commands = Commands() # used to parse and generate gcode
self.Triggers = None
self.PrintEndStatus = "Unknown"
self.LastStateChangeMessageTime = None
self.StateChangeMessageThread = None
# Settings that may be different after StartTimelapse is called
self.OctoprintPrinterProfile = None
self.PrintStartTime = None
self.FfMpegPath = None
self.Snapshot = None
self.Gcode = None
self.Printer = None
self.CaptureSnapshot = None
self.Position = None
self.Rendering = None
self.State = TimelapseState.Idle
self.IsTestMode = False
# State Tracking that should only be reset when starting a timelapse
self.SnapshotCount = 0
self.HasBeenStopped = False
self.TimelapseStopRequested = False
self.SavedCommand = None
self.SecondsAddedByOctolapse = 0
# State tracking variables
self.RequiresLocationDetectionAfterHome = False
# fetch position private variables
self._position_payload = None
self._position_timeout_long = 600.0
self._position_timeout_short = 10.0
self._position_signal = threading.Event()
self._position_signal.set()
# get snapshot async private variables
self._snapshot_success = False
# It shouldn't take more than 5 seconds to take a snapshot!
self._snapshot_timeout = 5.0
self._snapshot_signal = threading.Event()
self._snapshot_signal.set()
self._most_recent_snapshot_payload = None
self.CurrentProfiles = {}
self.CurrentFileLine = 0
# snapshot thread queue
self._snapshot_task_queue = Queue(maxsize=1)
self._rendering_task_queue = Queue(maxsize=1)
self._reset()
def start_timelapse(
self, settings, octoprint_printer_profile, ffmpeg_path, g90_influences_extruder):
# we must supply the settings first! Else reset won't work properly.
self._reset()
# in case the settings have been destroyed and recreated
self.Settings = settings
# time tracking - how much time did we add to the print?
self.SnapshotCount = 0
self.SecondsAddedByOctolapse = 0
self.RequiresLocationDetectionAfterHome = False
self.OctoprintPrinterProfile = octoprint_printer_profile
self.FfMpegPath = ffmpeg_path
self.PrintStartTime = time.time()
self.Snapshot = Snapshot(self.Settings.current_snapshot())
self.Gcode = SnapshotGcodeGenerator(
self.Settings, octoprint_printer_profile)
self.Printer = Printer(self.Settings.current_printer())
self.Rendering = Rendering(self.Settings.current_rendering())
self.CaptureSnapshot = CaptureSnapshot(
self.Settings, self.DataFolder, print_start_time=self.PrintStartTime)
self.Position = Position(
self.Settings, octoprint_printer_profile, g90_influences_extruder)
self.State = TimelapseState.WaitingForTrigger
self.IsTestMode = self.Settings.current_debug_profile().is_test_mode
self.Triggers = Triggers(self.Settings)
self.Triggers.create()
# take a snapshot of the current settings for use in the Octolapse Tab
self.CurrentProfiles = self.Settings.get_profiles_dict()
# send an initial state message
self._on_timelapse_start()
def on_position_received(self, payload):
if self.State != TimelapseState.Idle:
self._position_payload = payload
self._position_signal.set()
def send_snapshot_gcode_array(self, gcode_array):
self.OctoprintPrinter.commands(gcode_array, tags={"snapshot_gcode"})
def get_position_async(self, start_gcode=None, timeout=None):
if timeout is None:
timeout = self._position_timeout_long
self.Settings.current_debug_profile().log_print_state_change("Octolapse is requesting a position.")
# Warning, we can only request one position at a time!
if self._position_signal.is_set():
self._position_signal.clear()
# build the staret commands
commands_to_send = ["M400", "M114"]
# send any code that is to be run before the position request
if start_gcode is not None and len(start_gcode) > 0:
commands_to_send = start_gcode + commands_to_send
self.send_snapshot_gcode_array(commands_to_send)
event_is_set = self._position_signal.wait(timeout)
if not event_is_set:
# we ran into a timeout while waiting for a fresh position
# set the position signal
self._snapshot_signal.set()
self.Settings.current_debug_profile().log_warning(
"Warning: A timeout occurred while requesting the current position!."
)
return None
return self._position_payload
def _on_snapshot_success(self, *args, **kwargs):
# Increment the number of snapshots received
self.SnapshotCount += 1
self._snapshot_success = True
self._snapshot_signal.set()
def _on_snapshot_fail(self, *args, **kwargs):
reason = args[0]
message = "Failed to download the snapshot. Reason: {0}".format(
reason)
self.Settings.current_debug_profile().log_snapshot_download(message)
self._snapshot_success = False
self.SnapshotError = message
self._snapshot_signal.set()
def _on_snapshot_complete(self, *args, **kwargs):
self.Settings.current_debug_profile().log_snapshot_download("Snapshot download complete.")
def _take_snapshot_async(self):
snapshot_async_payload = {
"success": False,
"error": "Waiting on thread to signal, aborting"
}
if self._snapshot_signal.is_set():
# only clear signal and send a new M114 if we haven't already done that from another thread
self._snapshot_signal.clear()
# start the snapshot
self.Settings.current_debug_profile().log_snapshot_download("Taking a snapshot.")
snapshot_guid = str(uuid.uuid4())
snapshot_job = self.CaptureSnapshot.create_snapshot_job(
utility.get_currently_printing_filename(self.OctoprintPrinter),
self.SnapshotCount,
snapshot_guid,
self._snapshot_task_queue,
on_success=self._on_snapshot_success,
on_fail=self._on_snapshot_fail,
on_complete=self._on_snapshot_complete
)
self._snapshot_task_queue.put(snapshot_guid)
snapshot_thread = threading.Thread(target=snapshot_job)
snapshot_thread.daemon = True
snapshot_thread.start()
event_is_set = self._snapshot_signal.wait(self._snapshot_timeout)
if not event_is_set:
# we ran into a timeout while waiting for a fresh position
snapshot_async_payload["success"] = False
snapshot_async_payload["error"] = \
"Snapshot timed out in {0} seconds.".format(self._snapshot_timeout)
self._snapshot_signal.set()
else:
snapshot_async_payload["success"] = True
return snapshot_async_payload
def _take_timelapse_snapshot(
self, trigger, command_string, cmd, parameters, triggering_command_position, triggering_extruder_position
):
timelapse_snapshot_payload = {
"snapshot_position": None,
"return_position": None,
"snapshot_gcode": None,
"snapshot_payload": None,
"current_snapshot_time": 0,
"total_snapshot_time": 0,
"success": False,
"error": ""
}
try:
show_real_snapshot_time = self.Settings.show_real_snapshot_time
# create the GCode for the timelapse and store it
snapshot_gcode = self.Gcode.create_snapshot_gcode(
self.Position,
trigger,
command_string,
cmd,
parameters,
triggering_command_position,
triggering_extruder_position
)
# save the gcode fo the payload
timelapse_snapshot_payload["snapshot_gcode"] = snapshot_gcode
if snapshot_gcode is None:
self.Settings.current_debug_profile().log_warning(
"No snapshot gcode was generated."
)
return timelapse_snapshot_payload
assert (isinstance(snapshot_gcode, SnapshotGcode))
if not show_real_snapshot_time:
gcodes_to_send = snapshot_gcode.StartGcode + snapshot_gcode.SnapshotCommands
if len(gcodes_to_send) > 0:
self.Settings.current_debug_profile().log_snapshot_gcode(
"Sending snapshot start gcode and snapshot commands.")
snapshot_position = self.get_position_async(
start_gcode=snapshot_gcode.StartGcode + snapshot_gcode.SnapshotCommands
)
else:
self.Settings.current_debug_profile().log_snapshot_gcode(
"Sending snapshot start gcode.")
# send start commands and zhop/retract if they exist
if len(snapshot_gcode.StartGcode) > 0:
start_position = self.get_position_async(start_gcode=snapshot_gcode.StartGcode)
# Todo: Handle start_position = None
# park the printhead in the snapshot position and wait for the movement to complete
snapshot_start_time = time.time()
if len(snapshot_gcode.SnapshotCommands) > 0:
self.Settings.current_debug_profile().log_snapshot_gcode("Sending snapshot commands.")
snapshot_position = self.get_position_async(
start_gcode=snapshot_gcode.SnapshotCommands, timeout=self._position_timeout_short
)
# record the snapshot position
timelapse_snapshot_payload["snapshot_position"] = snapshot_position
# by now we should be ready to take a snapshot
snapshot_async_payload = self._take_snapshot_async()
timelapse_snapshot_payload["snapshot_payload"] = snapshot_async_payload
if not show_real_snapshot_time:
# return the printhead to the start position
gcode_to_send = snapshot_gcode.ReturnCommands + snapshot_gcode.EndGcode
if len (gcode_to_send) > 0:
self.Settings.current_debug_profile().log_snapshot_gcode("Sending snapshot return and end gcode.")
self.send_snapshot_gcode_array(gcode_to_send)
else:
if len(snapshot_gcode.ReturnCommands) > 0:
self.Settings.current_debug_profile().log_snapshot_gcode("Sending return gcode.")
return_position = self.get_position_async(
start_gcode=snapshot_gcode.ReturnCommands, timeout=self._position_timeout_short
)
timelapse_snapshot_payload["return_position"] = return_position
# calculate the total snapshot time
snapshot_end_time = time.time()
snapshot_time = snapshot_end_time - snapshot_start_time
self.SecondsAddedByOctolapse += snapshot_time
timelapse_snapshot_payload["current_snapshot_time"] = snapshot_time
timelapse_snapshot_payload["total_snapshot_time"] = self.SecondsAddedByOctolapse
if len(snapshot_gcode.EndGcode) > 0:
self.Settings.current_debug_profile().log_snapshot_gcode("Sending end gcode.")
self.send_snapshot_gcode_array(snapshot_gcode.EndGcode)
# we've completed the procedure, set success
timelapse_snapshot_payload["success"] = True
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
timelapse_snapshot_payload["error"] = "An unexpected error was encountered while running the timelapse " \
"snapshot procedure. "
return timelapse_snapshot_payload
# public functions
def to_state_dict(self):
try:
position_dict = None
position_state_dict = None
extruder_dict = None
trigger_state = None
if self.Settings is not None:
if self.Settings.show_position_changes and self.Position is not None:
position_dict = self.Position.to_position_dict()
if self.Settings.show_position_state_changes and self.Position is not None:
position_state_dict = self.Position.to_state_dict()
if self.Settings.show_extruder_state_changes and self.Position is not None:
extruder_dict = self.Position.Extruder.to_dict()
if self.Settings.show_trigger_state_changes and self.Triggers is not None:
trigger_state = {
"Name": self.Triggers.Name,
"Triggers": self.Triggers.state_to_list()
}
state_dict = {
"Extruder": extruder_dict,
"Position": position_dict,
"PositionState": position_state_dict,
"TriggerState": trigger_state
}
return state_dict
except Exception as e:
self.Settings.CurrentDebugProfile().log_exception(e)
# if we're here, we've reached and logged an error.
return {
"Extruder": None,
"Position": None,
"PositionState": None,
"TriggerState": None
}
def stop_snapshots(self, message=None, error=False):
self.State = TimelapseState.WaitingToRender
if self.TimelapseStoppedCallback is not None:
timelapse_stopped_callback_thread = threading.Thread(
target=self.TimelapseStoppedCallback, args=[message, error]
)
timelapse_stopped_callback_thread.daemon = True
timelapse_stopped_callback_thread.start()
return True
def on_print_failed(self):
if self.State != TimelapseState.Idle:
self.end_timelapse("FAILED")
def on_print_disconnecting(self):
if self.State != TimelapseState.Idle:
self.end_timelapse("DISCONNECTING")
def on_print_disconnected(self):
if self.State != TimelapseState.Idle:
self.end_timelapse("DISCONNECTED")
def on_print_canceled(self):
if self.State != TimelapseState.Idle:
self.end_timelapse("CANCELED")
def on_print_completed(self):
if self.State != TimelapseState.Idle:
self.end_timelapse("COMPLETED")
def end_timelapse(self, print_status):
self.PrintEndStatus = print_status
try:
if self.PrintStartTime is None:
self._reset()
elif self.PrintStartTime is not None and self.State in [
TimelapseState.WaitingForTrigger, TimelapseState.WaitingToRender, TimelapseState.WaitingToEndTimelapse
]:
if not self._render_timelapse(self.PrintEndStatus):
if self.OnRenderEndCallback is not None:
payload = RenderingCallbackArgs(
"Could not start timelapse job.",
-1,
"unknown",
"unknown",
"unknown",
"unknown",
"unknown",
"unknown",
False,
0,
0,
True,
"timelapse_start",
"The render_start function returned false"
)
render_end_callback_thread = threading.Thread(
target=self.OnRenderEndCallback, args=[payload]
)
render_end_callback_thread.daemon = True
render_end_callback_thread.start()
self._reset()
if self.State != TimelapseState.Idle:
self.State = TimelapseState.WaitingToEndTimelapse
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
if self.OnTimelapseEndCallback is not None:
self.OnTimelapseEndCallback()
def on_print_paused(self):
try:
if self.State == TimelapseState.Idle:
return
elif self.State < TimelapseState.WaitingToRender:
self.Settings.current_debug_profile().log_print_state_change("Print Paused.")
self.Triggers.pause()
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
def on_print_resumed(self):
try:
if self.State == TimelapseState.Idle:
return
elif self.State < TimelapseState.WaitingToRender:
self.Triggers.resume()
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
def is_timelapse_active(self):
if (
self.Settings is None
or self.State in [TimelapseState.Idle, TimelapseState.Initializing, TimelapseState.WaitingToRender]
or self.OctoprintPrinter.get_state_id() == "CANCELLING"
or self.Triggers is None
or self.Triggers.count() < 1
):
return False
return True
def get_is_rendering(self):
return self._rendering_task_queue.qsize() > 0
def on_print_start(self, tags):
self.OnPrintStartCallback(tags)
def on_print_start_failed(self, message):
self.OnPrintStartFailedCallback(message)
def on_gcode_queuing(self, command_string, cmd_type, gcode, tags):
self.detect_timelapse_start(command_string, tags)
# if the timelapse is not active, exit without changing any gcode
if not self.is_timelapse_active():
return
self.check_current_line_number(tags)
# update the position tracker so that we know where all of the axis are.
# We will need this later when generating snapshot gcode so that we can return to the previous
# position
is_snapshot_gcode_command = self._is_snapshot_command(command_string)
try:
self.Settings.current_debug_profile().log_gcode_queuing(
"Queuing Command: Command Type:{0}, gcode:{1}, cmd: {2}, tags: {3}".format(
cmd_type, gcode, command_string, tags
)
)
try:
cmd, parameters = Commands.parse(command_string)
except ValueError as e:
message = "An error was thrown by the gcode parser, stopping timelapse. Details: {0}".format(str(e))
self.Settings.current_debug_profile().log_warning(
message
)
self.stop_snapshots(message, True)
return None
# get the position state in case it has changed
# if there has been a position or extruder state change, inform any listener
if cmd is not None and not is_snapshot_gcode_command:
# create our state change dictionaries
self.Position.update(command_string, cmd, parameters)
# if this code is snapshot gcode, simply return it to the printer.
if {'plugin:octolapse', 'snapshot_gcode'}.issubset(tags):
return None
if not self.check_for_non_metric_errors():
if self.Position.has_position_error(0):
# There are position errors, report them!
self._on_position_error()
elif (self.State == TimelapseState.WaitingForTrigger
and (self.Position.requires_location_detection(1)) and self.OctoprintPrinter.is_printing()):
if 'source:script' in tags:
# warn user
self._send_plugin_message_async(
"warning",
"Octolapse could not acquire a position while sending"
" OctoPrint scripts (settings=>GCODE Scripts)."
" A fix is in the works, but a modification to OctoPrint itself"
" is required. For now please move your start gcode into the"
" actual gcode file.")
else:
self.State = TimelapseState.AcquiringLocation
if self.OctoprintPrinter.set_job_on_hold(True):
thread = threading.Thread(target=self.acquire_position, args=[command_string, cmd, parameters])
thread.daemon = True
thread.start()
return None,
elif (self.State == TimelapseState.WaitingForTrigger
and self.OctoprintPrinter.is_printing()
and not self.Position.has_position_error(0)):
# update the triggers with the current position
self.Triggers.update(self.Position, command_string)
# see if at least one trigger is triggering
_first_triggering = self.get_first_triggering()
if _first_triggering:
if 'source:script' in tags:
# warn user
self._send_plugin_message_async(
"warning",
"Octolapse could not take a snapshot while sending"
" OctoLapse scripts (settings=>GCODE Scripts)."
" A fix is in the works, but a modification to OctoPrint itself"
" is required. For now please move your start gcode into the"
" actual gcode file."
)
else:
# We are triggering, take a snapshot
self.State = TimelapseState.TakingSnapshot
# pause any timer triggers that are enabled
self.Triggers.pause()
# get the job lock
if self.OctoprintPrinter.set_job_on_hold(True):
# take the snapshot on a new thread
thread = threading.Thread(
target=self.acquire_snapshot, args=[command_string, cmd, parameters, _first_triggering]
)
thread.daemon = True
thread.start()
# suppress the current command, we'll send it later
return None,
elif self.State == TimelapseState.TakingSnapshot:
# Don't do anything further to any commands unless we are
# taking a timelapse , or if octolapse paused the print.
# suppress any commands we don't, under any cirumstances,
# to execute while we're taking a snapshot
if cmd in self.Commands.SuppressedSnapshotGcodeCommands:
command_string = None, # suppress the command
if is_snapshot_gcode_command:
# in all cases do not return the snapshot command to the printer.
# It is NOT a real gcode and could cause errors.
command_string = None,
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
raise
# notify any callbacks
self._send_state_changed_message()
# do any post processing for test mode
if command_string != (None,):
command_string = self._get_command_for_octoprint(command_string, cmd,parameters)
return command_string
def detect_timelapse_start(self, cmd, tags):
# detect print start
if (
self.Settings.is_octolapse_enabled and
self.State == TimelapseState.Idle and
{'trigger:comm.start_print', 'trigger:comm.reset_line_numbers'} <= tags and
# ({'trigger:comm.start_print', 'fileline:1'} <= tags or {'script:beforePrintStarted', 'trigger:comm.send_gcode_script'} <= tags) and
self.OctoprintPrinter.is_printing()
):
if self.OctoprintPrinter.set_job_on_hold(True):
try:
self.State = TimelapseState.Initializing
self.Settings.current_debug_profile().log_print_state_change(
"Print Start Detected. Command: {0}, Tags:{1}".format(cmd, tags)
)
# call the synchronous callback on_print_start
self.on_print_start(tags)
if self.State == TimelapseState.WaitingForTrigger:
# set the current line to 0 so that the plugin checks for line 1 below after startup.
self.CurrentFileLine = 0
finally:
self.OctoprintPrinter.set_job_on_hold(False)
else:
self.on_print_start_failed(
"Unable to start timelapse, failed to acquire a job lock. Print start failed."
)
def check_current_line_number(self, tags):
# check the current line number
if {'source:file'} in tags:
# this line is from the file, advance!
self.CurrentFileLine += 1
if "fileline:{0}".format(self.CurrentFileLine) not in tags:
actual_file_line = "unknown"
for tag in tags:
if len(tag) > 9 and tag.startswith("fileline:"):
actual_file_line = tag[9:]
message = "File line number {0} was expected, but {1} was received!".format(
self.CurrentFileLine + 1,
actual_file_line
)
self.Settings.current_debug_profile().log_error(message)
self.stop_snapshots(message, True)
def check_for_non_metric_errors(self):
# make sure we're not using inches
is_metric = self.Position.is_metric()
has_error = False
error_message = ""
if is_metric is None and self.Position.has_position_error():
has_error = True
error_message = "The printer profile requires an explicit G21 command before any position " \
"altering/setting commands, including any home commands. Stopping timelapse, " \
"but continuing the print. "
elif not is_metric and self.Position.has_position_error():
has_error = True
if self.Printer.units_default == "inches":
error_message = "The printer profile uses 'inches' as the default unit of measurement. In order to" \
" use Octolapse, a G21 command must come before any position altering/setting commands, including" \
" any home commands. Stopping timelapse, but continuing the print. "
else:
error_message = "The gcode file contains a G20 command (set units to inches), which Octolapse " \
"does not support. Stopping timelapse, but continuing the print."
if has_error:
self.stop_snapshots(error_message,has_error)
return has_error
def get_first_triggering(self):
try:
# make sure we're in a state that could want to check for triggers
if not self.State == TimelapseState.WaitingForTrigger:
return False
# see if the PREVIOUS command triggered (that means current gcode gets sent if the trigger[0]
# is triggering
first_trigger = self.Triggers.get_first_triggering(0, Triggers.TRIGGER_TYPE_IN_PATH)
if first_trigger:
self.Settings.current_debug_profile().log_triggering("An in-path snapshot is triggering")
return first_trigger
first_trigger = self.Triggers.get_first_triggering(1, Triggers.TRIGGER_TYPE_DEFAULT)
if first_trigger: # We're triggering
self.Settings.current_debug_profile().log_triggering("A snapshot is triggering")
return first_trigger
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
# no need to re-raise here, the trigger just won't happen
return False
def acquire_position(self, command_string, cmd, parameters):
try:
self.Settings.current_debug_profile().log_print_state_change(
"A position altering command has been detected. Fetching and updating position. "
"Position Command: {0}".format(cmd))
# Undo the last position update, we will be resending the command
self.Position.undo_update()
current_position = self.get_position_async()
if current_position is None:
self.PrintEndStatus = "POSITION_TIMEOUT"
self.State = TimelapseState.WaitingToEndTimelapse
self.Settings.current_debug_profile().log_print_state_change(
"Unable to acquire a position.")
else:
# update position
self.Position.update_position(
x=current_position["x"],
y=current_position["y"],
z=current_position["z"],
e=current_position["e"],
force=True,
calculate_changes=True)
# adjust the triggering command
if self.IsTestMode:
gcode = self.Commands.alter_for_test_mode(command_string, cmd, parameters, return_string=True)
else:
gcode = command_string
if gcode != "":
self.Settings.current_debug_profile().log_print_state_change(
"Sending triggering command for position acquisition - {0}.".format(gcode))
# send the triggering command
self.send_snapshot_gcode_array([gcode])
# set the state
if self.State == TimelapseState.AcquiringLocation:
self.State = TimelapseState.WaitingForTrigger
self.Settings.current_debug_profile().log_print_state_change("Position Acquired")
finally:
self.OctoprintPrinter.set_job_on_hold(False)
def acquire_snapshot(self, command_string, cmd, parameters, trigger):
try:
self.Settings.current_debug_profile().log_snapshot_download(
"About to take a snapshot. Triggering Command: {0}".format(cmd))
if self.OnSnapshotStartCallback is not None:
snapshot_callback_thread = threading.Thread(target=self.OnSnapshotStartCallback)
snapshot_callback_thread.daemon = True
snapshot_callback_thread.start()
# Capture and undo the last position update, we're not going to be using it!
triggering_command_position, triggering_extruder_position = self.Position.undo_update()
# take the snapshot
# Todo: We probably don't need the payload here.
self._most_recent_snapshot_payload = self._take_timelapse_snapshot(
trigger, command_string, cmd, parameters, triggering_command_position, triggering_extruder_position
)
self.Settings.current_debug_profile().log_snapshot_download("The snapshot has completed")
finally:
# set the state
if self.State == TimelapseState.TakingSnapshot:
self.State = TimelapseState.WaitingForTrigger
self.Triggers.resume()
self.OctoprintPrinter.set_job_on_hold(False)
# notify that we're finished, but only if we haven't just stopped the timelapse.
if self._most_recent_snapshot_payload is not None:
# send a copy of the dict in case it gets changed by threads.
self._on_trigger_snapshot_complete(self._most_recent_snapshot_payload.copy())
def on_gcode_sent(self, cmd, cmd_type, gcode, tags):
self.Settings.current_debug_profile().log_gcode_sent(
"Sent to printer: Command Type:{0}, gcode:{1}, cmd: {2}, tags: {3}".format(cmd_type, gcode, cmd, tags))
def on_gcode_received(self, comm, line, *args, **kwargs):
self.Settings.current_debug_profile().log_gcode_received(
"Received from printer: line:{0}".format(line)
)
return line
# internal functions
####################
def _get_command_for_octoprint(self, command_string, cmd, parameters):
if command_string is None or command_string == (None,):
return command_string
if self.IsTestMode and self.State >= TimelapseState.WaitingForTrigger:
return self.Commands.alter_for_test_mode(command_string, cmd, parameters)
# if we were given a list, return it.
if isinstance(command_string, list):
return command_string
# if we were given a command return None (don't change the command at all)
return None
def _send_state_changed_message(self):
"""Notifies any callbacks about any changes contained in the dictionaries.
If you send a dict here the client will get a message, so check the
settings to see if they are subscribed to notifications before populating the dictinaries!"""
delay_seconds = 0
# if another thread is trying to send the message, stop it
if self.StateChangeMessageThread is not None and self.StateChangeMessageThread.isAlive():
self.StateChangeMessageThread.cancel()
if self.LastStateChangeMessageTime is not None:
# do not send more than 1 per second
time_since_last_update = time.time() - self.LastStateChangeMessageTime
if time_since_last_update < 1:
delay_seconds = 1-time_since_last_update
if delay_seconds < 0:
delay_seconds = 0
try:
# Notify any callbacks
if self.OnStateChangedCallback is not None:
def send_change_message():
trigger_change_list = None
position_change_dict = None
position_state_change_dict = None
extruder_change_dict = None
trigger_changes_dict = None
# Get the changes
if self.Settings.show_trigger_state_changes:
trigger_change_list = self.Triggers.state_to_list()
if self.Settings.show_position_changes:
position_change_dict = self.Position.to_position_dict()
if self.Settings.show_position_state_changes:
position_state_change_dict = self.Position.to_state_dict()
if self.Settings.show_extruder_state_changes:
extruder_change_dict = self.Position.Extruder.to_dict()
# if there are any state changes, send them
if (
position_change_dict is not None
or position_state_change_dict is not None
or extruder_change_dict is not None
or trigger_change_list is not None
):
if trigger_change_list is not None and len(trigger_change_list) > 0:
trigger_changes_dict = {
"Name": self.Triggers.Name,
"Triggers": trigger_change_list
}
change_dict = {
"Extruder": extruder_change_dict,
"Position": position_change_dict,
"PositionState": position_state_change_dict,
"TriggerState": trigger_changes_dict
}
if (
change_dict["Extruder"] is not None
or change_dict["Position"] is not None
or change_dict["PositionState"] is not None
or change_dict["TriggerState"] is not None
):
self.OnStateChangedCallback(change_dict)
self.LastStateChangeMessageTime = time.time()
# Send a delayed message
self.StateChangeMessageThread = threading.Timer(
delay_seconds,
send_change_message
)
self.StateChangeMessageThread.daemon = True
self.StateChangeMessageThread.start()
except Exception as e:
# no need to re-raise, callbacks won't be notified, however.
self.Settings.current_debug_profile().log_exception(e)
def _send_plugin_message(self, message_type, message):
self.OnPluginMessageSentCallback(message_type, message)
def _send_plugin_message_async(self, message_type, message):
warning_thread = threading.Thread(target=self._send_plugin_message, args=[message_type, message])
warning_thread.daemon = True
warning_thread.start()
def _is_snapshot_command(self, command_string):
return command_string == self.Printer.snapshot_command
def _is_trigger_waiting(self):
# make sure we're in a state that could want to check for triggers
if not self.State == TimelapseState.WaitingForTrigger:
return None
# Loop through all of the active currentTriggers
waiting_trigger = self.Triggers.get_first_waiting()
if waiting_trigger is not None:
return True
return False
def _on_position_error(self):
message = self.Position.position_error(0)
self.Settings.current_debug_profile().log_error(message)
if self.OnPositionErrorCallback is not None:
position_error_callback_thread = threading.Thread(
target=self.OnPositionErrorCallback, args=[message]
)
position_error_callback_thread.daemon = True
position_error_callback_thread.start()
def _on_trigger_snapshot_complete(self, snapshot_payload):
if self.OnSnapshotCompleteCallback is not None:
payload = {
"success": snapshot_payload["success"],
"error": snapshot_payload["error"],
"snapshot_count": self.SnapshotCount,
"total_snapshot_time": snapshot_payload["total_snapshot_time"],
"current_snapshot_time": snapshot_payload["total_snapshot_time"]
}
if self.OnSnapshotCompleteCallback is not None:
snapshot_complete_callback_thread = threading.Thread(
target=self.OnSnapshotCompleteCallback, args=[payload]
)
snapshot_complete_callback_thread.daemon = True
snapshot_complete_callback_thread.start()
def _render_timelapse(self, print_end_state):
def _render_timelapse_async(render_job_id, timelapse_render_job):
try:
snapshot_thread = threading.Thread(target=timelapse_render_job, args=[])
snapshot_thread.daemon = True
num_snapshot_tasks = self._snapshot_task_queue.qsize()
if num_snapshot_tasks > 0:
self.Settings.current_debug_profile().log_render_start("Started Rendering Timelapse.")
else:
self.Settings.current_debug_profile().log_render_start(
"Waiting for {0} snapshot threads to complete".format(
self._snapshot_task_queue.qsize()))
self._snapshot_task_queue.join()
self.Settings.current_debug_profile().log_render_start(
"All snapshot tasks have completed, rendering timelapse"
)
# we are rendering, set the state before starting the rendering job.
self._rendering_task_queue.put(render_job_id)
snapshot_thread.start()
except Exception as e:
self.Settings.current_debug_profile().log_exception(e)
self._rendering_task_queue.get()
self._rendering_task_queue.task_done()
# make sure we have a non null TimelapseSettings object. We may have terminated the timelapse for some reason
if self.Rendering is not None and self.Rendering.enabled:
job_id = "TimelapseRenderJob_{0}".format(str(uuid.uuid4()))
job = Render.create_render_job(
self.Settings,
self.Snapshot,
self.Rendering,
self.DataFolder,
self.DefaultTimelapseDirectory,
self.FfMpegPath,
1,
self._rendering_task_queue,
job_id,
utility.get_currently_printing_filename(self.OctoprintPrinter),
self.PrintStartTime,
time.time(),
print_end_state,
self.SecondsAddedByOctolapse,
self._on_render_start,
self._on_render_end
)
rendering_thread = threading.Thread(target=_render_timelapse_async, args=[job_id, job])
rendering_thread.daemon = True
rendering_thread.start()
return True
return False
def _on_render_start(self, *args, **kwargs):
job_id = args[0]
self.Settings.current_debug_profile().log_render_start(
"Started rendering/synchronizing the timelapse. JobId: {0}".format(job_id))
payload = args[1]
# notify the caller
if self.OnRenderStartCallback is not None:
render_start_complete_callback_thread = threading.Thread(
target=self.OnRenderStartCallback, args=[payload]
)
render_start_complete_callback_thread.daemon = True
render_start_complete_callback_thread.start()
def _on_render_end(self, *args, **kwargs):
job_id = args[0]
payload = args[1]
self.Settings.current_debug_profile().log_render_complete("Completed rendering. JobId: {0}".format(job_id))
assert (isinstance(payload, RenderingCallbackArgs))
if not payload.HasError and self.Snapshot.cleanup_after_render_fail:
self.CaptureSnapshot.clean_snapshots(utility.get_snapshot_temp_directory(self.DataFolder))
elif self.Snapshot.cleanup_after_render_complete:
self.CaptureSnapshot.clean_snapshots(utility.get_snapshot_temp_directory(self.DataFolder))
if self.OnRenderEndCallback is not None:
render_end_complete_callback_thread = threading.Thread(
target=self.OnRenderEndCallback, args=[payload]
)
render_end_complete_callback_thread.daemon = True
render_end_complete_callback_thread.start()
def _on_timelapse_start(self):
if self.OnTimelapseStartCallback is None:
return
self.OnTimelapseStartCallback()
def _reset(self):
self.State = TimelapseState.Idle
self.CurrentFileLine = 0
if self.Triggers is not None:
self.Triggers.reset()
self.CommandIndex = -1
self.LastStateChangeMessageTime = None
self.PrintStartTime = None
self.SnapshotGcodes = None
self.SavedCommand = None
self.PositionRequestAttempts = 0
self.IsTestMode = False
self.ReturnPositionReceivedTime = None
# A list of callbacks who want to be informed when a timelapse ends
self.TimelapseStopRequested = False
self._snapshot_success = False
self.SnapshotError = ""
self.HasBeenStopped = False
self.CurrentProfiles = {
"printer": "",
"stabilization": "",
"snapshot": "",
"rendering": "",
"camera": "",
"debug_profile": ""
}
# fetch position private variables
self._position_payload = None
self._position_signal.set()
# get snapshot async private variables
self._snapshot_signal.set()
def _reset_snapshot(self):
self.State = TimelapseState.WaitingForTrigger
self.CommandIndex = -1
self.SnapshotGcodes = None
self.SavedCommand = None
self.PositionRequestAttempts = 0
self._snapshot_success = False
self.SnapshotError = ""
threads = [NNThread(train, test, lock) for _ in range(THREAD_NUM)]
for t in threads:
t.setDaemon(True)
t.start()
print 'Q11'
eta = 0.1
r = 0.1
M = [1, 6, 11, 16, 21]
e_out = {1: [], 6: [], 11: [], 16: [], 21: []}
tStart = time.time()
for m in M:
for i in range(experiment):
threadPool.put([eta, r, [2, m, 1], e_out[m]])
threadPool.join()
tEnd = time.time()
print 'Exec time: {}'.format(tEnd - tStart)
for m in M:
print 'm:{}, eout-avg: {}'.format(m, avg(e_out[m]))
print 'Q12'
eta = 0.1
R = [0, 0.001, 0.1, 10.0, 1000.0]
M = [2, 3, 1]
e_out = {0: [], 0.001: [], 0.1: [], 10.0: [], 1000.0: []}
tStart = time.time()
for r in R:
for i in range(experiment):
threadPool.put([eta, r, M, e_out[r]])
threadPool.join()
class GraphiteReporter(threading.Thread):
"""A graphite reporter thread."""
def __init__(self, host, port, maxQueueSize=10000):
"""Connect to a Graphite server on host:port."""
threading.Thread.__init__(self)
self.host, self.port = host, port
self.sock = None
self.queue = Queue()
self.maxQueueSize = maxQueueSize
self.daemon = True
def run(self):
"""Run the thread."""
while True:
try:
try:
name, value, valueType, stamp = self.queue.get()
except TypeError:
break
self.log(name, value, valueType, stamp)
finally:
self.queue.task_done()
def connect(self):
"""Connects to the Graphite server if not already connected."""
if self.sock is not None:
return
backoff = 0.01
while True:
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(5)
sock.connect((self.host, self.port))
self.sock = sock
return
except socket.error:
time.sleep(random.uniform(0, 2.0*backoff))
backoff = min(backoff*2.0, 5.0)
def disconnect(self):
"""Disconnect from the Graphite server if connected."""
if self.sock is not None:
try:
self.sock.close()
except socket.error:
pass
finally:
self.sock = None
def _sendMsg(self, msg):
"""Send a line to graphite. Retry with exponential backoff."""
if not self.sock:
self.connect()
backoff = 0.001
while True:
try:
self.sock.sendall(msg)
break
except socket.error:
self.disconnect()
time.sleep(random.uniform(0, 2.0*backoff))
backoff = min(backoff*2.0, 5.0)
self.connect()
def _sanitizeName(self, name):
"""Sanitize a metric name."""
return name.replace(' ', '-')
def log(self, name, value, valueType=None, stamp=None):
"""Log a named numeric value. The value type may be 'value',
'count', or None."""
if type(value) == float:
form = "%s%s %2.2f %d\n"
else:
form = "%s%s %s %d\n"
if valueType is not None and len(valueType) > 0 and valueType[0] != '.':
valueType = '.' + valueType
if not stamp:
stamp = time.time()
self._sendMsg(form % (self._sanitizeName(name), valueType or '', value, stamp))
def enqueue(self, name, value, valueType=None, stamp=None):
"""Enqueue a call to log."""
# If queue is too large, refuse to log.
if self.maxQueueSize and self.queue.qsize() > self.maxQueueSize:
return
# Stick arguments into the queue
self.queue.put((name, value, valueType, stamp))
def flush(self):
"""Block until all stats have been sent to Graphite."""
self.queue.join()
def shutdown(self):
"""Shut down the background thread."""
self.queue.put(None)
self.flush()
while True:
# get the song from queue
song = self.queue.get()
download_song(song)
# mark the song as downloaded
self.queue.task_done()
if __name__ == '__main__':
parser = argparse.ArgumentParser(description='fetch music from accuradio.com')
parser.add_argument('genre', help='something like jazz or adultalternative')
parser.add_argument('channel', help='Groove Jazz or Latin Jazz', nargs='?', default=None)
args = parser.parse_args()
channels = fetch_channels(args.genre)
if args.channel and args.channel in channels:
# create THREAD_AMOUNT of threads to download songs
for i in range(THREAD_AMOUNT):
thread = DownloadThread(queue)
thread.setDaemon(True)
thread.start()
fetch(args.channel, channels[args.channel])
else:
print '\n'.join(sorted(channels))
# wait for all songs to download
queue.join()
class UnknownRSThreadedCPUKernel(UnknownRSKernel):
def __init__(self):
UnknownRSKernel.__init__(self)
self.numthreads = None
self.threads = None
self.q = Queue()
self.G_lock = Lock()
self.sigma_lock = Lock()
def setup(self,params,diagout,statout,ostream):
UnknownRSKernel.setup(self, params, diagout, statout, ostream)
numthreads = params.get('num_threads','auto')
det_numthreads = multiprocessing.cpu_count()
if numthreads == 'auto':
numthreads = np.inf
use_numthreads = min(det_numthreads,numthreads)
print("Detected {0} cores, using {1} threads".format(det_numthreads, use_numthreads))
if self.threads == None:
self.numthreads = use_numthreads
self.threads = [Thread(target=self.worker) for i in range(self.numthreads)]
for th in self.threads:
th.daemon = True
th.start()
def precompute_projections(self,fM):
if self.using_precomp_slicing:
# precompute all possible slices
fM = np.require(fM, dtype=np.float32)
getslices_interp(fM, self.slice_ops, self.rad, self.beamstop_rad, res=self.slices.reshape(-1, self.N_T))
self.precomp_slices = self.slices.reshape((-1, self.N_T))
else:
# do on-the-fly slicing
self.precomp_slices = None
def worker(self):
g_tmp = None
lcl_sigma2_est = None
lcl_correlation = None
lcl_power = None
lcl_G = None
workspace = None
while True:
idxs, fM, res, compute_grad = self.q.get()
sigma2 = self.inlier_sigma2_trunc
inlier_const = self.inlier_const - res['totallike_logscale']
if lcl_sigma2_est is None or lcl_sigma2_est.shape[0] != self.N_T:
lcl_sigma2_est = np.zeros(self.N_T,dtype=np.float64)
else:
lcl_sigma2_est[:] = 0
if lcl_correlation is None or lcl_correlation.shape[0] != self.N_T:
lcl_correlation = np.zeros(self.N_T,dtype=np.float64)
else:
lcl_correlation[:] = 0
if lcl_power is None or lcl_power.shape[0] != self.N_T:
lcl_power = np.zeros(self.N_T,dtype=np.float64)
else:
lcl_power[:] = 0
# Result buffers
like = res['like']
Evar_like = res['Evar_like']
if compute_grad:
if lcl_G is None or lcl_G.shape != self.G.shape:
lcl_G = np.zeros_like(self.G)
else:
lcl_G[:] = 0
for idx in idxs:
tic = time.time()
if self.sampler_S is not None:
slice_ops, envelope, \
W_R_sampled, sampleinfo_R, slices_sampled, slice_inds, \
W_I_sampled, sampleinfo_I, rotd_sampled, rotc_sampled, \
W_S_sampled, sampleinfo_S, S_sampled = \
self.prep_operators(fM,idx,res=res)
else:
slice_ops, envelope, \
W_R_sampled, sampleinfo_R, slices_sampled, slice_inds, \
W_I_sampled, sampleinfo_I, rotd_sampled, rotc_sampled = \
self.prep_operators(fM,idx,res=res)
N_slices = slices_sampled.shape[0]
log_W_R = np.log(W_R_sampled)
log_W_I = np.log(W_I_sampled)
if self.sampler_S is not None:
log_W_S = np.log(W_S_sampled)
if compute_grad:
if g_tmp is None or g_tmp.shape[0] < N_slices or g_tmp.shape[1] != self.N_T:
g_tmp = np.empty((N_slices,self.N_T), dtype=self.G_datatype)
else:
g_tmp[:] = 0.0
g = g_tmp[0:N_slices]
g[:] = 0
else:
g = None
res['kern_timing']['prep'][idx] = time.time() - tic
# get angular correlation slices
if self.use_angular_correlation:
tic = time.time()
# ac_slices_sampled, ac_data_sampled
ac_indices = self.get_angular_correlation(
slices_sampled, rotd_sampled, rotc_sampled, envelope, W_I_sampled)
else:
pass # add timing
tic = time.time()
if self.sampler_S is not None:
raise NotImplementedError('no shifts')
if self.use_angular_correlation:
like[idx], (cphi_I, cphi_R), csigma2_est, ccorrelation, cpower, workspace = \
objective_kernels.doimage_ACRI(slices_sampled, envelope, \
rotc_sampled, rotd_sampled, \
# ac_slices_sampled, ac_data_sampled, \
ac_indices,
log_W_I, log_W_R, \
sigma2, g, workspace)
else:
if len(W_I_sampled) == 1:
like[idx], cphi_R, csigma2_est, ccorrelation, cpower, workspace = \
objective_kernels.doimage_R(slices_sampled, envelope,
rotc_sampled.reshape((-1,)), rotd_sampled.reshape((-1,)), \
log_W_R, sigma2, g, workspace)
cphi_I = np.array([0.0])
else:
like[idx], (cphi_I, cphi_R), csigma2_est, ccorrelation, cpower, workspace = \
objective_kernels.doimage_RI(slices_sampled, envelope, \
rotc_sampled, rotd_sampled, \
log_W_I, log_W_R, \
sigma2, g, workspace)
res['kern_timing']['work'][idx] = time.time() - tic
tic = time.time()
# like[idx] is the negative log likelihood of the image
like[idx] += self.inlier_like_trunc[idx]
# Evar_like[idx] is the expected error
Evar_like[idx] = (csigma2_est.sum() + self.imgpower_trunc[idx]) / self.N**2
lcl_sigma2_est += csigma2_est
lcl_correlation += ccorrelation
lcl_power += cpower
like[idx] += inlier_const
if compute_grad:
if self.using_precomp_slicing:
lcl_G[slice_inds] += g
else:
lcl_G += merge_slices(g, slice_ops, self.N, self.rad, self.beamstop_rad).reshape(self.N, self.N, self.N)
res['kern_timing']['proc'][idx] = time.time() - tic
tic = time.time()
if self.sampler_S is not None:
self.store_results(idx, 1, \
cphi_R,sampleinfo_R, \
cphi_I,sampleinfo_I, \
cphi_S=cphi_S, sampleinfo_S=sampleinfo_S, res=res, \
logspace_phis = True)
else:
self.store_results(idx, 1, \
cphi_R,sampleinfo_R, \
cphi_I,sampleinfo_I, \
res = res, logspace_phis = True)
res['kern_timing']['store'][idx] = time.time() - tic
if compute_grad:
self.G_lock.acquire()
self.G += lcl_G
self.G_lock.release()
self.sigma_lock.acquire()
res['sigma2_est'][self.truncmask] += lcl_sigma2_est/self.minibatch['N_M']
res['correlation'][self.truncmask] += lcl_correlation/self.minibatch['N_M']
res['power'][self.truncmask] += lcl_power/self.minibatch['N_M']
self.sigma_lock.release()
self.q.task_done()
def eval(self, fM, compute_gradient=True, M=None):
tic = time.time()
N_M = self.minibatch['N_M']
outputs = self.get_result_struct()
outputs['like_timing'] = {}
if compute_gradient:
self.G[:] = 0
outputs['like_timing']['setup'] = time.time() - tic
tic = time.time()
self.precompute_projections(fM)
outputs['like_timing']['slice'] = time.time() - tic
tic = time.time()
numJobs = min(N_M,3*self.numthreads)
#numJobs = int(self.numthreads + self.numthreads/2)
imsPerJob = int(np.ceil(float(N_M)/numJobs))
for jobId in xrange(numJobs):
idxs = range(imsPerJob*jobId, \
min(imsPerJob*(jobId+1),N_M))
self.q.put(( idxs, fM, outputs, compute_gradient ))
outputs['like_timing']['queue'] = time.time() - tic
tic = time.time()
self.q.join()
outputs['like_timing']['join'] = time.time() - tic
outputs['kern_timing'] = dict([(k,np.sum(v)/self.numthreads) for k,v in outputs['kern_timing'].items()])
# compute gradient of likelihood
if compute_gradient:
tic = time.time()
if self.using_precomp_slicing:
# dLdfM = self.slice_ops.T.dot(self.G.reshape((-1,))).reshape((self.N,self.N,self.N))
dLdfM = merge_slices(self.G, self.slice_ops, self.N, self.rad, self.beamstop_rad).reshape(self.N, self.N, self.N)
dLdfM /= N_M
else:
dLdfM = self.G/N_M
outputs['like_timing']['unslice'] = time.time() - tic
outputs['N_R'] = self.N_R
outputs['N_I'] = self.N_I
if self.sampler_S is not None:
outputs['N_S'] = self.N_S
outputs['N_Total'] = self.N_R*self.N_I*self.N_S
else:
outputs['N_Total'] = self.N_R * self.N_I
outputs['N_R_sampled_total'] = float(np.sum(outputs['N_R_sampled']))/self.N_R
outputs['N_I_sampled_total'] = float(np.sum(outputs['N_I_sampled']))/self.N_I
if self.sampler_S is not None:
outputs['N_S_sampled_total'] = float(np.sum(outputs['N_S_sampled']))/self.N_S
outputs['N_Total_sampled_total'] = float(np.sum(outputs['N_Total_sampled']))/(self.N_R*self.N_I*self.N_S)
else:
outputs['N_Total_sampled_total'] = float(np.sum(outputs['N_Total_sampled']))/(self.N_R * self.N_I)
L = outputs['like'].sum(dtype=np.float64)/N_M
outputs['L'] = L
if compute_gradient:
return L, dLdfM, outputs
else:
return L, outputs
class WorkerPool(object):
"""TODO:"""
halt_command='halt command'
def __init__(self,numWorkers=16,items=None):
self.workers=[]
self.should_stop=False
self.work_queue=Queue()
self.completed_queue=Queue()
self.num_assigned=0
if items is not None:
for item in items:
self.work_queue.put(item)
self.num_assigned += 1
for i in range(0,numWorkers):
w = Worker("worker%d" % i,self)
self.workers.append(w)
w.start()
self.numWorkers = numWorkers
self.logger = logger
###
def getNumWorkers(self):
return self.numWorkers
def getNextWorkItem(self):
return self.work_queue.get(block=True)
def addFinishedWorkItem(self,command):
self.completed_queue.put(command)
self.work_queue.task_done()
def markTaskDone(self):
self.work_queue.task_done()
def addCommand(self,cmd):
self.logger.debug("Adding cmd to work_queue: %s" % cmd.cmdStr)
self.work_queue.put(cmd)
self.num_assigned += 1
def wait_and_printdots(self,command_count,quiet=True):
while self.completed_queue.qsize() < command_count:
time.sleep(1)
if not quiet:
sys.stdout.write(".")
sys.stdout.flush()
if not quiet:
print " "
self.join()
def print_progress(self, command_count):
while True:
num_completed = self.completed_queue.qsize()
num_completed_percentage = 0
if command_count:
num_completed_percentage = float(num_completed) / command_count
logger.info('%0.2f%% of jobs completed' % (num_completed_percentage * 100))
if num_completed >= command_count:
return
time.sleep(10)
def join(self):
self.work_queue.join()
return True
def joinWorkers(self):
for w in self.workers:
w.join()
def getCompletedItems(self):
completedList=[]
try:
while True:
item=self.completed_queue.get(False)
if item is not None:
completedList.append(item)
except Empty:
return completedList
return completedList #just to be sure
def check_results(self):
""" goes through all items in the completed_queue and throws an exception at the
first one that didn't execute successfully
throws ExecutionError
"""
try:
while True:
item=self.completed_queue.get(False)
if not item.get_results().wasSuccessful():
raise ExecutionError("Error Executing Command: ",item)
except Empty:
return
def empty_completed_items(self):
while not self.completed_queue.empty():
self.completed_queue.get(False)
def isDone(self):
#TODO: not sure that qsize() is safe
return (self.num_assigned == self.completed_queue.qsize())
def haltWork(self):
self.logger.debug("WorkerPool haltWork()")
self.should_stop=True
for w in self.workers:
w.haltWork()
self.work_queue.put(self.halt_command)
class Stats(object):
def __request(self, path):
r = self.__session.get(path)
r.raise_for_status()
return r.json()
def __get_post_data(self):
while not self.__ids_to_process.empty():
post_id = self.__ids_to_process.get()
reactions = self.__request("https://graph.facebook.com/v2.10/" +
post_id + "/reactions")
while reactions["data"]:
for reaction in reactions["data"]:
user_id = reaction["id"]
reaction_type = reaction["type"]
self.__lock.acquire()
self.__user_names[user_id] = reaction["name"]
if user_id not in self.__users:
self.__users[user_id] = {}
self.__user_reactions[user_id] = 0
if reaction_type not in self.__users[user_id]:
self.__users[user_id][reaction_type] = 0
if post_id not in self.__initial_top_posts:
self.__users[user_id][reaction_type] += 1
self.__user_reactions[user_id] += 1
self.__total_reactions += 1
self.__posts[post_id] += 1
self.__lock.release()
if "next" not in reactions["paging"]:
break
reactions = self.__request(reactions["paging"]["next"])
self.__ids_to_process.task_done()
def __init__(self,
access_token,
post_ids,
top_reactors=[],
top_posts=[],
total_posts=0,
total_reactions=0,
num_threads=64):
self.__users = {}
self.__user_names = {}
self.__user_reactions = {}
for user_info in top_reactors:
user_id = user_info["id"]
reactions = user_info["reactions"]
self.__users[user_id] = reactions
self.__user_names[user_id] = user_info["name"]
self.__user_reactions[user_id] = sum(reactions.values())
self.__posts = {post_id: 0 for post_id in post_ids + top_posts}
self.__total_posts = len(post_ids) + total_posts
self.__total_reactions = total_reactions
self.__initial_top_posts = set(top_posts)
self.__session = Session()
self.__session.params = {"access_token": access_token}
self.__lock = threading.Lock()
self.__ids_to_process = Queue()
for post_id in post_ids + top_posts:
self.__ids_to_process.put(post_id)
for _ in range(num_threads):
thread = threading.Thread(target=self.__get_post_data)
thread.daemon = True
thread.start()
def __calculate_stats(self):
self.__ids_to_process.join()
try:
self.__top_posts
except AttributeError:
self.__top_posts = sorted(
[(post_id, reactions)
for post_id, reactions in self.__posts.items()],
key=lambda x: x[1],
reverse=True)
self.__top_reactors = []
for reactor_info in sorted(
[(user_id, reactions)
for user_id, reactions in self.__user_reactions.items()],
key=lambda x: x[1],
reverse=True):
user_id = reactor_info[0]
self.__top_reactors.append({
"id": user_id,
"name": self.__user_names[user_id],
"reactions": self.__users[user_id]
})
def get_top_posts(self, num_posts=100):
self.__calculate_stats()
return self.__top_posts[0:num_posts]
def get_top_post_message(self, num_posts=10):
self.__calculate_stats()
message = "***Top " + str(num_posts) + " Posts***\n"
total_reactions = 0
ranking = 1
for post_id, reactions in self.get_top_posts(num_posts):
message += str(ranking) + ". facebook.com/" + post_id + "\n"
total_reactions += reactions
ranking += 1
return message + "Average reactions per post: " + str(
float(total_reactions) / num_posts)
def get_top_reactors(self, num_reactors=100):
self.__calculate_stats()
return self.__top_reactors[0:num_reactors]
def get_top_reactor_message(self, num_reactors=10):
self.__calculate_stats()
emoticons = {
"LIKE": "\xF0\x9F\x91\x8D",
"LOVE": "\xF0\x9F\x92\x9F",
"HAHA": "\xF0\x9F\x98\x86",
"WOW": "\xF0\x9F\x98\xAE",
"SAD": "\xF0\x9F\x98\xA2",
"ANGRY": "\xF0\x9F\x98\xA1",
"THANKFUL": "\xF0\x9F\x8C\xB8",
"PRIDE": "\xF0\x9F\x8C\x88"
}
message = "***Top " + str(num_reactors) + " Reactors***\n"
ranking = 1
for reactor_info in self.get_top_reactors(num_reactors):
user_id = reactor_info["id"]
reactions = self.__user_reactions[user_id]
reactions_breakdown = " ".join([
" ".join([emoticons[reaction_type],
str(num)])
for (reaction_type,
num) in sorted(self.__users[user_id].items(),
key=lambda x: x[1],
reverse=True)
])
message += str(
ranking) + ". " + self.__user_names[user_id] + " - " + str(
reactions) + ": " + reactions_breakdown.decode(
"utf-8") + "\n"
ranking += 1
return message + "Average reactions per post: " + str(
float(self.__total_reactions) / float(self.__total_posts))
@property
def total_posts(self):
self.__calculate_stats()
return self.__total_posts
@property
def total_reactions(self):
self.__calculate_stats()
return self.__total_reactions
@property
def reactors(self):
self.__calculate_stats()
return self.__user_names
accu = Thread(target=accumulate_checkins, name='AccumulateCheckins')
accu.daemon = True
accu.start()
start = clock()
end = start + ARGS.duration * 60 * 60
failures = th.Failures(initial_waiting_time=2.0)
while clock() < end:
try:
read_twitter_stream(api, end)
except (KeyboardInterrupt, SystemExit):
CHECKINS_QUEUE.put_nowait(None)
raise
except:
msg = 'Fail to read or enqueue tweet\n'
cac.logging.exception(msg)
waiting_time = failures.fail()
if clock() + waiting_time > end or \
failures.recent_failures >= 5 or \
not accu.is_alive():
# We might as well quit right now, as stars are agains us
CHECKINS_QUEUE.put_nowait(None)
break
msg = 'Will wait for {:.0f} seconds'.format(waiting_time)
cac.logging.info(msg)
failures.do_sleep()
CHECKINS_QUEUE.join()
report = 'insert {} valid checkins in {:.2f}s (out of {}).'
cac.logging.info(report.format(NUM_VALID, clock() - start, NB_TWEETS))
sleep(5)
def check(testfile, indy_url, scan_dirs=False, vagrant_dir=None):
with open(testfile) as f:
build_config = yaml.safe_load(f)
vagrant_dir = mb.vagrant.find_vagrant_dir(vagrant_dir)
cwd = os.getcwd()
try:
project_dir = os.path.abspath(os.path.dirname(testfile))
reports_dir = os.path.join(project_dir, 'reports')
if not os.path.isdir(reports_dir):
os.makedirs(reports_dir)
tid_base = "build_%s" % os.path.basename(project_dir)
if build_config.get('vagrant') is not None:
mb.vagrant.init_ssh_config(vagrant_dir)
mb.vagrant.vagrant_env(build_config, 'pre-report', indy_url,
vagrant_dir, project_dir, reports_dir)
os.chdir(project_dir)
report = build_config['report']
report_queue = Queue()
try:
if scan_dirs is True:
task_ids = []
for builds_cand in os.listdir(project_dir):
if builds_cand.startswith("builds"):
for tid in os.listdir(
os.path.join(project_dir, builds_cand)):
if tid.startswith(tid_base):
task_ids.append(tid)
else:
task_ids = mb.reporter.get_sealed_reports(indy_url)
task_ids = [
tid for tid in task_ids if tid.startswith(tid_base)
]
print "\n".join(task_ids)
for t in range(int(report['threads'])):
thread = mb.reporter.Reporter(report_queue)
thread.daemon = True
thread.start()
for tid in task_ids:
builddir = os.path.join(reports_dir, tid)
if not os.path.isdir(builddir):
os.makedirs(builddir)
report_queue.put((builddir, indy_url, tid))
# mb.reporter.verify_report(builddir, args.indy_url, tid)
report_queue.join()
except (KeyboardInterrupt, SystemExit) as e:
print e
print "Quitting."
if build_config.get('vagrant') is not None:
mb.vagrant.vagrant_env(build_config, 'post-report', indy_url,
vagrant_dir, project_dir, reports_dir)
finally:
os.chdir(cwd)
while True:
sys.stdout.write(
"Directories left / total: {0} / {1}, Time: {2} s \r".format(
q.qsize(), s.qsize(), int(time.time() - start)))
sys.stdout.flush()
time.sleep(1)
s = Queue()
q = Queue()
for i in range(args.workers):
t = Thread(target=worker)
t.daemon = True
t.start()
t = Thread(target=status)
t.daemon = True
t.start()
if args.dirs:
for file in open(args.dirs).read().splitlines():
q.put(file.rstrip())
else:
q.put('/')
q.join()
print(
"\nTotal time: {0} s, Total directories: {1}".format(
int(time.time() - start)), s.qsize())
def add_data(h5_file, image_dir, prefix, args):
# Make a list of all images in the source directory
image_list = genlist(image_dir)
# Shuffle to mix images for different styles
np.random.shuffle(image_list)
num_images = len(image_list)
# Resize all images and copy them into the hdf5 file
# We'll bravely try multithreading
dset_name = os.path.join(prefix, 'images')
dset_size = (num_images, 3, args.height, args.width)
imgs_dset = h5_file.create_dataset(dset_name, dset_size, np.uint8)
# input_queue stores (idx, filename) tuples,
# output_queue stores (idx, resized_img) tuples
input_queue = Queue()
output_queue = Queue()
# Read workers pull images off disk and resize them
def read_worker():
while True:
idx, filename = input_queue.get()
img = imread(filename)
try:
# First crop the image so its size is a multiple of max_resize
H, W = img.shape[0], img.shape[1]
H_crop = H - H % args.max_resize
W_crop = W - W % args.max_resize
img = img[:H_crop, :W_crop]
img = imresize(img, (args.height, args.width))
except (ValueError, IndexError) as e:
print filename
print img.shape, img.dtype
print e
input_queue.task_done()
output_queue.put((idx, img))
# Write workers write resized images to the hdf5 file
def write_worker():
num_written = 0
while True:
idx, img = output_queue.get()
if img.ndim == 3:
#RGB image, transpose from H x W x C to C x H x W
imgs_dset[idx] = img.transpose(2, 0, 1)
#Throw gray images
#elif img.ndim == 2:
# Grayscale image; it is H x W so broadcasting to C x H x W will just copy
# grayscale values into all channels.
#imgs_dset[idx] = img
output_queue.task_done()
num_written = num_written + 1
if num_written % 100 == 0:
print 'Copied %d / %d images' % (num_written, num_images)
# Start the read workers.
for i in xrange(args.num_workers):
t = Thread(target=read_worker)
t.daemon = True
t.start()
# h5py locks internally, so we can only use a single write worker =(
t = Thread(target=write_worker)
t.daemon = True
t.start()
for idx, filename in enumerate(image_list):
if args.max_images > 0 and idx >= args.max_images: break
input_queue.put((idx, filename))
input_queue.join()
output_queue.join()
def build(testfile, indy_url, delay, vagrant_dir):
with open(testfile) as f:
build_config = yaml.safe_load(f)
if vagrant_dir is not None:
vagrant_dir = mb.vagrant.find_vagrant_dir(vagrant_dir)
if delay is None:
delay = 0
else:
delay = int(delay)
cwd = os.getcwd()
try:
project_dir = os.path.abspath(os.path.dirname(testfile))
builds_dir = "builds-%s" % dt.now().strftime("%Y%m%dT%H%M%S")
tid_base = "build_%s" % os.path.basename(project_dir)
build = build_config['build']
report = build_config.get('report')
if build_config.get('vagrant') is not None:
mb.vagrant.init_ssh_config(vagrant_dir)
mb.vagrant.vagrant_env(build_config, 'pre-build', indy_url,
vagrant_dir, project_dir,
os.path.join(project_dir, builds_dir))
os.chdir(project_dir)
project_src_dir = build.get('project-dir') or 'project'
project_src_dir = os.path.join(os.getcwd(), project_src_dir)
git_branch = build.get('git-branch') or 'master'
build_queue = Queue()
report_queue = Queue()
try:
for t in range(int(build['threads'])):
thread = mb.builder.Builder(build_queue, report_queue)
thread.daemon = True
thread.start()
for x in range(build['builds']):
builddir = mb.util.setup_builddir(builds_dir, project_src_dir,
git_branch, tid_base, x)
build_queue.put(
(builddir, indy_url, build_config.get('proxy-port'),
(x % int(build['threads'])) * int(delay)))
build_queue.join()
if build_config.get('vagrant') is not None:
mb.vagrant.vagrant_env(build_config, 'post-build', indy_url,
vagrant_dir, project_dir, builds_dir)
mb.vagrant.vagrant_env(build_config, 'pre-report', indy_url,
vagrant_dir, project_dir, builds_dir)
if report is not None:
if build_config.get('vagrant') is not None:
mb.vagrant.vagrant_env(build_config, 'pre-report',
indy_url, vagrant_dir, project_dir,
builds_dir)
for t in range(int(report['threads'])):
thread = mb.reporter.Reporter(report_queue)
thread.daemon = True
thread.start()
report_queue.join()
except Exception as e:
print e
print "Quitting."
if build_config.get('vagrant') is not None:
mb.vagrant.vagrant_env(build_config, 'post-report', indy_url,
vagrant_dir, project_dir, builds_dir)
finally:
os.chdir(cwd)
class MultiThreadESSearchClient():
def __init__(self, host, query_template, search_uris,
concurrency, threads_per_second, requests_queue=None):
self.host = host
self.query_template = query_template
self.requests_queue = Queue(maxsize=concurrency) if not isinstance(
requests_queue, Queue) else requests_queue
self.sess = self.create_request_session()
self.search_uris = search_uris
self.threads_per_second = threads_per_second
self.requests_count = 0
self.initial_requests_start_time = time()
self.start_threads(concurrency)
def start_threads(self, concurrency):
# Define threads
for i in range(concurrency):
t = Thread(target=self.do_work)
t.daemon = True
t.start()
def fire_requests(self):
for uri in self.search_uris:
es_query_keyword = self.get_query_word_from_url(uri)
request_params = dict(
search_uri=uri.replace('\r\n', ''),
query_keyword=es_query_keyword
)
self.requests_queue.put(request_params)
# Block until all items in the queue have been handled
self.requests_queue.join()
# Output total execution time after all requests are fired
total_time_elapsed = time() - self.initial_requests_start_time
print("\nTotal request time in seconds: %s" % total_time_elapsed)
def do_work(self):
while True:
# Initial start time
if not hasattr(self, 'start_time'):
self.start_time = time()
self.elapsed_time = time() - self.start_time
# If we have passed 1 seconds or had more than threads_per_seconds requestswithin a second
if self.elapsed_time >= 1 or self.requests_count > self.threads_per_second:
self.start_time = time()
self.requests_count = 0
# If had more requests than threads_per_seconds then pause for the diff time
if self.elapsed_time < 1:
sleep(1 - self.elapsed_time)
request_params = self.requests_queue.get()
response = self.perform_es_request(**request_params)
self.output_results(response, **request_params)
self.requests_queue.task_done()
self.requests_count += 1
def perform_es_request(self, query_keyword, search_uri):
params = dict(
url=self.host + '/_search',
headers={"content-type": 'application/json'},
data=self.query_template.replace("{{queryKeyword}}", query_keyword)
)
start_time = time()
response = self.do_request(**params)
end_time = time()
response.elapsed_time = (end_time - start_time) * 1000
return response
def do_request(self, **kwargs):
try:
response = self.sess.post(
timeout=60,
**kwargs
)
return response
except (requests.exceptions.Timeout, requests.exceptions.ConnectionError) as e:
print e.message
raise e
def output_results(self, response, query_keyword, search_uri):
response_to_json = response.json()
print('%s,%s,%s,%s' % (
response_to_json['took'], response.elapsed_time, search_uri, query_keyword))
def create_request_session(self):
sess = requests.Session()
adapter = requests.adapters.HTTPAdapter(
pool_connections=500,
pool_maxsize=100,
max_retries=3
)
sess.mount('http://', adapter)
return sess
def get_query_word_from_url(self, url):
url = unquote(url).decode('utf-8')
return re.search('q=([^\r]+)&', url).group(1).split('&')[0]
class KafkaThreadedImageDownloader(KafkaImageDownloader):
def __init__(self, global_conf_filename, prefix=default_prefix, pid=None):
super(KafkaThreadedImageDownloader,
self).__init__(global_conf_filename, prefix, pid)
# Get number of threads
self.nb_threads = self.get_required_param('nb_threads')
def set_pp(self):
self.pp = "KafkaThreadedImageDownloader"
if self.pid:
self.pp += "." + str(self.pid)
def process_one(self, msg):
self.print_stats(msg)
msg_value = json.loads(msg.value)
# From msg value get list_urls for image objects only
list_urls = self.get_images_urls(msg_value)
nb_img = len(list_urls)
if self.verbose > 2:
print_msg = "[{}.process_one: info] Got {} image urls from ad id {}"
print print_msg.format(self.pp, nb_img, msg_value['_id'])
# Initialize queues
from Queue import Queue
self.q_in = Queue(0)
self.q_out = Queue(0)
threads = []
if list_urls:
# Fill input queue
for item in list_urls:
self.q_in.put(item)
# Start threads, at most one per image
for i in range(min(self.nb_threads, len(list_urls))):
# should read (url, obj_pos) from self.q_in
# and push (url, obj_pos, buffer, img_info, start_process, end_process) to self.q_out
thread = ThreadedDownloader(self.q_in, self.q_out)
thread.start()
threads.append(thread)
# Wait for all tasks to be marked as done
if self.q_in.qsize() > 0:
self.q_in.join()
# Get images data and infos
dict_imgs = dict()
nb_dl_img = 0
nb_valid_dl_img = 0
while nb_dl_img < nb_img:
url, obj_pos, img_buffer, img_info, start_process, end_process, inst = self.q_out.get(
)
nb_dl_img += 1
if img_buffer is not None and img_info is not None:
# TODO: should we just discard buffer here?...
sha1, img_type, width, height = img_info
dict_imgs[url] = {
'obj_pos': obj_pos,
'img_buffer': img_buffer,
'sha1': sha1,
'img_info': {
'format': img_type,
'width': width,
'height': height
}
}
nb_valid_dl_img += 1
self.toc_process_ok(start_process, end_process)
else:
self.toc_process_failed(start_process, end_process)
if inst is not None:
if self.verbose > 0:
print_msg = "[{}.process_one: error] Could not download image from: {} ({})"
print print_msg.format(self.pp, url, inst)
sys.stdout.flush()
else:
if self.verbose > 1:
print_msg = "[{}.process_one: info] Could not download image from: {} (tried downloading for {}s)"
print print_msg.format(self.pp, url,
end_process - start_process)
sys.stdout.flush()
if self.verbose > 0:
if nb_valid_dl_img < nb_dl_img:
print_msg = "[{}.process_one: info] Found only {} valid images out of {} for ad id {}"
print print_msg.format(self.pp, nb_valid_dl_img, nb_dl_img,
msg_value['_id'])
sys.stdout.flush()
# Push to cdr_out_topic
#self.producer.send(self.cdr_out_topic, self.build_cdr_msg(msg_value, dict_imgs))
if self.cdr_out_topic:
self.producer.send(self.cdr_out_topic,
self.build_cdr_msg(msg_value, dict_imgs))
else:
print_msg = "[{}.process_one: warning] cdr_out_topic is not defined"
print print_msg.format(self.pp)
# Push to images_out_topic
for img_out_msg in self.build_image_msg(dict_imgs):
self.producer.send(self.images_out_topic, img_out_msg)
class CappedQueue(object):
""" Capped queue with opiniatied methods
"""
def __init__(self, maxsize=None, name=None):
if maxsize is None:
maxsize = CONFIG["MAX_QUEUE_LENGTH"]
self.maxsize = maxsize
self.queue = Queue(self.maxsize)
self.name = name or "queue"
self._count_ingress = itertools.count()
self._count_dropped = itertools.count()
self._count_egress = itertools.count()
self._last_count_ingress = 0
self._last_count_dropped = 0
self._last_count_egress = 0
def get(self, timeout, block=True):
""" Wait for up to timeout for an item to return and block while
waiting
"""
ret = self.queue.get(timeout=timeout, block=block)
next(self._count_egress)
return ret
def get_nowait(self):
""" Get without waiting, raise queue.Empty if nothing is present
"""
ret = self.queue.get_nowait()
next(self._count_egress)
return ret
def put(self, item):
""" Tries to put an item to the queue, if the queue is full, pop an
item and try again
"""
while True:
try:
ret = self.queue.put_nowait(item)
next(self._count_ingress)
return ret
except Full:
try:
self.queue.get_nowait()
self.queue.task_done()
next(self._count_dropped)
except Empty:
pass
def half_full(self):
""" Return True if the current queue size if at least half the maxsize
"""
return self.queue.qsize() > (self.maxsize / 2.)
def get_metrics(self, at=None):
""" Return the queue metrics, this method is *NOT* thread-safe.
"""
ingress_delta = next(self._count_ingress) - self._last_count_ingress
dropped_delta = next(self._count_dropped) - self._last_count_dropped
egress_delta = next(self._count_egress) - self._last_count_egress
self._last_count_ingress += ingress_delta + 1
self._last_count_dropped += dropped_delta + 1
self._last_count_egress += egress_delta + 1
at = at or now()
return [
("event_manager", at, "{}_ingress".format(self.name), ingress_delta),
("event_manager", at, "{}_dropped".format(self.name), dropped_delta),
("event_manager", at, "{}_egress".format(self.name), egress_delta),
]
def task_done(self):
""" Inform the queue that an event was processed.
"""
self.queue.task_done()
def join(self):
""" Block until all events are consumed
"""
self.queue.join()
import threading
import time
import sys
from Queue import Queue
from utils import do_work
pending_tasks = Queue()
def do_work_from_queue():
while True:
do_work(pending_tasks.get())
pending_tasks.task_done()
if __name__ == "__main__":
tasks = [sys.argv[1]] * int(sys.argv[2])
num_threads = int(sys.argv[3]) if len(sys.argv) > 3 else 4
[pending_tasks.put(task) for task in tasks]
start = time.time()
for i in range(num_threads):
t = threading.Thread(target=do_work_from_queue)
t.daemon = True
t.start()
pending_tasks.join()
print time.time() - start
class VideoRecorder:
def __init__(self):
"docstring"
try:
self.camera = PiCamera()
self.set_camera_params()
# self.camera.start_preview()
#self.camera.start_recording(self.my_stream, format="h264")
channel = grpc.insecure_channel('200.126.23.95:50052')
self.grpc_stub = FeatureExtractionApi_pb2_grpc.FeatureExtractionStub(
channel)
self.recording_stop = True
self.stream = io.BytesIO()
self.image_queue = Queue()
self.count = 0
logger.debug("Camera and grpc started")
except (PiCameraMMALError, PiCameraError) as error:
raise ConnectionError("Camera not available")
print(error)
def set_camera_params(self):
self.camera.resolution = CAMERA['resolution']
self.camera.framerate = CAMERA['framerate']
self.camera.brightness = CAMERA['brightness']
self.camera.saturation = CAMERA['saturation']
self.camera.contrast = CAMERA['contrast']
self.camera.hflip = True
self.camera.vflip = True
time.sleep(5)
def capture_continuous(self, filename):
print("capture continuos")
try:
self.count = 1
# Use the video-port for captures...
for foo in self.camera.capture_continuous(self.stream,
'jpeg',
use_video_port=True):
print("New frame: #", self.count)
self.stream.seek(0)
self.image_queue.put(
Image(source=self.stream.read(),
file_name=filename,
timestamp=str(datetime.now())))
if self.recording_stop:
break
self.stream.seek(0)
self.stream.truncate()
self.count += 1
finally:
self.stream.seek(0)
self.stream.truncate()
def generate_videos_iterator(self):
print("generate video iterator")
count = 1
while not self.recording_stop or not self.image_queue.empty():
try:
yield self.image_queue.get(block=True, timeout=1)
self.image_queue.task_done()
print "sent", count, "of", self.count, "captured"
count += 1
except Empty as ex:
print("No data in image queue")
print("Done generating images")
def start_recording(self, filename):
print("Start recording")
threading.Thread(target=self.capture_continuous,
args=(filename, )).start()
videos_iterator = self.generate_videos_iterator()
response = self.grpc_stub.processVideo(videos_iterator)
print(response)
def record(self):
filename = CONFIGS["session"]
self.recording_stop = False
threading.Thread(target=self.start_recording,
args=(filename, )).start()
def stop_record(self):
self.recording_stop = True
self.image_queue.join()
def clean(self):
self.camera.close()
def convert_to_mp4(self):
filename_mp4 = self.filename.split(".")[0] + ".mp4"
print("file .h264 saved.. Transforming to mp4...")
os.system("MP4Box -fps 30 -add " + self.filename + " " + filename_mp4)
print("File converted to mp4")
#print "Starting thread [" + str(i)+"]\n"
time.sleep(5)
while True:
proxy = q.get() # get proxy
proxy_check(proxy)
q.task_done()
if (len(sys.argv) < 4):
print(
"\n\nUsage: programe.py proxy_list.txt output_proxies.txt LMH \n\nType\n L = Low\n M = Med\n H = High\n"
)
else:
num_threads = 300
q_queue = Queue()
for i in range(num_threads): # number of threads to spin up
worker = Thread(target=get_proxy, args=(
i,
q_queue,
))
worker.setDaemon(True)
worker.start()
with open(sys.argv[1]) as fp:
for line in fp:
q_queue.put(line) # put list in queue
q_queue.join() # wehen queue is empty exit
print 'Finished'
class AsyncCamera:
def __init__(self, driver):
self._driver = driver
# NOTE: command queue is not necessary anymore, PTZ commands
# create a new thread, login/out could do the same
self._cmd_queue = Queue()
self._cmd_thread = Thread(target=self._control_fn)
self._cmd_thread.daemon = True
self._cmd_thread.start()
self._cmd_queue.put((self._driver.login, None))
self._cmd_queue.join() # will block creator
try:
self._pan, self._tilt, self._zoom = self._driver.getptz()
except:
self._pan, self._tilt, self._zoom = 0, 0, 0
self._kick_thread = Thread(target=self._kick_fn)
self._kick_thread.daemon = True
self._kick_thread.start()
self._jpg_queue = Queue()
self._is_previewing = False
self._preview_thread = Thread(target=self._preview_fn)
self._preview_thread.daemon = True
self._preview_thread.start()
def _control_fn(self):
'''Receives commands through queue and executes them in sequence'''
while True:
# TODO: use variable number of arguments
fn, args = self._cmd_queue.get()
if args is None:
fn()
else:
fn(args[0], args[1])
self._cmd_queue.task_done()
def _kick_fn(self):
'''Keeps kicking the camera to keep session alive'''
while True:
sleep(10)
if True != self._driver.kick():
self._driver.logout()
self._driver.login()
def _preview_fn(self):
'''Keeps fetching preview images.
Fetches JPEG images one-by-one, as fast as possible.'''
while True:
if not self._is_previewing:
# reduce CPU load by avoiding busy looping
sleep(.5)
continue
jpg = self._driver.getjpg()
if bool(jpg):
logging.info('preview: {0} bytes'.format(len(jpg)))
self._jpg_queue.put(jpg)
def close(self):
self._cmd_queue.put((self._driver.logout, None))
self._cmd_queue.join()
def pantilt(self, pan=0, tilt=0):
'''Pan and tilt camera to the given angles (degrees)'''
self._pan, self._tilt = pan, tilt
# Create a thread for the potentially long-running command.
# This means commands can run parallel (out-of-order).
# If this is not desirable (not sure?) use the command queue.
Thread(target=self._driver.pantilt, args=(pan, tilt)).start()
def zoom(self, zoom=10, speed=1):
'''Set zoom level using given speed'''
self._zoom = zoom
# Create a thread for the potentially long-running command.
# This means commands can run parallel (out-of-order).
# If this is not desirable (not sure?) use the command queue.
Thread(target=self._driver.zoom, args=(zoom, speed)).start()
def getjpg(self):
'''
Return the most recent image received by the preview thread.
'''
jpg = None
while not self._jpg_queue.empty():
jpg = self._jpg_queue.get()
return jpg
def ptz(self):
return self._pan, self._tilt, self._zoom
def start_preview(self):
self._is_previewing = True
def stop_preview(self):
self._is_previewing = False
# TODO: use decorator
def start_recording(self):
Thread(target=self._driver.start_recording).start()
def stop_recording(self):
Thread(target=self._driver.stop_recording).start()
class ThreadTool:
def __init__(self, isThread=1, needfinishqueue=0, deamon=True):
self.isThread = isThread
self.idletask = {}
self.Threads = []
self.alivenum = 0
self.needfinishqueue = needfinishqueue
self.running = 0
self.threads_num = 10
self.deamon = deamon
self.job = None
self.default_object = None
if self.isThread == 1:
# self.lock = Lock() #线程锁
self.lock = threading.Lock()
self.q_request = Queue() #任务队列
if needfinishqueue > 0:
self.q_finish = Queue() #完成队列
elif self.isThread == 0:
self.lock = multiprocessing.Lock()
self.q_request = multiprocessing.Queue()
if self.needfinishqueue > 0:
self.q_finish = multiprocessing.Queue()
else:
from gevent.queue import JoinableQueue as geventqueue
from gevent.lock import Semaphore
self.lock = Semaphore()
self.q_request = geventqueue()
if self.needfinishqueue > 0:
self.q_finish = geventqueue()
def __del__(self): #解构时需等待两个队列完成
time.sleep(0.5)
if self.isThread == 1 or self.isThread == 2:
self.q_request.join()
if self.needfinishqueue > 0:
self.q_finish.join()
def getqueue_size(self):
return self.q_request.qsize()
def set_Thread_size(self, threads_num=10):
self.threads_num = threads_num
def init_add(self, add_init_object):
self.default_object = add_init_object
def add_task(self, job):
self.job = job
#获取当前剩余的任务，用于集群操做
def get_work(self):
tmparray = []
if self.q_request.qsize() > 0:
try:
req = self.q_request.get(block=True, timeout=4)
tmparray.append(req)
return tmparray
except:
return tmparray
else:
return tmparray
def start(self):
sizenumber = min(self.threads_num, self.q_request.qsize())
if self.isThread == 1:
for i in range(sizenumber):
t = Thread(target=self.getTask)
print '线程' + str(i + 1) + ' 正在启动'
t.setDaemon(self.deamon)
t.start()
self.Threads.append(t)
with self.lock:
self.alivenum += 1
elif self.isThread == 0:
for i in range(sizenumber):
t = multiprocessing.Process(target=self.getTaskProcess)
print '进程' + str(i + 1) + ' 正在启动'
t.Daemon = self.deamon
t.start()
self.Threads.append(t)
with self.lock:
self.alivenum += 1
else:
for i in range(sizenumber):
t = gevent.spawn(self.getgeventTask)
print '协程' + str(i + 1) + ' 正在启动'
self.Threads.append(t)
with self.lock:
self.alivenum += 1
def get_running_size(self):
return self.running
def taskleft(self):
if self.needfinishqueue > 0:
return self.q_request.qsize() + self.q_finish.qsize(
) + self.running
else:
return self.q_request.qsize() + self.running
def push(self, req):
sizenum = len(req)
for urls in req:
self.q_request.put(urls)
threadnownum = 0
threaddie = []
dienum = 0
if self.isThread == 1:
tempnumb = 0
with self.lock:
tempnumb = self.alivenum
if tempnumb < self.threads_num:
for item in self.Threads:
if item.isAlive():
threadnownum = threadnownum + 1
with self.lock:
print str(threadnownum) + '活着的线程数'
self.Threads = filter(lambda x: x.isAlive() != False,
self.Threads)
print str(len(self.Threads)) + '清理后活着的进程数'
else:
threadnownum = self.threads_num
elif self.isThread == 0:
tempnumb = 0
with self.lock:
tempnumb = self.alivenum
if tempnumb < self.threads_num:
for item in self.Threads:
if item.is_alive():
threadnownum = threadnownum + 1
with self.lock:
print str(threadnownum) + '活着的进程数'
self.Threads = filter(lambda x: x.is_alive() != False,
self.Threads)
print str(len(self.Threads)) + '清理后活着的进程数'
else:
threadnownum = self.threads_num
sizenumber = min(self.threads_num - threadnownum, sizenum)
if self.isThread == 1:
for i in range(sizenumber):
t = Thread(target=self.getTask)
t.Daemon = self.deamon
t.start()
self.Threads.append(t)
with self.lock:
self.alivenum += 1
elif self.isThread == 0:
for i in range(sizenumber):
t = multiprocessing.Process(target=self.getTaskProcess)
t.Daemon = self.deamon
t.start()
self.Threads.append(t)
with self.lock:
self.alivenum += 1
else:
for i in range(self.threads_num):
print 'alive num', self.alivenum, self.threads_num
if self.alivenum < self.threads_num:
t = gevent.spawn(self.getgeventTask)
print '协程' + str(self.alivenum) + ' 正在启动'
self.Threads.append(t)
with self.lock:
self.alivenum += 1
else:
break
self.q_request.join()
def pop(self):
return self.q_finish.get()
def do_job(self, job, req, threadname):
return job(req, threadname)
def getTaskProcess(self):
while True:
# if self.taskleft()>0:
# try:
# req = self.q_request.get(block=True,timeout=10000)
# except:
# continue
# else:
# threadname=multiprocessing.current_process().name
# print threadname+'关闭'
# with self.lock:
# self.alivenum-=1
# break
req = self.q_request.get()
with self.lock: #要保证该操作的原子性，进入critical area
self.running = self.running + 1
threadname = multiprocessing.current_process().name
print '进程' + threadname + '发起请求: '
ans = self.do_job(self.job, req, threadname)
# ans = self.connectpool.getConnect(req)
# self.lock.release()
if self.needfinishqueue > 0:
self.q_finish.put((req, ans))
# self.lock.acquire()
with self.lock:
self.running = self.running - 1
threadname = multiprocessing.current_process().name
print '进程' + threadname + '完成请求'
# self.lock.release()
#self.q_request.task_done()
def getTask(self):
while True:
# if self.taskleft()>0:
# try:
# req = self.q_request.get(block=True,timeout=10000)
# except:
# continue
# else:
# threadname=threading.currentThread().getName()
# with self.lock:
# self.alivenum-=1
# print threadname+'关闭'
# break
req = self.q_request.get()
with self.lock: #要保证该操作的原子性，进入critical area
self.running = self.running + 1
threadname = threading.currentThread().getName()
print '线程' + threadname + '发起请求: '
ans = self.do_job(self.job, req, threadname)
# ans = self.connectpool.getConnect(req)
# self.lock.release()
if self.needfinishqueue > 0:
self.q_finish.put((req, ans))
# self.lock.acquire()
with self.lock:
self.running -= 1
threadname = threading.currentThread().getName()
print '线程' + threadname + '完成请求'
# self.lock.release()
self.q_request.task_done()
def getgeventTask(self):
while True:
# if self.taskleft()>0:
# try:
# req = self.q_request.get(block=True,timeout=10000)
# except:
# continue
# else:
# threadname=threading.currentThread().getName()
# with self.lock:
# self.alivenum-=1
# print threadname+'关闭'
# break
req = self.q_request.get()
with self.lock: #要保证该操作的原子性，进入critical area
self.running = self.running + 1
threadname = gevent.getcurrent()
print threadname, '协程发起请求: '
ans = self.do_job(self.job, req, threadname)
# ans = self.connectpool.getConnect(req)
# self.lock.release()
if self.needfinishqueue > 0:
self.q_finish.put((req, ans))
# self.lock.acquire()
with self.lock:
self.running -= 1
threadname = gevent.getcurrent()
print threadname, '协程发起请求: '
# self.lock.release()
self.q_request.task_done()
def main():
start_time = time.time()
#Read in arguments
parser = argparse.ArgumentParser(description='Chunk a file and then kick'
' off multiple SCP threads.'
'Speeds up transfers over '
'high latency links')
parser.add_argument('-c',
'--cypher',
help='cypher use with from transfer see: ssh',
default=default_cypher,
required=False)
parser.add_argument('-s',
'--size',
help='size of chunks to transfer.',
default='500M',
required=False,
type=human_sizes)
parser.add_argument('-r',
'--retries',
help='number of times to retry transfer.',
default=default_retries,
required=False,
type=int)
parser.add_argument('-t',
'--threads',
help='number of threads (default ' +
str(default_num_threads) + ')',
default=default_num_threads,
required=False,
type=int)
parser.add_argument('src', help='source file')
parser.add_argument('srv',
help='remote server and user if required'
' e.g [email protected]')
parser.add_argument('dst',
help='directory (if remote home dir then specify . )')
args = parser.parse_args()
ssh_crypto = args.cypher
try:
chunk_size = human2bytes(args.size)
except ValueError as e:
print 'Invalid chunk size ' + str(e)
exit(1)
num_threads = args.threads
src_file = args.src
dst_file = args.dst
remote_server = args.srv
retries = args.retries
(dest_path, _) = os.path.split(dst_file)
(_, src_filename) = os.path.split(src_file)
remote_dest_file = os.path.join(dest_path, src_filename)
remote_chunk_files = []
# Check args for errors + instantiate variables.
if not os.path.exists(src_file):
print 'Error: Source file does not exist', src_file
exit(1)
if not os.path.isfile(src_file):
print 'Error: Source is not a file', src_file
exit(1)
src_file_size = os.stat(src_file).st_size
# Split file and calc the file md5
local_chunk_start_time = time.time()
print "spliting file"
spinner = spinning_cursor()
sys.stdout.write(spinner.next())
sys.stdout.flush()
sys.stdout.write('\b')
(src_file_info, chunk_infos) = split_file_and_md5(src_file, src_filename,
chunk_size)
src_file_md5 = src_file_info[1]
local_chunk_end_time = time.time()
print "uploading MD5 ({0!s}) checksum to remote site".format(src_file_md5)
try:
checksum_filename = src_file + '.md5'
dest_checksum_filename = os.path.join(dest_path, src_filename + '.md5')
with open(checksum_filename, 'w+') as checksum_file:
checksum_file.write(src_file_md5 + ' ' + src_filename)
print 'copying ' + src_file + ' to ' + dest_checksum_filename
subprocess.check_call(['scp', '-c' + ssh_crypto, '-q',
'-oBatchMode=yes', checksum_filename,
remote_server + ':' + \
dest_checksum_filename])
except CalledProcessError as e:
print(e.returncode)
print "ERROR: Couldn't connect to remote server."
exit(1)
# Fill the queue of files to transfer
q = Queue()
chunk_num = 1
total_chunks = len(chunk_infos)
for (src_chunk_filename, chunk_md5) in chunk_infos:
# create destination path
(_, src_filename) = os.path.split(src_chunk_filename)
dest_chunk_filename = os.path.join(dest_path, src_filename)
remote_chunk_files.append(
(src_chunk_filename, dest_chunk_filename, chunk_md5))
q.put((src_chunk_filename, dest_chunk_filename, chunk_num,
total_chunks, retries))
chunk_num = chunk_num + 1
# Kick off threads
transfer_start_time = time.time()
print "starting transfers"
for i in range(num_threads):
t = WorkerThread(q, dst_file, remote_server, ssh_crypto)
t.daemon = True
t.start()
q.join()
transfer_end_time = time.time()
#join the chunks back together and check the md5
print "re-assembling file at remote end"
remote_chunk_start_time = time.time()
chunk_count = 0
for (chunk_filename, chunk_md5) in chunk_infos:
(path, remote_chunk_filename) = os.path.split(chunk_filename)
remote_chunk_file = os.path.join(dest_path, remote_chunk_filename)
spin('processing ' + remote_chunk_filename)
if chunk_count:
cmd = remote_chunk_file + '>> ' + remote_dest_file
else:
#truncate if the first chunk
cmd = remote_chunk_file + '> ' + remote_dest_file
subprocess.call(['ssh', remote_server, 'cat', cmd])
chunk_count += 1
print
print 're-assembled'
remote_chunk_end_time = time.time()
print "checking remote file checksum"
remote_checksum_start_time = time.time()
try:
# use openssl to be cross platform (OSX,Linux)
checksum = subprocess.check_output(
['ssh', remote_server, 'openssl', 'md5', remote_dest_file])
# MD5(2GB.mov)= d8ce4123aaacaec671a854f6ec74d8c0
if checksum.find(src_file_md5) != -1:
print 'PASSED checksums match'
else:
print 'ERROR: MD5s do not match local(' + src_file_md5 + \
') != (' + checksum.strip() + ')'
print ' File uploaded with errors - MD5 did not match.'
print ' local and remote chunks not cleared up'
exit(1)
except CalledProcessError as e:
print(e.returncode)
print 'ERROR: File uploaded with errors - MD5 did not match.'
print ' local and remote chunks not cleared up'
exit(1)
remote_checksum_end_time = time.time()
# clean up
print "cleaning up"
print "removing file chunks"
for (local_chunk, remote_chunk, chunk_md5) in remote_chunk_files:
spin("removing file chunk " + local_chunk)
os.remove(local_chunk)
try:
subprocess.call(['ssh', remote_server, 'rm', remote_chunk])
except CalledProcessError as e:
print(e.returncode)
print 'ERROR: failed to remove remote chunk ' + remote_chunk
print ''
print "transfer complete"
end_time = time.time()
print "-" * 80
print "file size :" + bytes2human(src_file_size) + "B"
print "transfer rate :" + bytes2human(src_file_size /
int(transfer_end_time - \
transfer_start_time)) + "B/s"
print " :" + bytes2human((src_file_size * 8) /
int(transfer_end_time - \
transfer_start_time)) + "b/s"
print "transfer time :" + str(humanize_time_to_string( \
humanize_time( \
int(transfer_end_time - \
transfer_start_time),
"seconds")))
print "local chunking time :" + str(humanize_time_to_string( \
humanize_time( \
int(local_chunk_end_time - \
local_chunk_start_time),
"seconds")))
print "remote reassembly time :" + str(humanize_time_to_string( \
humanize_time( \
int(remote_chunk_end_time - \
remote_chunk_start_time),
"seconds")))
print "remote checksum time :" + str(humanize_time_to_string(
humanize_time( \
int(remote_checksum_end_time - \
remote_checksum_start_time), \
"seconds")))
print "total transfer rate :" + bytes2human(src_file_size / \
int(end_time - \
start_time)) + "B/s"
print " :" + bytes2human((src_file_size * 8) / \
int(end_time - \
start_time)) + "b/s"
print "total time :" + str(humanize_time_to_string( \
humanize_time(int(end_time - \
start_time), "seconds")))
exit(0)
class GeoGroupTable:
'''Group OSM nodes by their geographical coordinates.
The coordinates of the globe are partitioned into disjoint areas.
Each partition is named by the geohash code of its (n,w) corner.
Grouping of nodes is implemented by restricting the length of
the geohash codes used.
'''
def __init__(self, config, options, db):
'''Initialize the table.
Keyword arguments:
config - A ConfigParser instance.
options - An optparse.OptionParser structure.
db - A DB object supporting 'get()' and 'store()'
methods.
'''
self.geodb = collections.defaultdict(NodeGroup)
self.db = db
lrusize = config.getint(C.DATASTORE, C.GEODOC_LRU_SIZE)
self.lru = BoundedLRUBuffer(bound=lrusize, callback=self._cb)
if options.nothreading:
nthreads = 0
else:
nthreads = config.getint(C.DATASTORE, C.GEODOC_LRU_THREADS)
self.nthreads = max(0, nthreads)
if self.nthreads:
self.wrthreads = []
self.wrqueue = Queue(self.nthreads)
self.wrcond = threading.Condition()
self.wrpending = []
for n in range(self.nthreads):
t = threading.Thread(target=self._worker)
t.name = "GeoWB-%d" % n
t.daemon = True
self.wrthreads.append(t)
t.start()
db.register_threads(self.wrthreads)
def _cb(self, key, value):
"Callback called when an LRU item is ejected."
nodeset = self.geodb.pop(key)
if self.nthreads: # Defer processing to a worker thread.
self.wrqueue.put((key, nodeset))
else: # Synchronous operation.
self._write_geodoc(key, nodeset)
def _worker(self):
"Helper method, used by worker threads."
while True:
# Retrieve a work item.
v = self.wrqueue.get()
if v is None: # Exit the thread.
self.wrqueue.task_done()
return
# Unpack the work item.
key, nodeset = v
# Mark the item as "I/O in progress".
with self.wrcond:
while key in self.wrpending:
self.wrcond.wait()
assert key not in self.wrpending
self.wrpending.append(key)
# Process this node set.
self._write_geodoc(key, nodeset)
# Remove the "I/O in progress" marker.
with self.wrcond:
assert key in self.wrpending
self.wrpending.remove(key)
self.wrcond.notifyAll()
self.wrqueue.task_done()
def _write_geodoc(self, key, nodegroup):
"Merge in a group of nodes into a geodoc."
assert isinstance(nodegroup, NodeGroup)
geodoc = self.db.retrieve_element(C.GEODOC, key)
if geodoc is None: # New document.
geodoc = new_osm_element(C.GEODOC, key)
nodegroup.update(geodoc[C.NODES])
geodoc[C.NODES] = nodegroup.aslist()
self.db.store_element(C.GEODOC, key, geodoc)
def add(self, elem):
'''Add information about a node 'elem' to the geo table.
Usage:
>>> gt = GeoGroupTable()
>>> gt = gt.add(elem)
The node 'elem' should have a 'lat' and 'lon' fields that
encode its latitude and longitude respectively. The 'id'
field specifies the node's "id".
'''
assert elem.namespace == C.NODE, "elem is not a node: %s" % str(elem)
# Determine the geo-key for the node.
ghkey = geohash_key_for_element(elem)
# Retrieve the partition covering this location.
ghdoc = self.geodb[ghkey]
elemid = elem.id
if elemid not in ghdoc:
ghdoc.add(elem)
self.lru[ghkey] = ghdoc
def flush(self):
"Wait pending I/Os"
# Flush items from the LRU.
self.lru.flush()
if self.nthreads:
# Wait for the work queue to drain.
self.wrqueue.join()
def recdata(name):
if name == "recipelist":
foodtype = request.query.type
recipeCount = request.query.number
#print recipeCount
foodtype.replace ("%20", "+")
apicall = "https://spoonacular-recipe-food-nutrition-v1.p.mashape.com/recipes/searchComplex?addRecipeInformation=true&cuisine=mexican&diet=vegetarian&excludeIngredients=onions%2C+garlic&fillIngredients=true&includeIngredients=tomato%2C+cilantro%2C+pepper%2C+carrot%2C+Coriander%2C+Cumin%2C+Beans%2C+potatoes%2C+capsicum&intolerances=egg%2C+seafood%2C+shellfish&limitLicense=false&maxCalories=1500&maxCarbs=100&maxFat=100&maxProtein=100&minCalories=150&minCarbs=5&minFat=5&minProtein=5&number="+recipeCount+"&offset=0&query=<required>&ranking=1&type="+foodtype
response = unirest.get(apicall,
headers={
"X-Mashape-Key": "FM6rhmzOJ1mshE29nA3PzJNQSNjup1cziiQjsnXdRgxLo859D4",
"Accept": "application/json"
}
)
#print response.body
with io.open('recjsonone.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(response.body,ensure_ascii=False)))
with open('recjsonone.json') as data_file:
data = json.load(data_file)
counter = 0;
# Create a queue to communicate with the worker threads
queue = Queue()
# Create 10 worker threads
for x in range(10):
worker = RecipeRunner(queue)
# Setting daemon to True will let the main thread exit even though the workers are blocking
worker.daemon = True
worker.start()
for val in data:
if val == 'results':
listrecm = []
for recipe in data[val]:
if len(recipe) <=13:
continue
queue.put((recipe,listrecm,recipe['pricePerServing']))
#recipelistgen(recipe,listrecm,recipe['pricePerServing'])
queue.join()
with io.open('recresponse.json', 'w', encoding='utf-8') as f:
f.write(unicode(json.dumps(listrecm,ensure_ascii=False)))
for item in listrecm:
del item['cookingInstructions']
return dict(data=listrecm)
elif name == "cookingsteps":
with open('recresponse.json') as data_file:
data = json.load(data_file)
recipeid = request.query.id
listrecm = []
for item in data:
if str(item['id']).strip() != str(recipeid).strip():
continue
listrecm = item
return dict(data=listrecm)
class AsyncRequest:
MIN_THREAD = 50
RETRY = 1
def __init__(self, request=None, retry=1, thread=50):
self.request = request or Request()
self.RETRY = retry
self.MIN_THREAD = thread
def __create_queue(self, urls):
print("*********************** Start Queue %d" % len(urls))
self.length = len(urls)
self.q = Queue(maxsize=self.length)
self.num_theads = min(self.MIN_THREAD, self.length)
self.dialog = xbmcgui.DialogProgress()
self.dialog.create('Get URL', "Loading 0/%d urls" % self.length)
self.results = [{} for x in urls]
for i in range(len(urls)):
self.q.put((i, urls[i]))
def __start_thread(self, args):
for i in range(self.num_theads):
worker = Thread(target=self.__request, args=args)
worker.setDaemon(True)
worker.start()
self.q.join()
print("*********************** All %s thread done" % self.length)
self.dialog.close()
def __request(self,
action,
params=None,
headers=None,
json=None,
redirect=False,
parser=None,
args=None):
while not self.q.empty():
work = self.q.get()
url = work[1]
if type(url) is dict:
params = 'params' in url and url['params'] or params
headers = 'headers' in url and url['headers'] or headers
redirect = 'redirect' in url and url['redirect'] or redirect
parser = 'parser' in url and url['parser'] or parser
args = 'args' in url and url['args'] or args
json = 'json' in url and url['json'] or json
required_response_header = 'responseHeader' in url and True or False
url = work[1]['url']
retry = self.RETRY
while retry > 0:
try:
if action is 'head':
data = self.request.head(url,
params=params,
headers=headers,
redirect=redirect)
if action is 'get':
data = self.request.get(url,
params=params,
headers=headers)
if action is 'post':
data = self.request.post(url,
params=params,
headers=headers,
json=json)
if parser:
if required_response_header:
response_headers = self.request.get_request(
).headers
data = parser(data, args, response_headers)
else:
data = parser(data, args)
# print('Requested %s' % work[1])
self.results[work[0]] = data
retry = 0
except:
print('Request %s fail retry %d' % (work[1], retry))
self.results[work[0]] = {}
finally:
retry -= 1
done = self.q.qsize()
progress = 100 - (done * 100 / self.length)
self.dialog.update(
progress,
'Processing %d/%d urls' % (self.length - done, self.length))
self.q.task_done()
return True
def head(self,
urls,
params=None,
headers=None,
redirect=False,
parser=None,
args=None):
self.__create_queue(urls)
self.__start_thread(('head', params, headers, redirect, parser, args))
return self.results
def get(self,
urls,
headers=None,
params=None,
redirect=False,
parser=None,
args=None):
self.__create_queue(urls)
self.__start_thread(('get', params, headers, redirect, parser, args))
return self.results
def post(self,
urls,
params=None,
headers=None,
json=None,
redirect=False,
parser=None,
args=None):
self.__create_queue(urls)
self.__start_thread(
('post', params, headers, json, redirect, parser, args))
return self.results
class Pool:
"""Pool of threads consuming tasks from a queue"""
def __init__(self,
thread_count,
batch_mode=True,
exception_handler=default_handler):
# batch mode means block when adding tasks if no threads available to process
self.queue = Queue(thread_count if batch_mode else 0)
self.resultQueue = Queue(0)
self.thread_count = thread_count
self.exception_handler = exception_handler
self.aborts = []
self.idles = []
self.threads = []
def __del__(self):
"""Tell my threads to quit"""
self.abort()
def run(self, block=False):
"""Start the threads, or restart them if you've aborted"""
# either wait for them to finish or return false if some arent
if block:
while self.alive():
sleep(1)
elif self.alive():
return False
# go start them
self.aborts = []
self.idles = []
self.threads = []
for n in range(self.thread_count):
abort = Event()
idle = Event()
self.aborts.append(abort)
self.idles.append(idle)
self.threads.append(
Worker('thread-%d' % n, self.queue, self.resultQueue, abort,
idle, self.exception_handler))
return True
def enqueue(self, func, *args, **kargs):
"""Add a task to the queue"""
self.queue.put((func, args, kargs))
def join(self):
"""Wait for completion of all the tasks in the queue"""
self.queue.join()
def abort(self, block=False):
"""Tell each worker that its done working"""
# tell the threads to stop after they are done with what they are currently doing
for a in self.aborts:
a.set()
# wait for them to finish if requested
while block and self.alive():
sleep(1)
def alive(self):
"""Returns True if any threads are currently running"""
return True in [t.is_alive() for t in self.threads]
def idle(self):
"""Returns True if all threads are waiting for work"""
return False not in [i.is_set() for i in self.idles]
def done(self):
"""Returns True if not tasks are left to be completed"""
return self.queue.empty()
def results(self, sleep_time=0):
"""Get the set of results that have been processed, repeatedly call until done"""
sleep(sleep_time)
results = []
try:
while True:
# get a result, raises empty exception immediately if none available
results.append(self.resultQueue.get(False))
self.resultQueue.task_done()
except:
return results
return results
class ThreadPool(object):
""" Pool of threads consuming tasks from a queue """
def __init__(self,
num_threads,
worker_class=None,
max_queue_size=0,
**kwargs):
# task queue
self.tasks = Queue(max_queue_size)
# flags
self._operating = True
# params
self.trace_logs = kwargs.pop('trace_logs', False)
self.name = kwargs.pop('name', id(self))
# counters
self._total_task_count = 0
self._tasks_ok_count = 0
self._tasks_nok_count = 0
self.worker_counters = {}
# workers
self._workers = {}
self._worker_class = worker_class or Worker
for i in range(num_threads):
self._add_worker(i)
def _add_worker(self, worker_id, raise_on_id_clash=False):
"""add a worker to the pool"""
if worker_id in self._workers:
if raise_on_id_clash:
raise Exception('worker with that id already exists',
worker_id)
worker = self._worker_class(worker_id=worker_id,
task_queue=self.tasks,
parent_pool=self)
self._workers[worker_id] = worker
self.worker_counters[worker_id] = 0
@property
def identification(self):
return '{}({})'.format(self.__class__.__name__, self.name)
def worker_notification(self, worker_id, log_func, message, **kwargs):
msg = '{} {}({}) {}: {}'.format(
self.identification, self._worker_class.__name__, worker_id,
message, ' '.join(
sorted(utils.convert_dict_params_to_list_of_string(kwargs))))
log_func(msg)
@property
def operating(self):
return self._operating
@property
def count_completed(self):
"""gets the current count of all completed tasks"""
return sum(self.worker_counters.values())
@property
def count_remaining(self):
"""gets the current count of all remaining tasks"""
return self.count_total - self.count_completed
@property
def count_total(self):
"""gets the total count of all tasks"""
return self._total_task_count
@property
def count_ok(self):
"""gets the total count of all tasks completed okay"""
return self._tasks_ok_count
@property
def count_nok(self):
"""gets the total count of all tasks completed not okay"""
return self._tasks_nok_count
@property
def all_workers_started(self):
"""gets if all workers have started working"""
return all(self.worker_counters.values())
@property
def any_workers_started(self):
"""gets if any workers have started working"""
return any(self.worker_counters.values())
def _increment_worker_count(self, worker_id):
"""count the number of tasks each worker completes"""
self.worker_counters[worker_id] += 1
def task_ok(self, worker_id, func, args, kwargs):
"""count the number of tasks completed okay"""
self._increment_worker_count(worker_id)
self._tasks_ok_count += 1
def task_nok(self, worker_id, func, args, kwargs):
"""count the number of tasks completed not okay"""
self._increment_worker_count(worker_id)
self._tasks_nok_count += 1
def add_task(self, func, *args, **kwargs):
""" Add a task to the queue """
self.tasks.put((func, args, kwargs))
self._total_task_count += 1
def map(self, func, args_list):
""" Add a list of tasks to the queue """
# Add the jobs in bulk to the thread pool. Alternatively you could use
# `add_task` to add single jobs. The code will block here, which
# makes it possible to cancel the thread pool with an exception when
# the currently running batch of workers is finished.
for args in args_list:
self.add_task(func, args)
def stop(self):
"""stops the operation of this thread pool"""
self._operating = False
@property
def finished(self):
"""are all tasks complete (based on count)"""
return bool(self.count_remaining == 0)
@property
def processing(self):
"""are tasks still being processed"""
return not self.finished
def wait_completion(self):
""" Wait for completion of all the tasks in the queue BLOCKING """
self.tasks.join()
def notify_progress(self):
"""sends the current progress status to the log"""
log.info('{} progress: ran={}/{} ok={} nok={}'.format(
self.identification, self.count_completed, self.count_total,
self.count_ok, self.count_nok))
def wait_with_progress(self, period=30, timeout=None):
log.info('{} waiting with progress: threads={} tasks={}'.format(
self.identification, len(self._workers), self.count_total))
while self.processing:
self.notify_progress()
time.sleep(period)
log.info('{} finished: ran={}/{} ok={} nok={}'.format(
self.identification, self.count_completed, self.count_total,
self.count_ok, self.count_nok))
def render_tiles(bbox, mapfile, tile_dir, minZoom=1, maxZoom=18, name="unknown", num_threads=NUM_THREADS,
tms_scheme=False):
print("{} Generating tiles for {} bbox zoom levels [ {} - {} ] using {} mapnik mapfile".format(
plus, bbox, minZoom, maxZoom, mapfile))
# Launch rendering threads
queue = Queue(32)
printLock = threading.Lock()
renderers = {}
for i in range(num_threads):
renderer = RenderThread(tile_dir, mapfile, queue, printLock, maxZoom)
render_thread = threading.Thread(target=renderer.loop)
render_thread.start()
print("{} Started render thread {}".format(plus, render_thread.getName()))
renderers[i] = render_thread
if not os.path.isdir(tile_dir):
os.mkdir(tile_dir)
gprj = GoogleProjection(maxZoom + 1)
ll0 = (bbox[0], bbox[3])
ll1 = (bbox[2], bbox[1])
for z in range(minZoom, maxZoom + 1):
px0 = gprj.fromLLtoPixel(ll0, z)
px1 = gprj.fromLLtoPixel(ll1, z)
# check if we have directories in place
zoom = "%s" % z
if not os.path.isdir(tile_dir + zoom):
os.mkdir(tile_dir + zoom)
for x in range(int(px0[0] / 256.0), int(px1[0] / 256.0) + 1):
# Validate x co-ordinate
if (x < 0) or (x >= 2 ** z):
continue
# check if we have directories in place
str_x = "%s" % x
if not os.path.isdir(tile_dir + zoom + '/' + str_x):
os.mkdir(tile_dir + zoom + '/' + str_x)
for y in range(int(px0[1] / 256.0), int(px1[1] / 256.0) + 1):
# Validate x co-ordinate
if (y < 0) or (y >= 2 ** z):
continue
# flip y to match OSGEO TMS spec
if tms_scheme:
str_y = "%s" % ((2 ** z - 1) - y)
else:
str_y = "%s" % y
tile_uri = tile_dir + zoom + '/' + str_x + '/' + str_y + '.png'
# Submit tile to be rendered into the queue
t = (name, tile_uri, x, y, z)
try:
queue.put(t)
except KeyboardInterrupt:
raise SystemExit("Ctrl-c detected, exiting...")
# Signal render threads to exit by sending empty request to queue
for i in range(num_threads):
queue.put(None)
# wait for pending rendering jobs to complete
queue.join()
for i in range(num_threads):
renderers[i].join()
class Worker(object):
def __init__(self,
work_dir,
file_filter,
input_service,
output_service,
threads_num=5,
max_size=30):
self._input_service = input_service
self._output_service = output_service
self._filter = file_filter
self._work_dir = work_dir
self._threads_num = threads_num
self._threads_pool = []
self._queue = Queue(maxsize=max_size)
self._stop = False
self._succ = 0
self._fail = 0
self._lock = Lock()
def __work_thread(self):
while not self._stop:
# logger.info("worker stop: " + str(self._stop))
try:
# logger.debug("try to get task")
task = self._queue.get_nowait()
# logger.debug("get task succeefully")
self._queue.task_done()
except Empty:
logger.debug("Empty queue" + str(self._stop))
if self._stop:
break
else:
import time
time.sleep(1)
continue
task_path = task.key
if task_path.startswith('/'):
task_path = task_path[1:]
if isinstance(task_path, str):
task_path = task_path.decode('utf-8')
import uuid
localpath = unicode(path.join(self._work_dir, uuid.uuid1().hex))
if task.size is None:
logger.warn(
"{file_path} task doesn't contains size attribute".format(
file_path=task_path.encode('utf-8')))
self._fail += 1
fail_logger.error(task_path)
continue
try:
try:
makedirs(path.dirname(localpath))
except OSError as e:
# directory is exists
logger.debug(str(e))
try:
ret = self._input_service.exists(task)
if ret:
logger.info("{file_path} exists".format(
file_path=task_path.encode('utf-8')))
with self._lock:
self._succ += 1
self._filter.add(task_path)
continue
except Exception as e:
logger.exception("exists failed")
try:
self._output_service.download(task, localpath)
except Exception as e:
logger.exception("download failed")
self._fail += 1
fail_logger.error(task_path)
continue
try:
self._input_service.upload(task, localpath)
except Exception:
logger.exception("upload {} failed".format(
task_path.encode('utf-8')))
self._fail += 1
fail_logger.error(task_path)
continue
try:
import os
if isinstance(localpath, unicode):
localpath = localpath.encode('utf-8')
os.remove(localpath)
try:
os.removedirs(path.dirname(localpath))
except OSError:
pass
except Exception as e:
logger.exception(str(e))
continue
try:
ret = self._input_service.exists(task)
if ret:
if isinstance(task_path, unicode):
logger.info("inc succ with {}".format(
task_path.encode('utf-8')))
else:
logger.info("inc succ with {}".format(
task_path.encode('utf-8')))
with self._lock:
self._succ += 1
self._filter.add(task_path)
continue
else:
logger.exception(
"there is not expected object in service, you need to re-run this application again"
)
except Exception as e:
logger.exception("exists failed")
except Exception:
logger.exception("try except for deleting file")
finally:
self._filter.del_doing_task(task_path)
import os
if isinstance(localpath, unicode):
localpath = localpath.encode('utf-8')
try:
os.remove(localpath)
os.removedirs(path.dirname(localpath))
except OSError:
pass
def add_task(self, task):
# blocking
self._queue.put(task)
def start(self):
self._threads_pool = [
Thread(target=self.__work_thread) for _ in range(self._threads_num)
]
for t in self._threads_pool:
t.start()
def stop(self):
self._queue.join()
self.term()
def term(self):
self._stop = True
logger.info("try to stop migrate process.")
# while any([t.is_alive() for t in self._threads_pool]):
# map(lambda i: i.join(5), filter(lambda j: j.is_alive(), self._threads_pool))
# print filter(lambda j: j.is_alive(), self._threads_pool)
map(lambda i: i.join(), self._threads_pool)
@property
def success_num(self):
return self._succ
@property
def failure_num(self):
return self._fail
