def get_gif_url_list(timeout):
gif_url_list_q = Queue()
for i in range(1,6):
process_list = []
for j in range(1 + (i - 1) * 10,1 + i * 10):
process_list.append(Process(target=fetch_gif_url,args=(data,j,gif_url_list_q)))
for j in range(1 + (i - 1) * 10,1 + i * 10):
process_list[j - (i - 1) * 10 - 1].start()
# print '任务 {0} 开始...'.format(j)
time_sum = 0
while 1:
for process in process_list:
if not process.is_alive():
process_list.remove(process)
if len(process_list) == 0:
break
elif time_sum == timeout:
for process in process_list:
process.terminate()
print '{0} 假死被杀掉了...'.format(process.name)
process_list.remove(process)
break
else:
time_sum += 1
sleep(1)
print str(time_sum) + '秒'
continue
print (gif_url_list_q.qsize()),'个结果'
gif_url_list = []
for i in range(gif_url_list_q.qsize()):
gif_url_list.append(gif_url_list_q.get())
return gif_url_list
def parse_input_files( input_files, num_threads, genomes, min_coverage, min_proportion ):
from multiprocessing import Process, Queue
#from queue import Queue
from time import sleep
input_q = Queue()
output_q = Queue()
for input_file in input_files:
input_q.put( input_file )
if num_threads > input_q.qsize():
num_threads = input_q.qsize()
sleep( 1 )
thread_list = []
for current_thread in range( num_threads ):
input_q.put( None )
current_thread = Process( target=manage_input_thread, args=[ genomes.reference(), min_coverage, min_proportion, input_q, output_q ] )
current_thread.start()
#manage_input_thread( genomes.reference(), min_coverage, min_proportion, input_q, output_q )
thread_list.append( current_thread )
sleep( 1 )
while num_threads > 0:
new_genome = output_q.get()
if new_genome is None:
num_threads -= 1
elif isinstance( new_genome, str ):
genomes.add_failed_genome( new_genome )
else:
genomes.add_genome( new_genome )
sleep( 1 )
for current_thread in thread_list:
current_thread.join()
def test_transfert_queue():
t1 = "testTopic"
topic = Topics()
q = Queue()
topic.process(t1,123)
topic.process(t1,456)
topic.process(t1,789)
assert q.empty()
topic.transfer(t1,q)
assert q.qsize() > 0
assert q.get() == [0, 123]
assert q.get() == [1, 456]
assert q.get() == [2, 789]
topic.process(t1,111)
topic.process(t1,222)
assert q.qsize() > 0
assert q.get() == [3, 111]
assert q.get() == [4, 222]
def run(self):
logger.info('starting horizon agent')
listen_queue = Queue(maxsize=settings.MAX_QUEUE_SIZE)
pid = getpid()
# Start the workers
for i in range(settings.WORKER_PROCESSES):
Worker(listen_queue, pid).start()
# Start the listeners
Listen(settings.PICKLE_PORT, listen_queue, pid, type="pickle").start()
Listen(settings.UDP_PORT, listen_queue, pid, type="udp").start()
# Start the roomba
Roomba(pid).start()
# Warn the Mac users
try:
listen_queue.qsize()
except NotImplementedError:
logger.info('WARNING: Queue().qsize() not implemented on Unix platforms like Mac OS X. Queue size logging will be unavailable.')
# Keep yourself occupied, sucka
while 1:
time.sleep(100)
def run(self):
logger.info('starting horizon agent')
listen_queue = Queue(maxsize=settings.MAX_QUEUE_SIZE)
pid = getpid()
#If we're not using oculus, don't bother writing to mini
try:
skip_mini = True if settings.OCULUS_HOST == '' else False
except Exception:
skip_mini = True
# Start the workers
for i in range(settings.WORKER_PROCESSES):
if i == 0:
Worker(listen_queue, pid, skip_mini, canary=True).start()
else:
Worker(listen_queue, pid, skip_mini).start()
# Start the listeners
Listen(settings.PICKLE_PORT, listen_queue, pid, type="pickle").start()
Listen(settings.UDP_PORT, listen_queue, pid, type="udp").start()
# Start the roomba
Roomba(pid, skip_mini).start()
# Warn the Mac users
try:
listen_queue.qsize()
except NotImplementedError:
logger.info('WARNING: Queue().qsize() not implemented on Unix platforms like Mac OS X. Queue size logging will be unavailable.')
# Keep yourself occupied, sucka
while 1:
time.sleep(100)
def test_cached(broker):
broker.purge_queue()
broker.cache.clear()
group = 'cache_test'
# queue the tests
task_id = async('math.copysign', 1, -1, cached=True, broker=broker)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.copysign', 1, -1, cached=True, broker=broker, group=group)
async('math.popysign', 1, -1, cached=True, broker=broker, group=group)
iter_id = async_iter('math.floor', [i for i in range(10)], cached=True)
# test wait on cache
# test wait timeout
assert result(task_id, wait=10, cached=True) is None
assert fetch(task_id, wait=10, cached=True) is None
assert result_group(group, wait=10, cached=True) is None
assert result_group(group, count=2, wait=10, cached=True) is None
assert fetch_group(group, wait=10, cached=True) is None
assert fetch_group(group, count=2, wait=10, cached=True) is None
# run a single inline cluster
task_count = 17
assert broker.queue_size() == task_count
task_queue = Queue()
stop_event = Event()
stop_event.set()
for i in range(task_count):
pusher(task_queue, stop_event, broker=broker)
assert broker.queue_size() == 0
assert task_queue.qsize() == task_count
task_queue.put('STOP')
result_queue = Queue()
worker(task_queue, result_queue, Value('f', -1))
assert result_queue.qsize() == task_count
result_queue.put('STOP')
monitor(result_queue)
assert result_queue.qsize() == 0
# assert results
assert result(task_id, wait=500, cached=True) == -1
assert fetch(task_id, wait=500, cached=True).result == -1
# make sure it's not in the db backend
assert fetch(task_id) is None
# assert group
assert count_group(group, cached=True) == 6
assert count_group(group, cached=True, failures=True) == 1
assert result_group(group, cached=True) == [-1, -1, -1, -1, -1]
assert len(result_group(group, cached=True, failures=True)) == 6
assert len(fetch_group(group, cached=True)) == 6
assert len(fetch_group(group, cached=True, failures=False)) == 5
delete_group(group, cached=True)
assert count_group(group, cached=True) is None
delete_cached(task_id)
assert result(task_id, cached=True) is None
assert fetch(task_id, cached=True) is None
# iter cached
assert result(iter_id) is None
assert result(iter_id, cached=True) is not None
broker.cache.clear()
def evaluate_paths(self, paths, P_t, current_step, horizon=-1):
jobs = collections.deque()
eval_queue = Queue()
evaluated_paths = []
paths_tmp = [(i, paths[i]) for i in xrange(len(paths)) if len(paths[i][0]) != 0]
for i in xrange(len(paths_tmp)):
logging.info("PathEvaluator: Evaluate path " + str(i))
p = Process(target=self.evaluate, args=(paths_tmp[i][0],
paths_tmp[i][1],
P_t,
current_step,
horizon,
self.robot,
eval_queue,))
p.start()
jobs.append(p)
if len(jobs) == self.num_cores - 1 or i == len(paths_tmp) - 1:
if i == len(paths_tmp) - 1 and not len(jobs) == self.num_cores - 1:
while not eval_queue.qsize() == len(paths_tmp) % (self.num_cores - 1):
time.sleep(0.00001)
else:
while not eval_queue.qsize() == self.num_cores - 1:
time.sleep(0.00001)
jobs.clear()
q_size = eval_queue.qsize()
for j in xrange(q_size):
eval_elem = eval_queue.get()
if eval_elem != None:
evaluated_paths.append(eval_elem)
else:
print "Path could not be evaluated"
path_rewards = [evaluated_paths[i][1] for i in xrange(len(evaluated_paths))]
if len(path_rewards) == 0:
return (None, None, None, None, None, None, None, None, None)
best_index = evaluated_paths[0][0]
best_path = evaluated_paths[0][2]
best_objective = path_rewards[0]
s_covariances = evaluated_paths[0][4]
deviation_covariances = evaluated_paths[0][5]
estimated_deviation_covariances = evaluated_paths[0][6]
for i in xrange(1, len(path_rewards)):
if path_rewards[i] > best_objective:
best_index = evaluated_paths[i][0]
best_objective = path_rewards[i]
best_path = evaluated_paths[i][2]
s_covariances = evaluated_paths[i][4]
deviation_covariances = evaluated_paths[i][5]
estimated_deviation_covariances = evaluated_paths[i][6]
logging.info("PathEvaluator: Objective value for the best path is " + str(best_objective))
return (best_index,
best_path[0],
best_path[1],
best_path[2],
best_path[3],
best_objective,
s_covariances,
deviation_covariances,
estimated_deviation_covariances)
class CommandQueue():
_STATE_NORMAL = 'normal'
_STATE_FLUSHING = 'flushing'
_STATE_FINISHED = 'finished'
def __init__(self):
self._in = Queue()
self._out = Queue()
self._state = self._STATE_NORMAL
self._state_lock = Lock()
self._terminating_commands = (
'notify_kunquat_exception', 'notify_libkunquat_error')
def update(self):
in_count = self._in.qsize()
with self._state_lock:
if self._state == self._STATE_FLUSHING:
get_counter = repeat(True)
else:
get_counter = range(in_count)
for _ in get_counter:
try:
command_data = self._in.get_nowait()
except Empty:
return
command, _ = command_data
if command in self._terminating_commands:
# Make sure we won't block the UI before the terminating command is sent
with self._state_lock:
self._state = self._STATE_FLUSHING
self._out.put(command_data)
def put(self, command, *args):
with self._state_lock:
is_state_normal = (self._state == self._STATE_NORMAL)
if is_state_normal:
self._in.put((command, args))
def get(self):
command_data = self._out.get_nowait()
command, _ = command_data
if command in self._terminating_commands:
with self._state_lock:
self._state = self._STATE_FINISHED
return command_data
def get_command_count(self):
return self._out.qsize()
def runTest(num_proc, query_num):
print "Simulate concurrent user: {}".format(num_proc)
jobs = []
queue = Queue()
try:
for num in range(num_proc):
p = Process(target=runCmdWorker, args=(query_num, queue))
jobs.append(p)
p.start()
for j in jobs:
j.join()
if j.exitcode != 0:
print '%s.exitcode = %s' % (j.name, j.exitcode)
finally:
total_avg = 0.0
for i in range(queue.qsize()):
result = queue.get()
proc_avg = float(result.split(':')[1])
total_avg = total_avg + proc_avg
#print result
print "average execution time: {:.8f}s".format(total_avg/num_proc)
print
def test_recycle(broker, monkeypatch):
# set up the Sentinel
broker.list_key = 'test_recycle_test:q'
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
start_event = Event()
stop_event = Event()
# override settings
monkeypatch.setattr(Conf, 'RECYCLE', 2)
monkeypatch.setattr(Conf, 'WORKERS', 1)
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, broker=broker)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
task_queue = Queue()
result_queue = Queue()
# push two tasks
pusher(task_queue, stop_event, broker=broker)
pusher(task_queue, stop_event, broker=broker)
# worker should exit on recycle
worker(task_queue, result_queue, Value('f', -1))
# check if the work has been done
assert result_queue.qsize() == 2
# save_limit test
monkeypatch.setattr(Conf, 'SAVE_LIMIT', 1)
result_queue.put('STOP')
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
broker.delete_queue()
def main():
# Join hub.py
try:
COMM = Communicator(MACHINEID, SERVER, "logAdGrabber")
LOG.info('Joining hub as %s' % MACHINEID)
response = COMM.send_request({}, 'join')
except Exception as e:
LOG.error(e)
return
LOG.debug('Init data structs')
pr = {}
input_queue = Queue(maxsize=NUMADGRABBERS)
# clean up adsFolder
for f in os.listdir(ADSFOLDER):
os.unlink(os.path.join(ADSFOLDER, f))
for i in range(NUMADGRABBERS):
if os.path.exists('%d.txt' % i):
os.unlink('%d.txt' % i)
LOG.info("Creating phantomjs process %d" % i)
pr[i] = Process(target=spawn_workers, args=(i, input_queue, COMM))
pr[i].start()
while True :
LOG.info('Telling hub I have %d idle slots' % (NUMADGRABBERS -
input_queue.qsize()))
job = COMM.send_request({}, 'idle')
if not job:
LOG.info("No jobs received")
sleep(5)
continue
else:
input_queue.put(job)
def test_scribe_tail_log_everything():
env = "fake_env"
stream_name = "fake_stream"
service = "fake_service"
levels = ["fake_level1", "fake_level2"]
components = ["build", "deploy"]
clusters = ["fake_cluster1", "fake_cluster2"]
instance = "fake_instance"
queue = Queue()
filter_fn = mock.Mock(return_value=True)
tailer = iter(
[
format_log_line(levels[0], clusters, instance, "build", "level: first. component: build."),
format_log_line(levels[1], clusters, instance, "deploy", "level: second. component: deploy."),
]
)
with contextlib.nested(mock.patch("paasta_tools.cli.cmds.logs.scribereader", autospec=True)) as (
mock_scribereader,
):
mock_scribereader.get_env_scribe_host.return_value = {"host": "fake_host", "port": "fake_port"}
mock_scribereader.get_stream_tailer.return_value = tailer
logs.scribe_tail(env, stream_name, service, levels, components, clusters, queue, filter_fn)
assert mock_scribereader.get_env_scribe_host.call_count == 1
mock_scribereader.get_stream_tailer.assert_called_once_with(stream_name, "fake_host", "fake_port")
assert queue.qsize() == 2
# Sadly, fetching with a timeout seems to be needed with
# multiprocessing.Queue (this was not the case with Queue.Queue). It
# failed 8/10 times with a get_nowait() vs 0/10 times with a 0.1s
# timeout.
first_line = queue.get(True, 0.1)
assert "level: first. component: build." in first_line
second_line = queue.get(True, 0.1)
assert "level: second. component: deploy." in second_line
def start_services(services, codegen, importcode, tornadoapp, forker, boot_function, template_path="."):
# loop through all services
# create hosts file
# update kill-file for them (processes)
# create all files (websocket server, and js/python client files)
host_file = {}
kill_file = {}
queue = Queue()
for service in services:
websocket_server_code = codegen(
service, "websocket_server_template.tpl", loader=template.Loader(template_path)
)
websocket_server_module = importcode(websocket_server_code)
websocket_server_class_name = "%s_websocket" % (service["servicename"])
websocket_server_application = tornadoapp(
[(r"/", getattr(websocket_server_module, websocket_server_class_name))]
)
websocket_server_process = forker(
0, queue, boot_function, websocket_server_application, service["servicename"], 0, service
)
print websocket_server_process
print "WSS, ", queue.qsize()
return queue
class QueueEventsSub:
def __init__(self, maxsize=0):
self._maxsize = maxsize
self._q = Queue(maxsize=maxsize)
def put_event(self, e):
if self._q.qsize() == self._maxsize and self._maxsize != 0:
self._q.get_nowait()
self._q.put(e)
def get_event(self):
if self._q.qsize() == 0:
return None
return self._q.get()
def execute(self):
"""
Executing every forest in collection, activating their networks.
By the way collecting data about best fitness function.
"""
process_list = []
forests_queue = Queue(self.power)
iterational = 0
print '| |-starting evaluation, training and validation'
for one_forest in self._forests:
process_list.append(
Process(target=main_async_method,
args=(forests_queue, copy(one_forest.to_portal()), iterational, self.settings)))
iterational += 1
for proc in process_list:
proc.start()
for proc in process_list:
proc.join()
for smth in range(forests_queue.qsize()):
tmp = forests_queue.get()
self._forests[tmp['place']].fitness = tmp['fitness']
fitness_summ = sum(map(lambda forest: forest.fitness, self._forests))
fss = map(lambda x: x.fitness, self._forests)
print 'avg = ', str(sum(fss) / len(fss)), 'max = ', max(fss)
self.roulet = map(lambda x: x.fitness / fitness_summ, self._forests)
def genDict(path_parent, path_dict_folder):
# 第一次遍历，成立词典，获取词频，文频等信息
p_pool = []
dict_queue = Queue()
p_num = 3
for i in range(p_num):
p = Process(target=generate_dict_subprocess,
args=(i,
p_num,
dict_queue,
cut_all,
# '/mnt/D/multiangle/DataSet/THUCNews'
path_parent
))
p_pool.append(p)
for p in p_pool: # 启动进程
# p = Process(p)
p.start()
while True: # 检测是否全部完成
if dict_queue.qsize() >= p_num:
break
time.sleep(1)
dictionary = corpora.Dictionary()
for i in range(p_num):
q_dict = dict_queue.get()
dictionary.merge_with(q_dict)
for p in p_pool:
p.terminate()
dictionary.save(os.path.join(path_dict_folder,'THUNews.dict'))
dictionary.save_as_text(os.path.join(path_dict_folder,'THUNews.txt'))
class MyOVBox(OVBox):
def __init__(self):
OVBox.__init__(self)
self.p = None
self.q = None
def f(self, queue):
while True:
queue.put('hello')
time.sleep(1)
def initialize(self):
print "process initialize!"
self.q = Queue()
self.p = Process(target=self.f, args=(self.q,))
self.p.start()
def process(self):
for i in range(self.q.qsize()):
print self.q.get()
def uninitialize(self):
print "process uninitialize!"
self.p.terminate()
self.p.join()
def run(self):
program_start = time.time()
wQ = Queue()
wQ.cancel_join_thread()
wP = []
for i in range(tp.threads):
p = Process(target=tp.process_file_write, args=(i,wQ,))
wP.append(p)
p.start()
# 10000 files enter here
sys.stdout.write(Fore.GREEN + "Reading from stdin...")
input_files = sys.stdin.readlines()
for input_file in input_files:
wQ.put(input_file.rstrip())
print Fore.GREEN + "\rDone reading from stdin. I found %d files." % (wQ.qsize())
for _ in wP:
wQ.put(None)
for p in wP:
p.join()
program_stop = time.time()
print Fore.GREEN + "Processed %d files in %f seconds (%fs avg)" % (len(input_files), program_stop-program_start, (program_stop-program_start)/len(input_files))
def load_prices_to_db():
date='2999-10-10'
try:
result = cur.execute('select Date from price_data order by date desc limit 1')
date = result.fetchone()[0]
except:
print 'Database query failed'
now = time.strftime("%Y-%m-%d")
if date>now or True:
print 'Updating Prices'
try:
result = cur.execute('DELETE from price_data')
con.commit()
except Exception as e:
print e
result = cur.execute('SELECT ticker from screens group by ticker')
tickers = result.fetchall()
tick_count = 0
ticker_queue = Queue()
for ticker in tickers:
ticker_queue.put(str(ticker[0]))
for i in range(cpu_count()*2):
p = Process(target = get_prices, args = (ticker_queue,))
p.start()
while ticker_queue.qsize()>0:
sleep(1)
def TestMultiUserProcess(filenames,threadnums,T):
allnums=sum(threadnums)
eachnums=allnums/Global.CPUs
actLists=[]
for i in range(len(threadnums),1,-1):
threadnums[i-1]=sum(threadnums[0:i])
for i in range(0,len(filenames)):
actList=[]
for line in open(filenames[i]):
words=line.split("||")
temp=[]
for w in words:
temp.append(w)
actList.append(temp)
actLists.append(actList)
del Global.result[:]
Q=Queue(Global.CPUs)
lock = multiprocessing.Lock()
for i in range(0,Global.CPUs):
p = multiprocessing.Process(target = MultiUserTest, args = (Q,lock,eachnums,threadnums,actLists,T,i,))
p.start()
st=time.time()
while True:
time.sleep(1)
if Q.qsize()==Global.CPUs:
break
for i in range(0,Global.CPUs):
Global.result.extend(Q.get(i))
def ValidateProxies(self, proxyList):
maxProc = 50
tests = ["http://www.baidu.com"]
result = Queue()
start = time.clock()
for i in proxyList:
p = Process(target=self.CheckProxy, args=(i, tests, result))
p.start()
if len(multiprocessing.active_children()) > maxProc:
#print('active_children: ', multiprocessing.active_children())
p.join()
while len(multiprocessing.active_children()) > 0:
time.sleep(3)
end = time.clock()
#print("total time for validation:", end - start, "s")
self.pool = []
for i in range(result.qsize()):
a = result.get()
self.pool += [Proxy(a[0], a[1])]
print("{0} validated".format(len(self.pool)))
def test_recycle(r):
# set up the Sentinel
list_key = 'test_recycle_test:q'
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
start_event = Event()
stop_event = Event()
# override settings
Conf.RECYCLE = 2
Conf.WORKERS = 1
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, list_key=list_key)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
task_queue = Queue()
result_queue = Queue()
# push two tasks
pusher(task_queue, stop_event, list_key=list_key, r=r)
pusher(task_queue, stop_event, list_key=list_key, r=r)
# worker should exit on recycle
worker(task_queue, result_queue, Value('f', -1))
# check if the work has been done
assert result_queue.qsize() == 2
# save_limit test
Conf.SAVE_LIMIT = 1
result_queue.put('STOP')
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
r.delete(list_key)
def crunch(file_name, ext_type, handler, pool_size=4, queue_size=40,
limit=None):
print 'Crunching file: %s, limit: %s' % (file_name, limit)
q = JoinableQueue(queue_size)
q_feats = Queue()
pool = Pool(pool_size, wrap_handler(handler), ((q, q_feats),))
with file_reader(file_name) as reader:
idx = 0
for entry in reader:
if (entry.pathname.find(ext_type) != -1):
text = [b for b in entry.get_blocks()]
key = entry.pathname.split('/')[-1].split('.')[0]
q.put((key, text), True)
idx += 1
print 'Processing:', entry.pathname, idx
if limit and idx >= limit:
print 'Reached the limit'
break
q.close()
q.join()
pool.close()
result = []
for i in range(q_feats.qsize()):
result.append(q_feats.get())
return result
class Quu(): #Singleton
def __init__(self):
self.queue = Queue()
logger.debug('new queue')
def addFolders(self, aLeftFolder, aRightFolder):
logger.debug('adding new item in the quu')
myComparator = FolderComparator(aLeftFolder, aRightFolder)
myComparator.setSyncType("keepboth")
self.queue.put(myComparator)
def getQuu(self):
return self.queue
def getNext(self):
"""
@return FolderComparator the folder
"""
try:
return self.queue.get_nowait()
except BaseException:
return None
def getSize(self):
return self.queue.qsize()
def test_scheduler(r):
list_key = 'scheduler_test:q'
r.delete(list_key)
schedule = create_schedule('math.copysign',
1, -1,
hook='django_q.tests.tasks.result',
schedule_type=Schedule.HOURLY,
repeats=1)
assert schedule.last_run() is None
# run scheduler
scheduler(list_key=list_key)
# set up the workflow
task_queue = Queue()
stop_event = Event()
stop_event.set()
# push it
pusher(task_queue, stop_event, list_key=list_key, r=r)
assert task_queue.qsize() == 1
assert r.llen(list_key) == 0
task_queue.put('STOP')
# let a worker handle them
result_queue = Queue()
worker(task_queue, result_queue, Value('b', -1))
assert result_queue.qsize() == 1
result_queue.put('STOP')
# store the results
monitor(result_queue)
assert result_queue.qsize() == 0
schedule = Schedule.objects.get(pk=schedule.pk)
assert schedule.repeats == 0
assert schedule.last_run() is not None
assert schedule.success() is True
task = fetch(schedule.task)
assert task is not None
assert task.success is True
assert task.result < 0
for t in Schedule.TYPE:
schedule = create_schedule('django_q.tests.tasks.word_multiply',
2,
word='django',
schedule_type=t[0],
repeats=1,
hook='django_q.tests.tasks.result'
)
assert schedule is not None
assert schedule.last_run() is None
scheduler()
def start_spider(spider_cls):
try:
result_queue = Queue()
network_response_queue = Queue()
shutdown_event = Event()
generator_done_event = Event()
taskq = QueueBackend('ng')
#from grab.spider.base import logger_verbose
#logger_verbose.setLevel(logging.DEBUG)
kwargs = {
'taskq': taskq,
'result_queue': result_queue,
'network_response_queue': network_response_queue,
'shutdown_event': shutdown_event,
'generator_done_event': generator_done_event,
'ng': True,
}
# Generator: OK
generator_waiting_shutdown_event = Event()
bot = spider_cls(waiting_shutdown_event=generator_waiting_shutdown_event, **kwargs)
generator = Process(target=bot.run_generator)
generator.start()
# Downloader: OK
downloader_waiting_shutdown_event = Event()
bot = spider_cls(waiting_shutdown_event=downloader_waiting_shutdown_event,
**kwargs)
downloader = Process(target=bot.run)
downloader.start()
# Parser: OK
events = []
for x in xrange(2):
parser_waiting_shutdown_event = Event()
events.append(parser_waiting_shutdown_event)
bot = spider_cls(waiting_shutdown_event=parser_waiting_shutdown_event,
**kwargs)
parser = Process(target=bot.run_parser)
parser.start()
while True:
time.sleep(2)
print('task size', taskq.size())
print('response size', network_response_queue.qsize())
if (downloader_waiting_shutdown_event.is_set() and
all(x.is_set() for x in events)):
shutdown_event.set()
break
time.sleep(1)
print('done')
finally:
for child in active_children():
logging.debug('Killing child process (pid=%d)' % child.pid)
child.terminate()
class BroadFirstGenerator(Process):
def __init__(self, config_path):
Process.__init__(self)
self.config = BroadFirstConfig(config_path)
self.data = Queue()
self.x = self.config.x
self.y = self.config.y
self.stop_produce = False
self.stop_consume = False
self.total = self.config.getTotalTileNum()
self.count = 0
def run(self):
init = True
while True and not (self.stop_produce):
dl = self.data.qsize()
if dl <= self.config.MAX_QUEUE/2 and not self.stop_produce:
for _ in range(self.config.MAX_QUEUE/4):
if not self.getNextXY():
self.stop_produce = True
break
else:
self.data.put((self.x, self.y, self.config.min_z))
if init:
init = False
num_consumers = multiprocessing.cpu_count() * 2
# num_consumers = 2
consumers = [BroadFirstDownloader(self.data, self.config) for _ in range(num_consumers)]
for consumer in consumers:
consumer.start()
if self.data.empty():
break
time.sleep(60)
def getNextXY(self):
# logger = multiprocessing.get_logger()
if self.config.debug:
if self.count >= self.config.debugTryTimes:
self.x = None
self.y = None
return False
self.count = self.count + 1
if self.count % 1 == 0 and self.x != None and self.y != None:
# process = self.config.getProcess(self.x, self.y, self.data.qsize())
# print '%s: Proceed:%f/100' % (time.ctime(), process)
# logger.debug('%s: Proceed:%f/100' % (time.ctime(), process))
self.config.updateState(self.x, self.y)
if self.y < self.config.max_y:
if self.x < self.config.max_x:
self.x = self.x + 1
else:
self.x = self.min_x
self.y = self.y + 1
return True
else:
self.x = None
self.y = None
return False
def start_spider(spider_cls):
try:
result_queue = Queue()
response_queue = Queue()
shutdown_event = Event()
generator_done_event = Event()
taskq = Queue()
args = (taskq, result_queue, response_queue, shutdown_event, generator_done_event)
kwargs = dict(verbose_logging=True)
generator_wating_shutdown_event = Event()
generator_cls = build_generator_spider(spider_cls)
generator = create_process(generator_cls, generator_wating_shutdown_event,
*args, **kwargs)
generator.start()
downloader_wating_shutdown_event = Event()
downloader = create_process(BaseDownloaderSpider, downloader_wating_shutdown_event,
*args, **kwargs)
downloader.start()
parser_wating_shutdown_event = Event()
parser_cls = build_parser_spider(spider_cls)
parser = create_process(parser_cls, parser_wating_shutdown_event,
*args, **kwargs)
parser.start()
while True:
time.sleep(2)
print 'task size', taskq.qsize()
print 'response size', response_queue.qsize()
if (downloader_wating_shutdown_event.is_set() and
parser_wating_shutdown_event.is_set()):
shutdown_event.set()
break
time.sleep(1)
print 'done'
finally:
for child in active_children():
logging.debug('Killing child process (pid=%d)' % child.pid)
child.terminate()
def parse_input_files(input_files, num_threads, genomes, min_coverage, min_proportion):
"""
Use a pool of worker threads to, in parallel, read in the input files.
Populate the genome collection with the read-in data.
This is the "poison pill" thread management algorithm, where threads are
each given a "you can stop now" task once the actual queue of tasks is
complete.
"""
# Lines below marked "Single-thread" can be uncommented and replace
# the lines marked "Multi-thread" to get single-thread behavior.
from multiprocessing import Process, Queue # Multi-thread
# from queue import Queue # Single-thread
from time import sleep
input_q = Queue()
output_q = Queue()
for input_file in input_files:
input_q.put(input_file)
# If the number of jobs is already smaller than the thread pool...
if num_threads > input_q.qsize():
num_threads = input_q.qsize()
sleep(1)
thread_list = []
for current_thread in range(num_threads):
input_q.put(None)
current_thread = Process(target=manage_input_thread,
args=[genomes.reference(), min_coverage, min_proportion, input_q,
output_q]) # Multi-thread
current_thread.start() # Multi-thread
#manage_input_thread( genomes.reference(), min_coverage, min_proportion, input_q, output_q ) # Single-thread
thread_list.append(current_thread) # Multi-thread
sleep(1)
while num_threads > 0:
new_genome = output_q.get()
if new_genome is None:
num_threads -= 1
elif isinstance(new_genome, str):
# Reading this file in failed. We only know the filename.
genomes.add_failed_genome(new_genome)
else:
genomes.add_genome(new_genome)
sleep(1)
for current_thread in thread_list: # Multi-thread
current_thread.join() # Multi-thread
def server_discover(servers=None, exclude_patterns=None, querry_timeout=5, qsize=8, info=0):
"""
issue: todo: use multiprocessing instead of thrading may faster,
qsize should be tuned to a better value
"""
#if servers is None:
if servers is None:
servers= []
if servers== 'all':
servers= (
['node%d'%i for i in range(1, 100)] #8cpu, normal, mem48, mem96
+ ['localhost', 'qtg7501']
)
print('run server_discover ... ')
res = []
msg = {'header': 'querry', 'what': 'is_exists'}
if 0: # serial
for s in servers:
rep=send_msg(msg, server=s, querry_timeout=querry_timeout, info=info)
if rep == 'Y' :
res.append(s)
else:
#qsize = 8 #this could not be too large, or else error may occor
from Queue import Queue
qq = Queue(qsize)
def func(s):
try:
rep=send_msg(msg, server=s, querry_timeout=querry_timeout, info=info)
except Exception as err:
rep = str(err)
#rep = 'error'
qq.put((s, rep))
n = len(servers)/qsize + 1
for i in range(n) :
tt = []
for s in servers[i*qsize: i*qsize + qsize]:
t = threading.Thread(target=func, args=(s, ))
t.start()
tt.append(t)
for t in tt:
t.join()
temp = [qq.get() for a in range(qq.qsize())]
#print_vars(vars(), ['i, temp'])
res.extend(temp)
res = [r[0] for r in res if r[1] == 'Y' ]
if exclude_patterns is not None:
for e in exclude_patterns:
res= [r for r in res if e not in r]
print('servers found are:\n\t %s'%(res, ))
return res
class worker(api):
def __init__(self, workdir, n_threads=0):
self.n_threads = cpu_count() if n_threads == 0 else n_threads
self.process_list = []
self.workdir = workdir
self.work_queue = Queue()
self.update_queues = []
self.testcase_report = Queue()
self.coverage = dict()
self.testcases = []
self.executed_testcases = Value('i', 0)
self.mutator = mutator()
if not os.path.exists(workdir):
os.makedirs(workdir)
os.chdir(workdir)
def run(self, n_testcases=0):
#cleanup workdir
for fname in glob.glob("*sancov") + glob.glob("t-*"):
os.remove(fname)
self.work_queue.put(self.mutator.initial_testcase())
self.executor = executor(self.cmd, ".")
#start worker threads
self.update_queues = []
for i in xrange(self.n_threads):
self.update_queues.append(Queue())
d = Process(target=self.worker, args=(i, ))
d.daemon = True
d.start()
self.process_list += [d]
self.client(n_testcases)
def apply_update(self, update):
if len(update["testcase_update"]) > 0:
for t in update["testcase_update"]:
pid = t["parent_id"]
self.testcases += [t]
if t["id"] > 0 and t["id"] < len(self.testcases):
self.testcases[pid]["childs"] += [t["id"]]
self.mutator.random_merge_cache = {}
if "coverage_update" in update:
for k in update["coverage_update"].keys():
if k not in self.coverage:
self.coverage[k] = set()
self.coverage[k].update(set(update["coverage_update"][k]))
self.crash = update["crash"]
#execute testcases and process results
def worker(self, worker_id):
print "worker started"
self.worker_id = worker_id
while True:
for mutated in self.get_testcases():
self.execute_testcase(mutated)
def execute_testcase(self, testcase):
self.executed_testcases.value += 1
try:
data = self.mutator.mutate_seed(testcase, self.seed_data)
stderr, crash, coverage = self.executor.call(data, self.ext)
if len(coverage) > 0:
self.process_result(testcase, stderr, crash, coverage, data)
except:
import traceback
traceback.print_exc()
pass
#detect new edges/crashes and appends to report queues
def process_result(self, testcase, stderr, crash, coverage, binary):
testcase["coverage"] = (coverage)
testcase["bin"] = b64encode(binary)
#detect new crash
if crash and crash not in self.crash:
print "New crash %s" % crash
testcase["crash"] = crash
testcase["stderr"] = stderr
self.testcase_report.put(testcase)
return
#found new blocks, requeue to remove unused mutations
if "minimize" not in testcase:
new_blocks = self.compute_new_blocks(coverage)
if new_blocks > 0:
#remove unused mutations
testcase["new_blocks"] = new_blocks
minimize = {}
minimize["i"] = 0
minimize["reference"] = deepcopy(testcase)
testcase["minimize"] = minimize
self.work_queue.put(testcase)
#remove unused mutations
else:
i = testcase["minimize"]["i"]
reference = testcase["minimize"]["reference"]
#test if testcase covered equal or more blocks
ge = False
if not crash:
new_blocks = self.compute_new_blocks(coverage)
testcase["new_blocks"] = new_blocks
if new_blocks >= reference["new_blocks"]:
ge = True
#done
if i >= len(testcase["mutations"]) - 1:
if not ge:
testcase = reference
if "minimize" in testcase: del testcase["minimize"]
if "random-merge" not in testcase["description"] and len(
testcase["mutations"]) > 0:
state = testcase["mutations"][-1]
testcase["description"] = (
"offset=%d: " % state["offset"]) + state["description"]
print "Minimized testcase: New blocks: %d Parent: %d Description: %s " % (
testcase["new_blocks"], testcase["id"],
testcase["description"]),
print "Mutations: %d" % len(testcase["mutations"]),
print "Report Queue: %d" % self.testcase_report.qsize()
new_blocks = self.compute_new_blocks(coverage)
self.testcase_report.put(testcase)
return
#mutation was unused
if ge:
#print "%d unused" % i
del testcase["minimize"]["reference"]
testcase["minimize"]["reference"] = deepcopy(testcase)
testcase["minimize"]["i"] = i
else:
#print "%d used" % i
testcase["minimize"]["i"] = i + 1
testcase["mutations"] = reference["mutations"][:i] + reference[
"mutations"][i + 1:]
del testcase["coverage"]
self.work_queue.put(testcase)
#genetic methods
def get_testcases(self):
while True:
#apply updates
try:
update = self.update_queues[self.worker_id].get(False)
self.apply_update(update)
except:
pass
#initial and deterministic testcases
try:
while True:
testcase = self.work_queue.get(False)
yield testcase
except:
pass
#choose
if len(self.testcases) == 0: continue
s = reduce(lambda x, y: x + y["new_blocks"], self.testcases, 0)
r = randrange(s)
for t in sorted(self.testcases,
key=lambda t: t["new_blocks"],
reverse=True):
r -= t["new_blocks"]
if r < 0:
break
tid = self.testcases.index(t)
merged = self.mutator.random_merge(self.testcases, tid)
if merged: yield merged
#get
testcase = self.testcases[tid]
#mutate
if tid == 0:
mutated = self.mutator.random_mutation(testcase, maximum=8)
else:
mutated = self.mutator.random_mutation(testcase, maximum=8)
mutated["parent_id"] = tid
yield mutated
def stop(self):
try:
self.executor.watchDog.exit()
except:
pass
for p in self.process_list:
print "killing", p.pid
os.kill(p.pid, 9)
self.process_list = []
self.executed_testcases.value = 0
def crash_fuzz(self, files):
self.addrs = []
self.crashes = []
for fname in files:
with open(fname, "r") as f:
testcase = json.loads(f.read())
testcase["description"] = ""
stderr, crash, coverage = self.callback(self, testcase)
self.crash_fuzz_process_crash(stderr, testcase)
#fuzz crash
while True:
testcase = choice(self.crashes)
mutated = deepcopy(testcase)
mutated["mutators"]["data"] = self.mutator.get_random_mutations(
testcase["mutators"]["data"],
maximum=1) #, mutations=[3]) ##, start=711-16, stop=711+16)
mutated["mutators"]["data"] = self.mutator.get_random_mutations(
testcase["mutators"]["data"],
maximum=1,
mutations=[3],
start=0,
stop=0)
stderr, crash, coverage = self.callback(self, mutated)
self.crash_fuzz_process_crash(stderr, mutated)
def crash_fuzz_process_crash(self, stderr, testcase):
for line in stderr.split("\n"):
if "ERROR: AddressSanitizer:" in line:
try:
addr = re.findall("on [a-z ]*address 0x[0-9a-f]*", line)[0]
addr = re.findall("0x[0-9a-f]*", addr)[0]
crash = re.findall("pc 0x[0-9a-f]*", line)[0]
addr = "%s-%s" % (crash, addr)
if addr not in self.addrs:
if "READ" in stderr:
cause = "READ"
elif "WRITE" in stderr:
cause = "WRITE"
else:
cause = "OTHER"
stderr, crash, coverage = self.callback(
self,
testcase,
dumpfile="crashFuzz-%s-%s.bin" % (addr, cause))
save_data("crashFuzz-%s-%s.stderr" % (addr, cause),
stderr)
print cause, addr, testcase["description"]
self.addrs += [addr]
self.crashes += [testcase]
except:
#print stderr
import traceback
traceback.print_exc()
i += 1
class MultistreamCache():
'''
Input sample cache that employs a set of worker threads which collect new input samples from files to insert into the cache.
Can produce sample batches by randomly selecting items from cache.
Ensures at least a certain number of items are refreshed after each new batch is generated.
'''
def __init__(
self,
worker_method,
worker_options, # replace at least this many entries on each cache update. Can be fractional
alpha_smoother=0.99
): # the higher the more temporally smoothed is the average_replacement_rate. Not very important
self.num_workers = worker_options["num_workers"]
self.worker_method = worker_method
self.worker_options = worker_options
self.cache_size = worker_options["cache_size"]
self.min_replacement_rate = worker_options["min_replacement_rate"]
self.alpha_smoother = alpha_smoother
# Internal Data Structures
self.communication_queue = Queue(maxsize=150) #TODO hardcoded for now
self.worker_handles = []
self.cache = [None] * self.cache_size
self.idx_next_item_to_be_updated = 0
self.average_replacement_rate = self.min_replacement_rate
self.exit_flag = Event()
self.exit_flag.clear()
self.counter_cache_items_updated = 0
# call seed if this is used from different threads / processes
seed()
def start_workers(self):
for k in range(self.num_workers):
p = Process(target=self.worker_method,
args=(self.communication_queue, self.exit_flag,
self.worker_options))
#print(str(self.worker_options))
print('in start_workers: ', str(k))
p.start()
self.worker_handles.append(p)
# Fill cache
print('----- Filling cache (Size: {}) -------'.format(self.cache_size))
for k in range(self.cache_size):
try:
data = self.communication_queue.get(timeout=10)
self.update_next_cache_item(data)
except Empty as error:
print('Timeout: {}'.format(str(error)))
print('qsize: ' + str(self.communication_queue.qsize()))
# print(str(self.cache[self.idx_next_item_to_be_updated - 1]))
print('----- Cache Filled -------')
# We reset the update counter when starting the workers
self.counter_cache_items_updated = 0
def stop_workers(self):
# We just kill them assuming there is nothing to be shut down properly.
# This is somewhat brutal but simplifies things a lot and is enough for now
self.exit_flag.set()
for worker in self.worker_handles:
worker.join(timeout=5)
worker.terminate() # try harder to kill it off if necessary
def update_next_cache_item(self, data):
self.cache[self.idx_next_item_to_be_updated] = data
self.idx_next_item_to_be_updated = (self.idx_next_item_to_be_updated +
1) % self.cache_size
self.counter_cache_items_updated += 1
def update_cache_from_queue(self):
# Implements a minimum update rate in terms of an average
# number of items that have to be replaced in a call to this
# function. If the average is not achieved, this functions
# blocks until the required number of items are replaced in the
# cache.
num_replacements_current = 0
average_replacement_rate_prev = self.average_replacement_rate
while True:
average_replacement_rate_updated = (
1 - self.alpha_smoother
) * num_replacements_current + self.alpha_smoother * average_replacement_rate_prev
if (average_replacement_rate_updated >= self.min_replacement_rate):
break
if num_replacements_current == self.cache_size: # entire cache replaced? Your IO is super fast!
break
#print('Loading new item into cache from data list starting with ' + self.worker_options["file_list"][0][0].path)
self.update_next_cache_item(self.communication_queue.get())
num_replacements_current += 1
print('num_replacements_current: ' + str(num_replacements_current))
# Final update of self.num_replacements_smoothed
self.average_replacement_rate = average_replacement_rate_updated
print('ave_replace_rate: ' + str(self.average_replacement_rate))
def get_cache_item(self, idx):
return self.cache[idx]
def set_cache_item(self, idx, item):
self.cache[idx] = item
# nproc is the number of parallel processes to initiate (each process will
# take jobs from the inputq until it is empty)
processes = [
Process(target=remote_command, args=(inputq, outputq))
for i in range(nproc)
]
for p in processes:
p.start()
for p in processes:
#sys.stderr.write( str(p.pid) + "\n" )
p.join()
#print ("outputq.size:", outputq.qsize())
#print ("inputq.size:", inputq.qsize())
while outputq.qsize() > 0:
machine, data_area, output = outputq.get()
#print (machine, data_area)
#print ("output", output)
if machine not in diskreport.keys():
diskreport[machine] = dict()
if data_area not in diskreport[machine].keys():
diskreport[machine][data_area] = dict()
if disk_query == "du":
#diskreport[machine][data_area]["du"] = (output.split("\n"))
diskreport[machine][data_area]["du"] = []
for directory in output.split(b"\n")[0:-1]:
du_output = directory.split()
du_output[0] = int(du_output[0])
diskreport[machine][data_area]["du"].append(du_output)
elif disk_query == "df":
print(q.full()) # 还只放2条消息，没有满False
q.put("消息3")
print(q.full()) #True
# 因为消息列队已满下面的try都会抛出异常，
# 第一个try会等待2秒后再抛出异常，第二个Try会立刻抛出异常
# put函数有一个参数block默认为True，如果已经没有空间可写入，
# 此时程序将被阻塞（停在写入状态），直到从消息列队腾出空间为止，
# 如果设置了timeout，则会等待timeout秒，若还没空间，则抛出"Queue.Full"异常；
# 因此使用try函数，消息满了，超时2秒，然后执行下面的except语句
try:
q.put("消息4",True,2)
except:
print("消息列队已满，现有消息数量:%s" % q.qsize())
try:
# 如果block值为False，消息列队如果没有空间可写入，则会立刻抛出"Queue.Full"异常
# Queue.put_nowait(item)：相当Queue.put(item, False)
# 立即抛出异常
q.put_nowait("消息4")
except:
print("消息列队已满，现有消息数量:%s" % q.qsize())
#推荐的方式，先判断消息列队是否已满，再写入
if not q.full():
q.put_nowait("消息4")
except Exception as e:
print 'Could not start server.'
printv(e,2)
for thr in threadPool:
thr.terminate()
sys.exit(0)
while True: #Master loop
try:
conn, addr = s.accept()
printv('[Master] New connection from'+str(addr),3)
if q.empty() and len(threadPool)>minThreadNum:
if threadStatus[-1]==False:
threadPool[-1].terminate()
threadPool.pop()
threadStatus.pop()
printv('Killed thread. New thread count : '+str(len(threadPool)),2)
if q.qsize()>2 and len(threadPool)<maxThreadNum:
threadStatus.append(False)
p=Process(target=serverProcess, args=(q,len(threadPool),threadStatus,))
threadPool.append(p)
p.start()
printv('Started thread. New thread count : '+str(len(threadPool)),2)
q.put(forking_dumps([conn,addr]))
conn.close()
except KeyboardInterrupt:
printv('Server stopped from keyboard',1)
sys.exit(0)
except socket.timeout:
pass
def startTicker(q):
logging.info(
'starting threaded websocket connection, loading initial state')
# Initialise
global access_token_A
access_token_A = ReadAccessTokenFile(access_token_A_file)
#global access_token_B
#access_token_B=ReadAccessTokenFile(access_token_B_file)
global kws
kws = KiteTicker(api_key_A, access_token_A)
#setting queue for DB update
qDB = Queue()
#put df in queue
#jobReconnect()
global df
#if oracle_db.dialect.has_table(oracle_db, 's3stockmaster'):
# df=jobImportDB()
# logging.info('putting db imported dataframe in multiprocessing queue on RAM')
# q.put(df)
#else:
# pass
# print('does not exist')
qDB.put(1)
kws.on_ticks = on_ticks
kws.on_connect = on_connect
kws.on_close = on_close
kws.connect(threaded=True)
flagMargin = False
flagStartOver = False
counterStockTick = 0
df['profit'] = 0
df['rank'] = 0
df['rank2'] = 0
df['loser'] = 0
df['skip'] = 0
df['buy'] = 0
df['sell'] = 0
df['high'] = 0
df['low'] = 0
#t = timer()
global df_original
df_original = df
while True:
#index and make changes in a copy of df called df_calc
df_calc = df
df_calc = df_calc.set_index('instrument_token')
#traverse dataframe
for index, row in df_calc.iterrows():
#set base buy and sell quantity
if df_calc.loc[index, 'action'] == 'BaseSet':
pass
#identify the stock to be traded
elif df_calc.loc[index, 'action'] == 'StockSelect':
if df_calc.loc[index,
'rank'] == 1 and df_calc.loc[index, 'v'] >= 2:
df_calc.at[index, 'qty'] = math.floor(
344 / df_calc.loc[index, 'ltp'] *
df_calc.loc[index, 'x'])
df_calc.at[index, 'action'] = 'Start'
elif df_calc.loc[index,
'rank'] == 1 and df_calc.loc[index, 'v'] < 2:
flagStartOver = True
#place order on zerodha for the selected trade
elif df_calc.loc[index, 'action'] == 'Start':
if skip_kite == False:
if direction_switch == True:
df_calc.at[index, 'id_a_sell'] = placeMarketOrder(
api_key_A, access_token_A, 'SELL',
df_calc.loc[index, 'zid'], df_calc.loc[index,
'qty'])
df_calc.at[index,
'sl'] = 1.003 * df_calc.loc[index, 'ltp']
else:
df_calc.at[index, 'id_a_buy'] = placeMarketOrder(
api_key_A, access_token_A, 'BUY',
df_calc.loc[index, 'zid'], df_calc.loc[index,
'qty'])
df_calc.at[index,
'sl'] = 0.997 * df_calc.loc[index, 'ltp']
df_calc.at[index, 'action'] = 'VerifyStart'
df_calc.at[index, 'pExec'] = df_calc.loc[index, 'ltp']
#verify if the initial zerodha order is completed
elif df_calc.loc[index, 'action'] == 'VerifyStart':
if skip_kite == False:
if direction_switch == True:
status = checkExecutionStatus(
api_key_A, access_token_A,
df_calc.loc[index, 'id_a_sell'])
if status == "COMPLETE":
df_calc.at[index, 'id_a_buy'] = placeLimitOrder(
api_key_A, access_token_A, 'BUY',
df_calc.loc[index, 'zid'], df_calc.loc[index,
'qty'],
round(0.997 * df_calc.loc[index, 'ltp'], 1))
df_calc.at[index, 'action'] = 'RunTrade'
elif status == "REJECTED":
df_calc.at[index, 'action'] = 'FatalError'
else:
status = checkExecutionStatus(
api_key_A, access_token_A, df_calc.loc[index,
'id_a_buy'])
if status == "COMPLETE":
df_calc.at[index, 'id_a_sell'] = placeLimitOrder(
api_key_A, access_token_A, 'SELL',
df_calc.loc[index, 'zid'], df_calc.loc[index,
'qty'],
round(1.003 * df_calc.loc[index, 'ltp'], 1))
df_calc.at[index, 'action'] = 'RunTrade'
elif status == "REJECTED":
df_calc.at[index, 'action'] = 'FatalError'
else:
df_calc.at[index, 'action'] = 'RunTrade'
elif df_calc.loc[index, 'action'] == 'ExitManual':
pass
#verify if the closing zerodha order is completed
elif df_calc.loc[index, 'action'] == 'RunTrade':
if skip_kite == False:
slflag = 0
if direction_switch == True:
if df_calc.loc[index, 'ltp'] > df_calc.loc[index,
'sl']:
#df_calc.at[index,'id_a_buy']=placeMarketOrder(api_key_A,access_token_A,'BUY',df_calc.loc[index,'zid'],df_calc.loc[index,'qty'])
#df_calc.at[index,'action']='TradeComplete'
df_calc.at[index, 'action'] = 'ExitManual'
slflag = 1
else:
if df_calc.loc[index, 'ltp'] < df_calc.loc[index,
'sl']:
#df_calc.at[index,'id_a_sell']=placeMarketOrder(api_key_A,access_token_A,'SELL',df_calc.loc[index,'zid'],df_calc.loc[index,'qty'])
#df_calc.at[index,'action']='TradeComplete'
df_calc.at[index, 'action'] = 'ExitManual'
slflag = 1
if slflag == 0:
if direction_switch == True:
status = checkExecutionStatus(
api_key_A, access_token_A,
df_calc.loc[index, 'id_a_buy'])
else:
status = checkExecutionStatus(
api_key_A, access_token_A,
df_calc.loc[index, 'id_a_sell'])
if status == "COMPLETE":
df_calc.at[index, 'action'] = 'TradeComplete'
elif status == "REJECTED":
df_calc.at[index, 'action'] = 'FatalError'
else:
df_calc.at[index, 'action'] = 'TradeComplete'
#trade completed on zerodha, start over
elif df_calc.loc[index, 'action'] == 'TradeComplete':
df_calc.at[index,
'skip'] = 0 #skip 1 to prevent repetition of stock
flagStartOver = True
elif df_calc.loc[index, 'action'] == 'FatalError':
df_calc.at[index, 'skip'] = 1
#df_calc.at[index,'action']='StockSelect'
else:
if df_calc.loc[index, 'ltp'] > 0.1:
df_calc.at[index, 'pBase'] = df_calc.loc[index, 'ltp']
df_calc.at[index, 'action'] = 'BaseSet'
counterStockTick += 1
else:
if counterStockTick > 100:
df_calc.at[index, 'action'] = 'BaseSet'
else:
df_calc.at[index, 'action'] = 'Waiting'
#if all action status is the same -- steps for global changes
if df_calc.action.nunique() == 1:
if 'StockSelect' in df_calc.values:
#filter out stocks
df_calc = df_calc[df_calc.skip == 0]
df_calc = df_calc[df_calc.buy > 10]
df_calc = df_calc[df_calc.sell > 10]
df_calc = df_calc[df_calc.high > 10]
df_calc = df_calc[df_calc.low > 10]
df_calc = df_calc[df_calc.ltp < 1000]
df_calc = df_calc[df_calc.x > 1]
#calculate volatility and rank
if direction_switch == True:
df_calc['v'] = df_calc['sell'] / df_calc['buy']
df_calc['v2'] = (df_calc['ltp'] - df_calc['low']) / (
df_calc['high'] - df_calc['low'])
else:
df_calc['v'] = df_calc['buy'] / df_calc['sell']
df_calc['v2'] = (df_calc['high'] - df_calc['ltp']) / (
df_calc['high'] - df_calc['low'])
df_calc = df_calc[df_calc.v2 > 0.5]
df_calc['rank'] = df_calc['v'].rank(ascending=False)
df_calc['rank2'] = df_calc['v2'].rank(ascending=False)
elif 'BaseSet' in df_calc.values:
#set values to prevent error
df_calc.loc[df_calc['sell'] == 0, 'sell'] += 1
df_calc.loc[df_calc['buy'] == 0, 'buy'] += 1
#t = timer()
if flagMargin == False:
if 12.5 in df_calc.values:
pass
else:
getMarginX(df_calc)
df_original = df_original.set_index('instrument_token')
df_original = df_calc
df_original = df_original.reset_index()
flagMargin = True
df_calc['action'] = 'StockSelect'
df_calc = df_calc.reset_index()
df = df_calc
if flagStartOver == True:
counterStockTick = 0
df = df_original
flagStartOver = False
os.system("cls")
#pd.set_option('display.max_rows', len(df_calc))
if direction_switch == True:
print('Direction: Sell -> Buy')
else:
print('Direction: Buy -> Sell')
print(f'Stock count: {df.shape[0]}')
print(df.nsmallest(5, 'rank'))
if qDB.qsize() == 1:
qDB.get()
#pDB = Process(target=jobUpdateDB, args=(df_calc,qDB,1))
#pDB.start()
#else:
#logging.info('dataframe updated on RAM only, skipping DB write')
#q.put(df_calc)
time.sleep(.2)
q.put_nowait(obj) 如果队列已满, 抛异常
q.get_nowait() 如果队列为空, 抛异常
q.full() 队列是否已满
q.empty() 队列是否为空
q.qsize() 获取当前队列的大小(队列中元素的个数)
"""
from multiprocessing import Queue
# 创建队列对象
q = Queue(3) # 创建一个长度为3的队列
print(q.full()) # False 判断当前队列满了吗
print(q.empty()) # True 判断当前队列是不是空的
print(q.qsize()) # 0
q.put("李小花")
q.put("王二狗")
q.put("刘三胖")
print(q.full()) # True
print(q.qsize()) # 3
# q.put("史狗剩") # 阻塞式的, 阻塞程序的执行, 直到将"史狗剩"放到队列中为止
# q.put("刘铁蛋", timeout=3) # 阻塞式的, 阻塞时间为timeout, 如果超出阻塞时间, 抛出异常raise Full, 打断程序的执行
# q.put("赵铁柱", block=False) # 如果设置block为False, 非阻塞式, 代表如果队列已满, 直接抛出异常
# q.put_nowait("隔壁老王") # raise Full
print("---------------")
def main(args):
os.chdir(os.path.dirname(os.path.abspath(__file__)))
config = importlib.import_module(args.config).Config
app_timer = Stopwatch()
app_timer.start()
# --- Setup Serial ---
print('Serial Start up...')
sensor_output = Queue()
serial_process = Process(target=sensor.reader,
daemon=True,
args=(config.port, config.baudrate,
config.sampling_rate,
config.sensor_data_labels, sensor_output))
serial_process.start()
if sensor_output.get() is sensor.FAIL:
raise RuntimeError('Serial Process Fail.')
# --- Setup GUI ---
print('GUI is launching...')
pygame.init()
GUI.init()
screen = pygame.display.set_mode((720, 483))
pygame.display.set_caption("STEM")
pygame.display.set_icon(
GUI.make_text('|†|',
font=pygame.font.Font(
"fonts/Ubuntu Mono derivative Powerline Bold.ttf",
64),
color=GUI.color.black))
# End Setup GUI ---
# --- Setup Model ---
print('Model Setup...')
base_model = importlib.import_module(config.model_path).Model()
inputs = tf.keras.Input(shape=(
config.model_sensor_data_inputs,
len(config.sensor_feature_labels),
),
batch_size=None)
x = base_model(inputs)
outputs = Duplicate()(x)
model = Model(inputs, outputs)
model.compile(loss=facenet.triplet_loss(), optimizer='adam')
base_model.summary()
model.summary()
if os.path.exists(config.checkpoint):
print('last checkpoint found. Loading the weights...')
model.load_weights(config.checkpoint)
# model.save_weights(args.checkpoint)
# End Setup Model ---
is_running = True
profile = {
'keyboard_input': 0,
'sensor_read': 0,
'gui_update': 0,
'screen_update': 0
}
input_queue = deque(maxlen=config.model_sensor_data_inputs)
# 0 -> 'label A', 1 -> 'label B', ...
id2label = [None for _ in range(len(config.possible_states))]
label2id = {}
precedents_dict = [
deque(maxlen=config.precedents_maxlen)
for _ in range(len(config.possible_states) + 1)
]
estimator = facenet.Estimator(
model=model,
precedents_dict=precedents_dict,
)
trainor = facenet.Trainor(precedents_dict=precedents_dict)
prev_saving_precedent_time = app_timer.elapsed
fixed_loop = FixedLoop(1 / config.frame_rate)
fixed_loop.reset()
fixed_loop.sync()
estimated = None
while is_running:
screen.fill(GUI.color.screen_backgorund)
submitted_estimator = False
submitted_trainor = False
supervised_state_label = None
app_timer.lap()
# --- Keyboard Input ---
pressed_keys = pygame.key.get_pressed()
for label, settings in config.possible_states.items():
if pressed_keys[settings['key']]:
supervised_state_label = label
for event in pygame.event.get():
if event.type == QUIT:
running = False
if event.type == KEYDOWN:
if event.key == K_ESCAPE:
is_running = False
profile['keyboard_input'] = app_timer.lap()
# End Keyboard Input ---
# --- Control Module Behaviors ---
# print(sensor.read_latest_data(ser))
# sensor_data = sensor.read_latest_data(ser, config.sensor_data_labels)
# sensor_data = dotdict(
# {'timestamp': time.time(), 'pulse_width': 100, 'flow_amount': 200})
received_size = sensor_output.qsize()
for _ in range(received_size):
sensor_data = sensor_output.get()
if sensor_data is sensor.DROP:
input_queue.clear()
elif sensor_data is sensor.FAIL:
raise RuntimeError('Serial Process Fail.')
else:
features = [
sensor_data[label]
for label in config.sensor_feature_labels
]
input_queue.append(features)
profile['sensor_read'] = app_timer.lap()
if len(input_queue) >= input_queue.maxlen:
input_list = np.array([input_queue])
if not estimator.is_running:
submitted_estimator = True
estimator.run(inputs=input_list,
supervised_state_label=supervised_state_label)
count = 0
for state_id in range(len(precedents_dict) - 1):
count += len(precedents_dict[state_id])
if count > len(config.possible_states):
precedents_dict[-1].clear()
if not estimator.results.empty():
estimated = estimator.results.get()
if (estimated.supervised_state_label is not None) and (
estimated.estimated_state < len(config.possible_states)):
if id2label[estimated.estimated_state] is None:
id2label[
estimated.
estimated_state] = estimated.supervised_state_label
label2id[
estimated.
supervised_state_label] = estimated.estimated_state
if label2id.get(estimated.supervised_state_label) is None:
aligned_id = 0
for i, label in enumerate(id2label):
if label is None:
aligned_id = i
break
id2label[aligned_id] = estimated.supervised_state_label
label2id[estimated.supervised_state_label] = aligned_id
estimated.supervised_state = label2id[
estimated.supervised_state_label]
# print(estimated.supervised_state)
if (
estimated.supervised_state_label is not None
) or app_timer.elapsed > prev_saving_precedent_time + config.precedent_interval:
if not trainor.is_running:
submitted_trainor = True
trainor.run(model=model, anchor=estimated)
major_state = facenet.get_major_state(estimated)
precedents_dict[major_state].append(estimated)
prev_saving_precedent_time = app_timer.elapsed
if not trainor.results.empty():
pass
# print(len(input_queue))
profile['submodule_control'] = app_timer.lap()
# End Control Module Behaviors ---
# --- Update GUI Elements ---
if supervised_state_label is None:
screen.blit(GUI.make_text('Self Learning...', GUI.font.large),
(400, 0))
else:
screen.blit(
GUI.make_text(
'Supervised... {0}'.format(supervised_state_label),
GUI.font.large), (400, 0))
screen.blit(GUI.make_text('State: ', GUI.font.large), (0, 0))
if estimated is not None and estimated.estimated_state < len(
config.possible_states):
current_state = facenet.get_major_state(estimated)
screen.blit(
GUI.make_text(
'{0} {1}'.format(
current_state, '?' if id2label[current_state] is None
else id2label[current_state]), GUI.font.large),
(80, 0))
else:
screen.blit(GUI.make_text('?', GUI.font.large), (80, 0))
GUI.begin_multilines((400, 30))
GUI.draw_multiline_text(screen, "Sensor:")
if sensor_data is sensor.DROP:
GUI.draw_multiline_text(screen, ' DROP!')
else:
GUI.draw_multiline_text(
screen, "\n".join([
" {0}: {1}".format(label, sensor_data[label])
for label in config.sensor_data_labels
]))
GUI.draw_multiline_text(screen, ("App:\n"
" Time : {0:.3f}\n"
" input_queue.size: {1}\n").format(
app_timer.elapsed,
len(input_queue),
))
GUI.draw_multiline_text(screen, (" precedents.size :" + (", ".join(
["{0}".format(len(precedents))
for precedents in precedents_dict]))))
GUI.draw_multiline_text(screen, "Estimator:")
GUI.draw_multiline_text(
screen,
' o',
color=GUI.color.green if submitted_estimator else GUI.color.red)
GUI.draw_multiline_text(screen, "Trainor:")
GUI.draw_multiline_text(
screen,
' o',
color=GUI.color.green if submitted_trainor else GUI.color.red)
GUI.draw_multiline_text(
screen, ("Profile:\n"
" Keyboard Input : {1:.4f}\n"
" Sensor Read : {2:.4f}\n"
" Submodule Control: {3:.4f}\n"
" GUI Update : {4:.4f}\n"
" Screen Update : {5:.4f}\n").format(
estimator.is_running, profile['keyboard_input'],
profile['sensor_read'], profile['submodule_control'],
profile['gui_update'], profile['screen_update']))
if fixed_loop.last_delay_time >= 0:
screen.blit(
GUI.make_text('Frame: Sync ({0:.3f})'.format(
fixed_loop.last_delay_time),
color=GUI.color.green), (0, 463))
else:
screen.blit(
GUI.make_text('Frame: Busy ({0:.3f})'.format(
fixed_loop.last_delay_time),
color=GUI.color.red), (0, 463))
if sensor_data is sensor.DROP:
screen.blit(GUI.make_text('Sensor: Busy', color=GUI.color.red),
(240, 463))
else:
screen.blit(GUI.make_text('Sensor: Sync', color=GUI.color.green),
(240, 463))
pygame.draw.rect(screen, (0x11, 0x11, 0x11),
pygame.Rect(10, 30, 380, 380))
meta, plots = facenet.make_visualized_graph_plots(
precedents_dict, estimated)
if meta is not None:
scale = meta.max - meta.min
a = 190 / scale.max()
root = np.array([10 + 190, 30 + 190])
for plot in plots:
position = root + plot.position * a
position = position.astype(np.int64)
if plot.supervised_state is not None:
pygame.draw.circle(screen,
config.id2color[plot.supervised_state],
position, 6)
if plot.estimated_state is not None:
pygame.draw.circle(screen,
config.id2color[plot.estimated_state],
position, 4)
profile['gui_update'] = app_timer.lap()
# End Update GUI Elements ---
pygame.display.update()
profile['screen_update'] = app_timer.lap()
if not fixed_loop.sync():
fixed_loop.reset()
pygame.quit()
exit()
def run(cam_source, yolo_engine, tf_weight_pickle, dnnweaver2_weight_pickle, in_videofile, out_videofile):
# Synchronous queues
frame_q = Queue(maxsize=1)
bbox_q = Queue(maxsize=1)
kill_q = Queue(maxsize=1)
key_q = Queue(maxsize=1)
num_processes = 2
done_q = Queue(maxsize=num_processes)
# Multiprocessing locks
frame_l = Lock()
bbox_l = Lock()
key_l = Lock()
# Drone management process
if cam_source == "drone":
droneProcess = Process(target=drone_control, args=(frame_q, frame_l, bbox_q, bbox_l, key_q, key_l, kill_q, done_q, ))
droneProcess.start()
elif cam_source == "webcam":
webcamProcess = Process(target=webcam_control, args=(frame_q, frame_l, bbox_q, bbox_l, kill_q, done_q, ))
webcamProcess.start()
elif cam_source == "videofile":
videofileProcess = Process(target=videofile_control, args=(frame_q, frame_l, bbox_q, bbox_l, kill_q, done_q, in_videofile, out_videofile, ))
videofileProcess.start()
# Object detection process using YOLO algorithm
if yolo_engine == "tf-cpu":
proc = "cpu"
elif yolo_engine == "tf-gpu":
proc = "gpu"
elif yolo_engine == "dnnweaver2":
proc = "gpu"
detectionProcess = Process(target=detection, args=(yolo_engine, tf_weight_pickle, dnnweaver2_weight_pickle, frame_q, frame_l, bbox_q, bbox_l, kill_q, done_q, proc, ))
# detectionProcess = Process(target=detection, args=(yolo_engine, tf_weight_pickle, dnnweaver2_weight_pickle, frame_q, frame_l, bbox_q, bbox_l, kill_q, done_q, proc, True, ))
detectionProcess.start()
# Keyboard input handler
inkey = GetKey()
thread_print ("Keyboard Input Handler Starts")
while True:
try:
key = inkey()
key = ord(key)
with key_l:
if key_q.empty():
key_q.put(key)
if key == 101: # key = 'e'
break
except KeyboardInterrupt:
break
thread_print ("Keyboard Input Handler Ends")
# Notifying all processes/threads to die
kill_q.put(True)
print ("Sent KILL Signal")
# Wait the processes to end
while done_q.qsize() != num_processes:
sleep(0.5)
# Flush all entires in queueus
drain_queue([frame_q, bbox_q, kill_q, key_q])
if cam_source == "drone":
droneProcess.join()
elif cam_source == "webcam":
webcamProcess.join()
detectionProcess.join()
if args.username and args.password:
es = Elasticsearch(url.netloc,
request_timeout=5,
timeout=args.timeout,
http_auth=(args.username, args.password))
if url.scheme == 'https':
es = Elasticsearch(url.netloc,
use_ssl=True,
verify_certs=False,
request_timeout=5,
timeout=args.timeout,
http_auth=(args.username, args.password))
else:
es = Elasticsearch(url.netloc, request_timeout=5, timeout=args.timeout)
if url.scheme == 'https':
es = Elasticsearch(url.netloc,
use_ssl=True,
verify_certs=False,
request_timeout=5,
timeout=args.timeout)
outq = Queue(maxsize=50000)
alldone = Event()
dumpproc = Process(target=dump, args=(es, outq, alldone))
dumpproc.daemon = True
dumpproc.start()
while not alldone.is_set() or outq.qsize() > 0:
try:
print json.dumps(outq.get(block=False))
except:
time.sleep(0.1)
# vgg16_image_name_to_features
evaluator = Evaluator(
model,
data_path,
image_path,
image_name_to_features_filename=object_image_features_filename)
image_names = evaluator.full_data['image_names'].tolist()
tweet_list = evaluator.full_data['tweets'].tolist()
tweet_dict = {}
for image_arg, image_name in tqdm(enumerate(image_names)):
tweet = str(tweet_list[image_arg])
tweet_dict[image_name] = tweet
if __name__ == "__main__":
ts = time()
seconds = 0
queue = Queue()
for img_file in image_names:
queue.put(img_file)
print(queue.qsize())
procs = [
Process(target=run_main, args=[i, queue]) for i in range(num_processes)
]
for p in procs:
p.start()
for p in procs:
p.join()
print('Took {}s'.format(time() - ts))
class WorkerProcess(object):
def __init__(self, idnum, topic, collname, in_counter_value,
out_counter_value, drop_counter_value, queue_maxsize,
mongodb_host, mongodb_port, mongodb_name, nodename_prefix):
self.name = "WorkerProcess-%4d-%s" % (idnum, topic)
self.id = idnum
self.topic = topic
self.collname = collname
self.queue = Queue(queue_maxsize)
self.out_counter = Counter(out_counter_value)
self.in_counter = Counter(in_counter_value)
self.drop_counter = Counter(drop_counter_value)
self.worker_out_counter = Counter()
self.worker_in_counter = Counter()
self.worker_drop_counter = Counter()
self.mongodb_host = mongodb_host
self.mongodb_port = mongodb_port
self.mongodb_name = mongodb_name
self.nodename_prefix = nodename_prefix
self.quit = Value('i', 0)
self.process = Process(name=self.name, target=self.run)
self.process.start()
def init(self):
global use_setproctitle
if use_setproctitle:
setproctitle("mongodb_log %s" % self.topic)
self.mongoconn = Connection(self.mongodb_host, self.mongodb_port)
self.mongodb = self.mongoconn[self.mongodb_name]
self.mongodb.set_profiling_level = SLOW_ONLY
self.collection = self.mongodb[self.collname]
self.collection.count()
self.queue.cancel_join_thread()
rospy.init_node(WORKER_NODE_NAME %
(self.nodename_prefix, self.id, self.collname),
anonymous=False)
self.subscriber = None
while not self.subscriber:
try:
msg_class, real_topic, msg_eval = rostopic.get_topic_class(
self.topic, blocking=True)
self.subscriber = rospy.Subscriber(real_topic, msg_class,
self.enqueue, self.topic)
except rostopic.ROSTopicIOException:
print("FAILED to subscribe, will keep trying %s" % self.name)
time.sleep(randint(1, 10))
except rospy.ROSInitException:
print("FAILED to initialize, will keep trying %s" % self.name)
time.sleep(randint(1, 10))
self.subscriber = None
def run(self):
self.init()
print("ACTIVE: %s" % self.name)
# run the thread
self.dequeue()
# free connection
# self.mongoconn.end_request()
def is_quit(self):
return self.quit.value == 1
def shutdown(self):
if not self.is_quit():
#print("SHUTDOWN %s qsize %d" % (self.name, self.queue.qsize()))
self.quit.value = 1
self.queue.put("shutdown")
while not self.queue.empty():
sleep(0.1)
#print("JOIN %s qsize %d" % (self.name, self.queue.qsize()))
self.process.join()
self.process.terminate()
def sanitize_value(self, v):
if isinstance(v, rospy.Message):
return self.message_to_dict(v)
elif isinstance(v, Time):
t = datetime.fromtimestamp(v.secs)
return t + timedelta(microseconds=v.nsecs / 1000.)
elif isinstance(v, Duration):
return v.secs + v.nsecs / 1000000000.
elif isinstance(v, list):
return [self.sanitize_value(t) for t in v]
else:
return v
def message_to_dict(self, val):
d = {}
for f in val.__slots__:
d[f] = self.sanitize_value(getattr(val, f))
return d
def qsize(self):
return self.queue.qsize()
def enqueue(self, data, topic, current_time=None):
if not self.is_quit():
if self.queue.full():
try:
self.queue.get_nowait()
self.drop_counter.increment()
self.worker_drop_counter.increment()
except Empty:
pass
self.queue.put((topic, data, current_time or datetime.now()))
self.in_counter.increment()
self.worker_in_counter.increment()
def dequeue(self):
while not self.is_quit():
t = None
try:
t = self.queue.get(True)
except IOError:
# Anticipate Ctrl-C
#print("Quit W1: %s" % self.name)
self.quit.value = 1
break
if isinstance(t, tuple):
self.out_counter.increment()
self.worker_out_counter.increment()
topic = t[0]
msg = t[1]
ctime = t[2]
if isinstance(msg, rospy.Message):
doc = self.message_to_dict(msg)
doc["__recorded"] = ctime or datetime.now()
doc["__topic"] = topic
try:
#print(self.sep + threading.current_thread().getName() + "@" + topic+": ")
#pprint.pprint(doc)
self.collection.insert(doc)
except InvalidDocument, e:
print("InvalidDocument " + current_process().name +
"@" + topic + ": \n")
print e
except InvalidStringData, e:
print("InvalidStringData " + current_process().name +
"@" + topic + ": \n")
print e
else:
from multiprocessing import Queue
#创建队列
q = Queue(3)
q.put(1)
print(q.full())
q.put(2)
q.put(3)
print(q.full())
#设置超时时间为3秒
q.put(4, True, 3)
print(q.get())
print('队列中还有%d条消息' % q.qsize())
print(q.empty())
q.close()
class Reactor(object):
"""
Reactor and container of genetic algorithm
"""
def __init__(self, size, chromolen, alphabet, fitnessfunc, pmut, sortfunc=None, engine="process"):
"""
Params:
size (int): Number of individuals in reactor population
chromolen (int): Number of components in a individual (chromosome)
alphabet (string[]): List with the string representation of each possible gene
fitnessfunc (func): Function receiving a chromosome and returning a int
pmut (float): Probability of mutating an individual in each population
sortfunc (func): Overrides the default sorting function based on fitness
engine (string): Type of paralellism used ("process" or "thread")
"""
self.size = size
self.chromolen = chromolen
self.alphabet = alphabet
self.fitness = fitnessfunc
self.pmut = pmut
if sortfunc is None:
self.sortfunc = self.fitness
else:
self.sortfunc = sortfunc
self.engine = engine
self.cores = 1
self.running_cores = []
self.result_queue = Queue()
self.population = randpopulation(self.size, self.chromolen, self.alphabet)
def addCores(self, cores):
if self.cores + cores >= 1:
self.cores += cores
return self
def run(self, steps, ongeneration=None):
#TODO: add a sync=True parameter to run a task in local thread or all task in paralell
# create cores
if self.engine.lower() == "process":
from multiprocessing import Process as Task
elif self.engine.lower() == "thread":
from threading import Thread as Task
else:
raise ValueError("Unknown engine {}".format(self.engine))
for i in range(self.cores):
# tasks.append(Process(target=self._loop, args=(i, steps, ongeneration, results)))
self.running_cores.append(Task(target=self._loop, args=(i, steps, ongeneration, self.result_queue)))
# launch
for task in self.running_cores:
task.start()
# create local process
# tasks.append(Process(target=self._loop, args=(0, steps, ongeneration, results)))
# self._loop(0, steps, ongeneration, results)
# join launched processes
# for task in tasks:
# task.join()
# return [results.get() for _ in range(results.qsize())]
return self
def get_results(self):
for task in range(len(self.running_cores)):
self.running_cores.pop().join()
return [self.result_queue.get() for _ in range(self.result_queue.qsize())]
def _loop(self, core, steps, ongeneration=None, queue=None):
pop = self.population[:] # local copy
top = None
for step in range(steps):
pop, best = cycle(
population=pop,
alphabet=self.alphabet,
pmut=self.pmut,
sortfunc=self.sortfunc
)
if ongeneration is not None:
ongeneration(step, core, best)
if top is None:
top = best
if self.fitness(best) > self.fitness(top):
top = best
if queue is not None:
queue.put(top)
return
else:
return top
print(accession, datetime.now(), q)
if len(variant_coordinaten) != 0:
#Erstellen des Graphen, sowie das dranhängen der Knoten und Kanten
process_these_peptides = get_the_peptide_to_be_parsed(
zwischensumme_list, variants_on_peptidelevel,
aa_seq, SQL_protein_ID, SQL_organism_ID,
variant_coordinaten, variant_char, variants,
accession)
i = 0
while i < len(process_these_peptides):
worker_queue.put(process_these_peptides[i])
i = i + 1
print(accession, datetime.now(), q, "len var",
len(variants), worker_queue.qsize(),
"are in the queue")
#process_these_peptides = process_these_peptides[19:23]
protein_acc = line.rstrip('\n')
protein_seq = ""
q = q + 1
if q % 2 == 0:
conn.commit()
else:
protein_seq += line.rstrip('\n')
procs = []
#letztes protein
def reade(q: Queue):
for i in range(q.qsize()):
print(f'读取{q.get()}')
time.sleep(1)
threads_count = 1 + (new_pages_count - done) / 2
if threads_count > max_threads_count:
threads_count = max_threads_count
workers = [
Process(target=collect_childs, args=(queue, results_queue))
for i in xrange(threads_count)
]
for w in workers:
w.daemon = True
[w.start() for w in workers]
for w in workers:
w.join(timeout=0.1)
for _ in xrange(results_queue.qsize() - done):
bar.next()
done += 1
recursion += 1
if recursion > max_recursion:
break
while not results_queue.empty():
childs += results_queue.get()
bar.finish()
with open("all_tested_links", 'a+') as f:
for page in CACHE:
f.write(page + "\n")
queue.put([4, 5])
queue.put({'a': 1})
# 不能一次性添加多个数据
# queue.put(1, 'a', (1, 2), [4, 5], {'a': 1})
# 当队列消息已放满，再put()数据会等待
# queue.put(2323)
# 使用put_nowait()不等待，直接报错
# queue.put_nowait(2323)
# 查看队列是否满了
print('满了没？', queue.full())
# 查看队列消息个数
print('个数:', queue.qsize())
# 获取队列消息
value = queue.get()
value = queue.get()
print(value)
# 当队列消息get完后再次使用get会等待
value = queue.get()
value = queue.get()
value = queue.get()
# 再次执行一下代码会等待，只有队列put值后才会取
# 使用get_nowait()不等待，直接报错，不建议使用
# value = queue.get()
# value = queue.get_nowait()
subs.setDaemon(True)
pubs = FacePublisher("tcp://127.0.0.1:812354", face_msg_queue)
pubs.setDaemon(True)
yololock = Lock()
facelock = Lock()
face_proc = []
for i in range(procnum):
face_proc.append(
FaceRecognition(load_mode_finish_q, yolo_msg_queue, yololock,
face_msg_queue, facelock))
for i in range(procnum):
face_proc[i].start()
while load_mode_finish_q.qsize() < procnum:
#print(load_mode_finish_q.get())
time.sleep(1)
pubs.start()
subs.start()
for i in range(len(face_proc)):
load_mode_finish_q.get()
config = edict()
config.mode = 16
config.data_dir = 1e-4
config.classifier = 'KNN'
config.use_split_dataset = 0
config.test_data_dir = ''
from multiprocessing import Queue,Process,Pool
from multiprocessing import Manager
import time
import os
import random
'''
Queue.qsize()：返回当前队列包含的消息数量；
Queue.empty()：如果队列为空，返回True，反之False ；
Queue.full()：如果队列满了，返回True,反之False；
Queue.get([block[, timeout]])：获取队列中的⼀条消息，然后将其从列队
中移除，block默认值为True；
1）如果block使⽤默认值，且没有设置timeout（单位秒），消息列队如果为
空，此时程序将被阻塞（停在读取状态），直到从消息列队读到消息为⽌，
如果设置了timeout，则会等待timeout秒，若还没读取到任何消息，则抛
出"Queue.Empty"异常；
2）如果block值为False，消息列队如果为空，则会⽴刻抛
出"Queue.Empty"异常；
Queue.get_nowait()：相当Queue.get(False)；
Queue.put(item,[block[, timeout]])：将item消息写⼊队列，block默认值
为True；
1）如果block使⽤默认值，且没有设置timeout（单位秒），消息列队如果已
经没有空间可写⼊，此时程序将被阻塞（停在写⼊状态），直到从消息列队
腾出空间为⽌，如果设置了timeout，则会等待timeout秒，若还没空间，则抛
出"Queue.Full"异常；
2）如果block值为False，消息列队如果没有空间可写⼊，则会⽴刻抛
出"Queue.Full"异常；
Queue.put_nowait(item)：相当Queue.put(item, False)；
'''
#Process进程间的通信 使用Queue()就行
class NanoCET(BaseExperiment):
""" Experiment class for performing a nanoCET measurement."""
BACKGROUND_NO_CORRECTION = 0 # No backround correction
BACKGROUND_SINGLE_SNAP = 1
def __init__(self, filename=None):
super().__init__() # Initialize base class
self.logger = get_logger(name=__name__)
self.worker_saver_queue = Queue()
self.load_configuration(filename)
self.dropped_frames = 0
self.keep_acquiring = True
self.acquiring = False # Status of the acquisition
self.camera = None # This will hold the model for the camera
self.current_height = None
self.current_width = None
self.max_width = None
self.max_height = None
self.background = None
self.temp_image = None # Temporary image, used to quickly have access to 'some' data and display it to the user
self.movie_buffer = None # Holds few frames of the movie in order to be able to do some analysis, save later, etc.
self.last_index = 0 # Last index used for storing to the movie buffer
self.stream_saving_running = False
self.async_threads = [] # List holding all the threads spawn
self.stream_saving_process = None
self.do_background_correction = False
self.background_method = self.BACKGROUND_SINGLE_SNAP
self._stop_evet = Event()
self.waterfall_index = 0
self.locations_queue = Queue()
# self.connect(self.print_me, 'free_run')
def initialize_camera(self):
""" Initializes the camera to be used to acquire data. The information on the camera should be provided in the
configuration file and loaded with :meth:`~self.load_configuration`. It will load the camera assuming
it is located in pynta/model/cameras/[model].
.. todo:: Define how to load models from outside of pynta. E.g. from a user-specified folder.
"""
try:
self.logger.info('Importing camera model {}'.format(
self.config['camera']['model']))
self.logger.debug('pynta.model.cameras.' +
self.config['camera']['model'])
camera_model_to_import = 'pynta.model.cameras.' + self.config[
'camera']['model']
cam_module = importlib.import_module(camera_model_to_import)
except ModuleNotFoundError:
self.logger.error(
'The model {} for the camera was not found'.format(
self.config['camera']['model']))
raise
except:
self.logger.exception('Unhandled exception')
raise
cam_init_arguments = self.config['camera']['init']
if 'extra_args' in self.config['camera']:
self.logger.info('Initializing camera with extra arguments')
self.logger.debug('cam_module.camera({}, {})'.format(
cam_init_arguments, self.config['camera']['extra_args']))
self.camera = cam_module.camera(
cam_init_arguments, *self.config['Camera']['extra_args'])
else:
self.logger.info('Initializing camera without extra arguments')
self.logger.debug(
'cam_module.camera({})'.format(cam_init_arguments))
self.camera = cam_module.camera(cam_init_arguments)
self.current_width, self.current_height = self.camera.getSize()
self.logger.info('Camera sensor ROI: {}px X {}px'.format(
self.current_width, self.current_height))
self.max_width = self.camera.GetCCDWidth()
self.max_height = self.camera.GetCCDHeight()
self.logger.info('Camera sensor size: {}px X {}px'.format(
self.max_width, self.max_height))
self.camera.initializeCamera()
@check_camera
@check_not_acquiring
def snap_background(self):
""" Snaps an image that will be stored as background.
"""
self.logger.info('Acquiring background image')
self.camera.configure(self.config['camera'])
self.camera.setAcquisitionMode(self.camera.MODE_SINGLE_SHOT)
self.camera.triggerCamera()
self.background = self.camera.readCamera()[-1]
self.logger.debug('Got an image of {} pixels'.format(
self.backgound.shape))
@check_camera
@check_not_acquiring
def set_roi(self, x, y, width, height):
""" Sets the region of interest of the camera, provided that the camera supports cropping. All the technicalities
should be addressed on the camera model, not in this method.
:param int x: horizontal position for the start of the cropping
:param int y: vertical position for the start of the cropping
:param int width: width in pixels for cropping
:param int height: height in pixels for the cropping
:raises ValueError: if either dimension of the cropping goes out of the camera total amount of pixels
:returns: The final cropping dimensions, it may be that the camera limits the user desires
"""
X = [x, x + width - 1]
Y = [y, y + height - 1]
self.logger.debug('Setting new camera ROI to x={},y={}'.format(X, Y))
Nx, Ny = self.camera.setROI(X, Y)
self.current_width, self.current_height = self.camera.getSize()
self.logger.debug('New camera width: {}px, height: {}px'.format(
self.current_width, self.current_height))
self.tempimage = np.zeros((Nx, Ny))
@check_camera
@check_not_acquiring
def clear_roi(self):
""" Clears the region of interest and returns to the full frame of the camera.
"""
self.logger.info('Clearing ROI settings')
self.camera.setROI(1, 1, self.max_width, self.max_height)
@check_camera
@check_not_acquiring
@make_async_thread
def snap(self):
""" Snap a single frame. It is not an asynchronous method. To make it async, it should be placed within
a different thread.
"""
self.logger.info('Snapping a picture')
self.camera.configure(self.config['camera'])
self.camera.setAcquisitionMode(self.camera.MODE_SINGLE_SHOT)
self.camera.triggerCamera()
self.check_background()
data = self.camera.readCamera()[-1]
self.queue.put({'topic': 'snap', 'data': data})
self.temp_image = data[-1]
self.logger.debug('Got an image of {} pixels'.format(
self.temp_image.shape))
@make_async_thread
@check_not_acquiring
@check_camera
def start_free_run(self):
""" Starts continuous acquisition from the camera, but it is not being saved. This method is the workhorse
of the program. While this method runs in its thread, it will broadcast the images to be consumed by other
methods. In this way it is possible to continuously save to hard drive, track particles, etc.
"""
self.logger.info('Starting a free run acquisition')
first = True
self.keep_acquiring = True # Change this attribute to stop the acquisition
self.camera.configure(self.config['camera'])
while self.keep_acquiring:
if first:
self.logger.debug('First frame of a free_run')
self.camera.setAcquisitionMode(self.camera.MODE_CONTINUOUS)
self.camera.triggerCamera() # Triggers the camera only once
first = False
data = self.camera.readCamera()
self.logger.debug('Got {} new frames'.format(len(data)))
for img in data:
if self.do_background_correction and self.background_method == self.BACKGROUND_SINGLE_SNAP:
img -= self.background
# This will broadcast the data just acquired with the current timestamp
# The timestamp is very unreliable, especially if the camera has a frame grabber.
self.queue.put({
'topic': 'free_run',
'data': [time.time(), img]
})
if self._stop_evet.is_set():
break
self.temp_image = data[-1]
self.camera.stopAcq()
def stop_free_run(self):
self._stop_evet.set()
def save_image(self):
""" Saves the last acquired image. The file to which it is going to be saved is defined in the config.
"""
if self.temp_image:
self.logger.info('Saving last acquired image')
# Data will be appended to existing file
file_name = self.config['saving']['filename_photo'] + '.hdf5'
file_dir = self.config['saving']['directory']
if not os.path.exists(file_dir):
os.makedirs(file_dir)
self.logger.debug('Created directory {}'.format(file_dir))
with h5py.File(os.path.join(file_dir, file_name), "a") as f:
now = str(datetime.now())
g = f.create_group(now)
g.create_dataset('image', data=self.temp_image)
g.create_dataset('metadata', data=json.dumps(self.config))
f.flush()
self.logger.debug('Saved image to {}'.format(
os.path.join(file_dir, file_name)))
else:
self.logger.warning(
'Tried to save an image, but no image was acquired yet.')
def add_to_stream_queue(self, data):
""" This method is a buffer between the publisher and the ``save_stream`` method. The idea is that in order
to be quick (saving to disk may be slow), whatever the publisher sends will be added to a Queue. Another
process will read from the queue and save it to disk on a separate process.
"""
img = data[1]
self.worker_saver_queue.put(img)
def save_stream(self):
""" Saves the queue to a file continuously. This is an async function, that can be triggered before starting
the stream. It relies on the multiprocess library. It uses a queue in order to get the data to be saved.
In normal operation, it should be used together with ``add_to_stream_queue``.
"""
if self.save_stream_running:
self.logger.warning('Tried to start a new instance of save stream')
raise StreamSavingRunning(
'You tried to start a new process for stream saving')
self.logger.info('Starting to save the stream')
file_name = self.config['saving']['filename_video'] + '.hdf5'
file_dir = self.config['saving']['directory']
if not os.path.exists(file_dir):
os.makedirs(file_dir)
self.logger.debug('Created directory {}'.format(file_dir))
file_path = os.path.join(file_dir, file_name)
self.stream_saving_process = Process(target=worker_saver,
args=(file_path,
json.dumps(self.config),
self.worker_saver_queue))
self.stream_saving_process.start()
self.logger.debug('Started the stream saving process')
def stop_save_stream(self):
""" Stops saving the stream.
"""
self.logger.info('Stopping the saving stream process')
if not self.save_stream_running:
self.logger.warning(
'The saving stream is not running. Nothing will be done.')
return
self.worker_saver_queue.put('Exit')
@property
def save_stream_running(self):
if self.stream_saving_process is not None:
return self.stream_saving_process.is_alive()
return False
def empty_queue(self):
""" Empties the queue where the data from the movie is being stored.
"""
self.logger.info('Clearing the saver queue')
while not self.worker_saver_queue.empty(
) or self.worker_saver_queue.qsize() > 0:
self.worker_saver_queue.get()
self.logger.debug('Queue cleared')
def calculate_waterfall(self, image):
""" A waterfall is the product of summing together all the vertical values of an image and displaying them
as lines on a 2D image. It is how spectrometers normally work. A waterfall can be produced either by binning the
image in the vertical direction directly at the camera, or by doing it in software.
The first has the advantage of speeding up the readout process. The latter has the advantage of working with any
camera.
This method will work either with 1D arrays or with 2D arrays and will generate a stack of lines.
"""
if self.waterfall_index == self.config['waterfall'][
'length_waterfall']:
self.waterfall_data = np.zeros(
(self.config['waterfall']['length_waterfall'],
self.camera.width))
self.waterfall_index = 0
center_pixel = np.int(self.camera.height /
2) # Calculates the center of the image
vbinhalf = np.int(self.config['waterfall']['vertical_bin'])
if vbinhalf >= self.current_height / 2 - 1:
wf = np.array([np.sum(image, 1)])
else:
wf = np.array([
np.sum(
image[:, center_pixel - vbinhalf:center_pixel + vbinhalf],
1)
])
self.waterfall_data[self.waterfall_index, :] = wf
self.waterfall_index += 1
self.queue.put({'topic': 'waterfall_data', 'data': wf})
def check_background(self):
""" Checks whether the background is set.
"""
if self.do_background_correction:
self.logger.info('Setting up the background corretion')
if self.background_method == self.BACKGROUND_SINGLE_SNAP:
self.logger.debug('Bacground single snap')
if self.background is None or self.background.shape != [
self.current_width, self.current_height
]:
self.logger.warning(
'Background not set. Defaulting to no background...')
self.background = None
self.do_background_correction = False
def calculate_positions_image(self, image):
""" Calculates the positions of the particles on an image. It used the trackpy package, which may not be
installed by default.
"""
locations = np.random.rand((10, 2))
# if trackpy:
# locations = tp.locate(image, self.config['trackpy']['size'])
# self.queue.put({'topic': 'trackpy', 'data': locations})
# else:
# self.logger.warn('Trackpy is not detected and is the only tracking algorithm available')
# raise TrackpyNotInstalled('Trackpy is not installed on this computer')
def put_position_to_queue(self, locations):
""" Accumulates the locations into a Queue in order to use them in a generator.
"""
self.locations_queue.put(locations)
def link_trajecktories(self, locations):
print(locations)
def __exit__(self, *args):
super(NanoCET, self).__exit__(*args)
if not self.worker_saver_queue.empty():
self.logger.info('Emptying the saver queue')
self.logger.debug('There are {} elements in the queue'.format(
self.worker_saver_queue.qsize()))
while not self.worker_saver_queue.empty():
self.worker_saver_queue.get()
class SafariBooks:
HEADERS = {
"accept":
"text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,image/apng,*/*;q=0.8",
"accept-encoding":
"gzip, deflate, br",
"accept-language":
"it-IT,it;q=0.9,en-US;q=0.8,en;q=0.7",
"cache-control":
"no-cache",
"cookie":
"",
"pragma":
"no-cache",
"referer":
"https://www.safaribooksonline.com/home/",
"upgrade-insecure-requests":
"1",
"user-agent":
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) "
"Chrome/62.0.3202.94 Safari/537.36"
}
BASE_URL = "https://www.safaribooksonline.com"
LOGIN_URL = BASE_URL + "/accounts/login/"
API_TEMPLATE = BASE_URL + "/api/v1/book/{0}/"
BASE_01_HTML = "<!DOCTYPE html>\n" \
"<html lang=\"en\" xml:lang=\"en\" xmlns=\"http://www.w3.org/1999/xhtml\"" \
" xmlns:xsi=\"http://www.w3.org/2001/XMLSchema-instance\"" \
" xsi:schemaLocation=\"http://www.w3.org/2002/06/xhtml2/" \
" http://www.w3.org/MarkUp/SCHEMA/xhtml2.xsd\"" \
" xmlns:epub=\"http://www.idpf.org/2007/ops\">\n" \
"<head>\n" \
"{0}\n" \
"<style type=\"text/css\">" \
"body{{background-color:#fbfbfb!important;margin:1em;}}" \
"#sbo-rt-content *{{text-indent:0pt!important;}}#sbo-rt-content .bq{{margin-right:1em!important;}}"
KINDLE_HTML = "#sbo-rt-content *{{word-wrap:break-word!important;" \
"word-break:break-word!important;}}#sbo-rt-content table,#sbo-rt-content pre" \
"{{overflow-x:unset!important;overflow:unset!important;" \
"overflow-y:unset!important;white-space:pre-wrap!important;}}"
BASE_02_HTML = "</style>" \
"</head>\n" \
"<body>{1}</body>\n</html>"
CONTAINER_XML = "<?xml version=\"1.0\"?>" \
"<container version=\"1.0\" xmlns=\"urn:oasis:names:tc:opendocument:xmlns:container\">" \
"<rootfiles>" \
"<rootfile full-path=\"OEBPS/content.opf\" media-type=\"application/oebps-package+xml\" />" \
"</rootfiles>" \
"</container>"
# Format: ID, Title, Authors, Description, Subjects, Publisher, Rights, Date, CoverId, MANIFEST, SPINE, CoverUrl
CONTENT_OPF = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n" \
"<package xmlns=\"http://www.idpf.org/2007/opf\" unique-identifier=\"bookid\" version=\"2.0\" >\n" \
"<metadata xmlns:dc=\"http://purl.org/dc/elements/1.1/\" " \
" xmlns:opf=\"http://www.idpf.org/2007/opf\">\n"\
"<dc:title>{1}</dc:title>\n" \
"{2}\n" \
"<dc:description>{3}</dc:description>\n" \
"{4}" \
"<dc:publisher>{5}</dc:publisher>\n" \
"<dc:rights>{6}</dc:rights>\n" \
"<dc:language>en-US</dc:language>\n" \
"<dc:date>{7}</dc:date>\n" \
"<dc:identifier id=\"bookid\">{0}</dc:identifier>\n" \
"<meta name=\"cover\" content=\"{8}\"/>\n" \
"</metadata>\n" \
"<manifest>\n" \
"<item id=\"ncx\" href=\"toc.ncx\" media-type=\"application/x-dtbncx+xml\" />\n" \
"{9}\n" \
"</manifest>\n" \
"<spine toc=\"ncx\">\n{10}</spine>\n" \
"<guide><reference href=\"{11}\" title=\"Cover\" type=\"cover\" /></guide>\n" \
"</package>"
# Format: ID, Depth, Title, Author, NAVMAP
TOC_NCX = "<?xml version=\"1.0\" encoding=\"UTF-8\" standalone=\"no\" ?>" \
"<!DOCTYPE ncx PUBLIC \"-//NISO//DTD ncx 2005-1//EN\"" \
" \"http://www.daisy.org/z3986/2005/ncx-2005-1.dtd\">" \
"<ncx xmlns=\"http://www.daisy.org/z3986/2005/ncx/\" version=\"2005-1\">" \
"<head>" \
"<meta content=\"ID:ISBN:{0}\" name=\"dtb:uid\"/>" \
"<meta content=\"{1}\" name=\"dtb:depth\"/>" \
"<meta content=\"0\" name=\"dtb:totalPageCount\"/>" \
"<meta content=\"0\" name=\"dtb:maxPageNumber\"/>" \
"</head>" \
"<docTitle><text>{2}</text></docTitle>" \
"<docAuthor><text>{3}</text></docAuthor>" \
"<navMap>{4}</navMap>" \
"</ncx>"
def __init__(self, args):
self.args = args
self.display = Display("info_%s.log" % escape(args.bookid))
self.display.intro()
self.cookies = {}
if not args.cred:
if not os.path.isfile(COOKIES_FILE):
self.display.exit(
"Login: unable to find cookies file.\n"
" Please use the --cred option to perform the login.")
self.cookies = json.load(open(COOKIES_FILE))
else:
self.display.info("Logging into Safari Books Online...",
state=True)
self.do_login(
*[c.replace("'", "").replace('"', "") for c in args.cred])
if not args.no_cookies:
json.dump(self.cookies, open(COOKIES_FILE, "w"))
self.book_id = args.bookid
self.api_url = self.API_TEMPLATE.format(self.book_id)
self.display.info("Retrieving book info...")
self.book_info = self.get_book_info()
self.display.book_info(self.book_info)
self.display.info("Retrieving book chapters...")
self.book_chapters = self.get_book_chapters()
self.chapters_queue = self.book_chapters[:]
if len(self.book_chapters) > sys.getrecursionlimit():
sys.setrecursionlimit(len(self.book_chapters))
self.book_title = self.book_info["title"]
self.base_url = self.book_info["web_url"]
self.clean_book_title = "".join(
self.escape_dirname(
self.book_title).split(",")[:2]) + " ({0})".format(
self.escape_dirname(", ".join(
a["name"] for a in self.book_info["authors"][:2]),
clean_space=True))
books_dir = os.path.join(PATH, "Books")
if not os.path.isdir(books_dir):
os.mkdir(books_dir)
self.BOOK_PATH = os.path.join(books_dir, self.clean_book_title)
self.css_path = ""
self.images_path = ""
self.create_dirs()
self.display.info("Output directory:\n %s" % self.BOOK_PATH)
self.chapter_title = ""
self.filename = ""
self.css = []
self.images = []
self.display.info("Downloading book contents... (%s chapters)" %
len(self.book_chapters),
state=True)
self.BASE_HTML = self.BASE_01_HTML + (
self.KINDLE_HTML if not args.no_kindle else "") + self.BASE_02_HTML
self.cover = False
self.get()
if not self.cover:
self.cover = self.get_default_cover()
cover_html = self.parse_html(
html.fromstring(
"<div id=\"sbo-rt-content\"><img src=\"Images/{0}\"></div>"
.format(self.cover)), True)
self.book_chapters = [{
"filename": "default_cover.xhtml",
"title": "Cover"
}] + self.book_chapters
self.filename = self.book_chapters[0]["filename"]
self.save_page_html(cover_html)
self.css_done_queue = Queue(
0) if "win" not in sys.platform else WinQueue()
self.display.info("Downloading book CSSs... (%s files)" %
len(self.css),
state=True)
self.collect_css()
self.images_done_queue = Queue(
0) if "win" not in sys.platform else WinQueue()
self.display.info("Downloading book images... (%s files)" %
len(self.images),
state=True)
self.collect_images()
self.display.info("Creating EPUB file...", state=True)
self.create_epub()
if not args.no_cookies:
json.dump(self.cookies, open(COOKIES_FILE, "w"))
self.display.done(os.path.join(self.BOOK_PATH, self.book_id + ".epub"))
self.display.unregister()
if not self.display.in_error and not args.log:
os.remove(self.display.log_file)
sys.exit(0)
def return_cookies(self):
return " ".join(
["{0}={1};".format(k, v) for k, v in self.cookies.items()])
def return_headers(self, url):
if "safaribooksonline" in urlsplit(url).netloc:
self.HEADERS["cookie"] = self.return_cookies()
else:
self.HEADERS["cookie"] = ""
return self.HEADERS
def update_cookies(self, jar):
for cookie in jar:
self.cookies.update({cookie.name: cookie.value})
def requests_provider(self,
url,
post=False,
data=None,
update_cookies=True,
**kwargs):
try:
response = getattr(requests, "post" if post else "get")(
url, headers=self.return_headers(url), data=data, **kwargs)
self.display.last_request = (url, data, kwargs,
response.status_code, "\n".join([
"\t{}: {}".format(*h)
for h in response.headers.items()
]), response.text)
except (requests.ConnectionError, requests.ConnectTimeout,
requests.RequestException) as request_exception:
self.display.error(str(request_exception))
return 0
if update_cookies:
self.update_cookies(response.cookies)
return response
def do_login(self, email, password):
response = self.requests_provider(self.BASE_URL)
if response == 0:
self.display.exit(
"Login: unable to reach Safari Books Online. Try again...")
csrf = []
try:
csrf = html.fromstring(response.text).xpath(
"//input[@name='csrfmiddlewaretoken'][@value]")
except (html.etree.ParseError,
html.etree.ParserError) as parsing_error:
self.display.error(parsing_error)
self.display.exit(
"Login: error trying to parse the home of Safari Books Online."
)
if not len(csrf):
self.display.exit("Login: no CSRF Token found in the page."
" Unable to continue the login."
" Try again...")
csrf = csrf[0].attrib["value"]
response = self.requests_provider(
self.LOGIN_URL,
post=True,
data=(("csrfmiddlewaretoken", ""), ("csrfmiddlewaretoken", csrf),
("email", email), ("password1", password),
("is_login_form", "true"), ("leaveblank", ""), ("dontchange",
"http://")),
allow_redirects=False)
if response == 0:
self.display.exit(
"Login: unable to perform auth to Safari Books Online.\n Try again..."
)
if response.status_code != 302:
try:
error_page = html.fromstring(response.text)
errors_message = error_page.xpath(
"//ul[@class='errorlist']//li/text()")
recaptcha = error_page.xpath("//div[@class='g-recaptcha']")
messages = ([" `%s`" % error for error in errors_message
if "password" in error or "email" in error] if len(errors_message) else []) +\
([" `ReCaptcha required (wait or do logout from the website).`"] if len(recaptcha) else[])
self.display.exit(
"Login: unable to perform auth login to Safari Books Online.\n"
+ self.display.SH_YELLOW + "[*]" +
self.display.SH_DEFAULT + " Details:\n"
"%s" % "\n".join(messages if len(messages) else
[" Unexpected error!"]))
except (html.etree.ParseError,
html.etree.ParserError) as parsing_error:
self.display.error(parsing_error)
self.display.exit(
"Login: your login went wrong and it encountered in an error"
" trying to parse the login details of Safari Books Online. Try again..."
)
def get_book_info(self):
response = self.requests_provider(self.api_url)
if response == 0:
self.display.exit("API: unable to retrieve book info.")
response = response.json()
if not isinstance(response, dict) or len(response.keys()) == 1:
self.display.exit(self.display.api_error(response))
if "last_chapter_read" in response:
del response["last_chapter_read"]
return response
def get_book_chapters(self, page=1):
response = self.requests_provider(
urljoin(self.api_url, "chapter/?page=%s" % page))
if response == 0:
self.display.exit("API: unable to retrieve book chapters.")
response = response.json()
if not isinstance(response, dict) or len(response.keys()) == 1:
self.display.exit(self.display.api_error(response))
if "results" not in response or not len(response["results"]):
self.display.exit("API: unable to retrieve book chapters.")
if response["count"] > sys.getrecursionlimit():
sys.setrecursionlimit(response["count"])
result = []
result.extend([
c for c in response["results"]
if "cover" in c["filename"] or "cover" in c["title"]
])
for c in result:
del response["results"][response["results"].index(c)]
result += response["results"]
return result + (self.get_book_chapters(page +
1) if response["next"] else [])
def get_default_cover(self):
response = self.requests_provider(self.book_info["cover"],
update_cookies=False,
stream=True)
if response == 0:
self.display.error("Error trying to retrieve the cover: %s" %
self.book_info["cover"])
return False
file_ext = response.headers["Content-Type"].split("/")[-1]
with open(os.path.join(self.images_path, "default_cover." + file_ext),
'wb') as i:
for chunk in response.iter_content(1024):
i.write(chunk)
return "default_cover." + file_ext
def get_html(self, url):
response = self.requests_provider(url)
if response == 0:
self.display.exit(
"Crawler: error trying to retrieve this page: %s (%s)\n From: %s"
% (self.filename, self.chapter_title, url))
root = None
try:
root = html.fromstring(response.text, base_url=self.BASE_URL)
except (html.etree.ParseError,
html.etree.ParserError) as parsing_error:
self.display.error(parsing_error)
self.display.exit(
"Crawler: error trying to parse this page: %s (%s)\n From: %s"
% (self.filename, self.chapter_title, url))
return root
@staticmethod
def url_is_absolute(url):
return bool(urlparse(url).netloc)
def link_replace(self, link):
if link:
if not self.url_is_absolute(link):
if "cover" in link or "images" in link or "graphics" in link or \
link[-3:] in ["jpg", "peg", "png", "gif"]:
link = urljoin(self.base_url, link)
if link not in self.images:
self.images.append(link)
self.display.log("Crawler: found a new image at %s" %
link)
image = link.split("/")[-1]
return "Images/" + image
return link.replace(".html", ".xhtml")
else:
if self.book_id in link:
return self.link_replace(link.split(self.book_id)[-1])
return link
@staticmethod
def get_cover(html_root):
images = html_root.xpath(
"//img[contains(@id, 'cover') or contains(@class, 'cover') or"
"contains(@name, 'cover') or contains(@src, 'cover')]")
if len(images):
return images[0]
divs = html_root.xpath(
"//div[contains(@id, 'cover') or contains(@class, 'cover') or"
"contains(@name, 'cover') or contains(@src, 'cover')]//img")
if len(divs):
return divs[0]
a = html_root.xpath(
"//a[contains(@id, 'cover') or contains(@class, 'cover') or"
"contains(@name, 'cover') or contains(@src, 'cover')]//img")
if len(a):
return a[0]
return None
def parse_html(self, root, first_page=False):
if random() > 0.5:
if len(root.xpath("//div[@class='controls']/a/text()")):
self.display.exit(self.display.api_error(" "))
book_content = root.xpath("//div[@id='sbo-rt-content']")
if not len(book_content):
self.display.exit(
"Parser: book content's corrupted or not present: %s (%s)" %
(self.filename, self.chapter_title))
page_css = ""
stylesheet_links = root.xpath("//link[@rel='stylesheet']")
if len(stylesheet_links):
stylesheet_count = 0
for s in stylesheet_links:
css_url = urljoin("https:", s.attrib["href"]) if s.attrib["href"][:2] == "//" \
else urljoin(self.base_url, s.attrib["href"])
if css_url not in self.css:
self.css.append(css_url)
self.display.log("Crawler: found a new CSS at %s" %
css_url)
page_css += "<link href=\"Styles/Style{0:0>2}.css\" " \
"rel=\"stylesheet\" type=\"text/css\" />\n".format(stylesheet_count)
stylesheet_count += 1
stylesheets = root.xpath("//style")
if len(stylesheets):
for css in stylesheets:
if "data-template" in css.attrib and len(
css.attrib["data-template"]):
css.text = css.attrib["data-template"]
del css.attrib["data-template"]
try:
page_css += html.tostring(
css, method="xml", encoding='unicode') + "\n"
except (html.etree.ParseError,
html.etree.ParserError) as parsing_error:
self.display.error(parsing_error)
self.display.exit(
"Parser: error trying to parse one CSS found in this page: %s (%s)"
% (self.filename, self.chapter_title))
# TODO: add all not covered tag for `link_replace` function
svg_image_tags = root.xpath("//image")
if len(svg_image_tags):
for img in svg_image_tags:
image_attr_href = [x for x in img.attrib.keys() if "href" in x]
if len(image_attr_href):
svg_url = img.attrib.get(image_attr_href[0])
svg_root = img.getparent().getparent()
new_img = svg_root.makeelement("img")
new_img.attrib.update({"src": svg_url})
svg_root.remove(img.getparent())
svg_root.append(new_img)
book_content = book_content[0]
book_content.rewrite_links(self.link_replace)
xhtml = None
try:
if first_page:
is_cover = self.get_cover(book_content)
if is_cover is not None:
page_css = "<style>" \
"body{display:table;position:absolute;margin:0!important;height:100%;width:100%;}" \
"#Cover{display:table-cell;vertical-align:middle;text-align:center;}" \
"img{height:90vh;margin-left:auto;margin-right:auto;}" \
"</style>"
cover_html = html.fromstring("<div id=\"Cover\"></div>")
cover_div = cover_html.xpath("//div")[0]
cover_img = cover_div.makeelement("img")
cover_img.attrib.update({"src": is_cover.attrib["src"]})
cover_div.append(cover_img)
book_content = cover_html
self.cover = is_cover.attrib["src"]
xhtml = html.tostring(book_content,
method="xml",
encoding='unicode')
except (html.etree.ParseError,
html.etree.ParserError) as parsing_error:
self.display.error(parsing_error)
self.display.exit(
"Parser: error trying to parse HTML of this page: %s (%s)" %
(self.filename, self.chapter_title))
return page_css, xhtml
@staticmethod
def escape_dirname(dirname, clean_space=False):
if ":" in dirname:
if dirname.index(":") > 15:
dirname = dirname.split(":")[0]
elif "win" in sys.platform:
dirname = dirname.replace(":", ",")
for ch in [
'~', '#', '%', '&', '*', '{', '}', '\\', '<', '>', '?', '/',
'`', '\'', '"', '|', '+'
]:
if ch in dirname:
dirname = dirname.replace(ch, "_")
return dirname if not clean_space else dirname.replace(" ", "")
def create_dirs(self):
if os.path.isdir(self.BOOK_PATH):
self.display.log("Book directory already exists: %s" %
self.BOOK_PATH)
else:
os.makedirs(self.BOOK_PATH)
oebps = os.path.join(self.BOOK_PATH, "OEBPS")
if not os.path.isdir(oebps):
self.display.book_ad_info = True
os.makedirs(oebps)
self.css_path = os.path.join(oebps, "Styles")
if os.path.isdir(self.css_path):
self.display.log("CSSs directory already exists: %s" %
self.css_path)
else:
os.makedirs(self.css_path)
self.display.css_ad_info.value = 1
self.images_path = os.path.join(oebps, "Images")
if os.path.isdir(self.images_path):
self.display.log("Images directory already exists: %s" %
self.images_path)
else:
os.makedirs(self.images_path)
self.display.images_ad_info.value = 1
def save_page_html(self, contents):
self.filename = self.filename.replace(".html", ".xhtml")
open(os.path.join(self.BOOK_PATH, "OEBPS", self.filename), "wb")\
.write(self.BASE_HTML.format(contents[0], contents[1]).encode("utf-8", 'xmlcharrefreplace'))
self.display.log("Created: %s" % self.filename)
def get(self):
len_books = len(self.book_chapters)
for _ in range(len_books):
if not len(self.chapters_queue):
return
first_page = len_books == len(self.chapters_queue)
next_chapter = self.chapters_queue.pop(0)
self.chapter_title = next_chapter["title"]
self.filename = next_chapter["filename"]
if os.path.isfile(
os.path.join(self.BOOK_PATH, "OEBPS",
self.filename.replace(".html", ".xhtml"))):
if not self.display.book_ad_info and \
next_chapter not in self.book_chapters[:self.book_chapters.index(next_chapter)]:
self.display.info(
"File `%s` already exists.\n"
" If you want to download again all the book%s,\n"
" please delete the `<BOOK NAME>/OEBPS/*.xhtml` files and restart the program."
%
(self.filename.replace(".html", ".xhtml"),
" (especially because you selected the `--no-kindle` option)"
if self.args.no_kindle else ""))
self.display.book_ad_info = 2
else:
self.save_page_html(
self.parse_html(self.get_html(next_chapter["web_url"]),
first_page))
self.display.state(len_books, len_books - len(self.chapters_queue))
def _thread_download_css(self, url):
css_file = os.path.join(self.css_path,
"Style{0:0>2}.css".format(self.css.index(url)))
if os.path.isfile(css_file):
if not self.display.css_ad_info.value and url not in self.css[:self
.css.
index(
url
)]:
self.display.info(
"File `%s` already exists.\n"
" If you want to download again all the CSSs,\n"
" please delete the `<BOOK NAME>/OEBPS/*.xhtml` and `<BOOK NAME>/OEBPS/Styles/*`"
" files and restart the program." % css_file)
self.display.css_ad_info.value = 1
else:
response = self.requests_provider(url, update_cookies=False)
if response == 0:
self.display.error(
"Error trying to retrieve this CSS: %s\n From: %s" %
(css_file, url))
with open(css_file, 'wb') as s:
for chunk in response.iter_content(1024):
s.write(chunk)
self.css_done_queue.put(1)
self.display.state(len(self.css), self.css_done_queue.qsize())
def _thread_download_images(self, url):
image_name = url.split("/")[-1]
image_path = os.path.join(self.images_path, image_name)
if os.path.isfile(image_path):
if not self.display.images_ad_info.value and url not in self.images[:self
.
images
.
index(
url
)]:
self.display.info(
"File `%s` already exists.\n"
" If you want to download again all the images,\n"
" please delete the `<BOOK NAME>/OEBPS/*.xhtml` and `<BOOK NAME>/OEBPS/Images/*`"
" files and restart the program." % image_name)
self.display.images_ad_info.value = 1
else:
response = self.requests_provider(urljoin(self.BASE_URL, url),
update_cookies=False,
stream=True)
if response == 0:
self.display.error(
"Error trying to retrieve this image: %s\n From: %s" %
(image_name, url))
with open(image_path, 'wb') as img:
for chunk in response.iter_content(1024):
img.write(chunk)
self.images_done_queue.put(1)
self.display.state(len(self.images), self.images_done_queue.qsize())
def _start_multiprocessing(self, operation, full_queue):
if len(full_queue) > 5:
for i in range(0, len(full_queue), 5):
self._start_multiprocessing(operation, full_queue[i:i + 5])
else:
process_queue = [
Process(target=operation, args=(arg, )) for arg in full_queue
]
for proc in process_queue:
proc.start()
for proc in process_queue:
proc.join()
def collect_css(self):
self.display.state_status.value = -1
if "win" in sys.platform:
# TODO
for css_url in self.css:
self._thread_download_css(css_url)
else:
self._start_multiprocessing(self._thread_download_css, self.css)
def collect_images(self):
if self.display.book_ad_info == 2:
self.display.info(
"Some of the book contents were already downloaded.\n"
" If you want to be sure that all the images will be downloaded,\n"
" please delete the `<BOOK NAME>/OEBPS/*.xhtml` files and restart the program."
)
self.display.state_status.value = -1
if "win" in sys.platform:
# TODO
for image_url in self.images:
self._thread_download_images(image_url)
else:
self._start_multiprocessing(self._thread_download_images,
self.images)
def create_content_opf(self):
self.css = next(os.walk(self.css_path))[2]
self.images = next(os.walk(self.images_path))[2]
manifest = []
spine = []
for c in self.book_chapters:
c["filename"] = c["filename"].replace(".html", ".xhtml")
item_id = escape("".join(c["filename"].split(".")[:-1]))
manifest.append(
"<item id=\"{0}\" href=\"{1}\" media-type=\"application/xhtml+xml\" />"
.format(item_id, c["filename"]))
spine.append("<itemref idref=\"{0}\"/>".format(item_id))
for i in set(self.images):
dot_split = i.split(".")
head = "img_" + escape("".join(dot_split[:-1]))
extension = dot_split[-1]
manifest.append(
"<item id=\"{0}\" href=\"Images/{1}\" media-type=\"image/{2}\" />"
.format(head, i, "jpeg" if "jp" in extension else extension))
for i in range(len(self.css)):
manifest.append(
"<item id=\"style_{0:0>2}\" href=\"Styles/Style{0:0>2}.css\" "
"media-type=\"text/css\" />".format(i))
authors = "\n".join(
"<dc:creator opf:file-as=\"{0}\" opf:role=\"aut\">{0}</dc:creator>"
.format(escape(aut["name"])) for aut in self.book_info["authors"])
subjects = "\n".join(
"<dc:subject>{0}</dc:subject>".format(escape(sub["name"]))
for sub in self.book_info["subjects"])
return self.CONTENT_OPF.format(
(self.book_info["isbn"]
if self.book_info["isbn"] else self.book_id),
escape(self.book_title), authors,
escape(self.book_info["description"]), subjects, ", ".join(
escape(pub["name"]) for pub in self.book_info["publishers"]),
escape(self.book_info["rights"]), self.book_info["issued"],
self.cover, "\n".join(manifest), "\n".join(spine),
self.book_chapters[0]["filename"].replace(".html", ".xhtml"))
@staticmethod
def parse_toc(l, c=0, mx=0):
r = ""
for cc in l:
c += 1
if int(cc["depth"]) > mx:
mx = int(cc["depth"])
r += "<navPoint id=\"{0}\" playOrder=\"{1}\">" \
"<navLabel><text>{2}</text></navLabel>" \
"<content src=\"{3}\"/>".format(
cc["fragment"] if len(cc["fragment"]) else cc["id"], c,
escape(cc["label"]), cc["href"].replace(".html", ".xhtml").split("/")[-1]
)
if cc["children"]:
sr, c, mx = SafariBooks.parse_toc(cc["children"], c, mx)
r += sr
r += "</navPoint>\n"
return r, c, mx
def create_toc(self):
response = self.requests_provider(urljoin(self.api_url, "toc/"))
if response == 0:
self.display.exit(
"API: unable to retrieve book chapters. "
"Don't delete any files, just run again this program"
" in order to complete the `.epub` creation!")
response = response.json()
if not isinstance(response, list) and len(response.keys()) == 1:
self.display.exit(
self.display.api_error(response) +
" Don't delete any files, just run again this program"
" in order to complete the `.epub` creation!")
navmap, _, max_depth = self.parse_toc(response)
return self.TOC_NCX.format(
(self.book_info["isbn"] if self.book_info["isbn"] else
self.book_id), max_depth, self.book_title,
", ".join(aut["name"]
for aut in self.book_info["authors"]), navmap)
def create_epub(self):
open(os.path.join(self.BOOK_PATH, "mimetype"),
"w").write("application/epub+zip")
meta_info = os.path.join(self.BOOK_PATH, "META-INF")
if os.path.isdir(meta_info):
self.display.log("META-INF directory already exists: %s" %
meta_info)
else:
os.makedirs(meta_info)
open(os.path.join(meta_info, "container.xml"), "wb").write(
self.CONTAINER_XML.encode("utf-8", "xmlcharrefreplace"))
open(os.path.join(self.BOOK_PATH, "OEBPS", "content.opf"),
"wb").write(self.create_content_opf().encode(
"utf-8", "xmlcharrefreplace"))
open(os.path.join(self.BOOK_PATH, "OEBPS", "toc.ncx"),
"wb").write(self.create_toc().encode("utf-8",
"xmlcharrefreplace"))
zip_file = os.path.join(PATH, "Books", self.book_id)
if os.path.isfile(zip_file + ".zip"):
os.remove(zip_file + ".zip")
shutil.make_archive(zip_file, 'zip', self.BOOK_PATH)
os.rename(zip_file + ".zip",
os.path.join(self.BOOK_PATH, self.book_id) + ".epub")
class HDevice:
""" Function to read/write to device """
def __init__(self, proto):
super(HDevice, self).__init__()
self.fifoin = Queue()
self.fifoout = Queue()
self.proto = proto
if proto == "ftdi":
self.processdev = _DeviceProcess(self.fifoin, self.fifoout)
else:
import re
globals()["re"] = re
self.processdev = _DeviceProcessserial(self.fifoin, self.fifoout, self.proto)
def launch(self):
""" Launch the process for device communication """
self.processdev.start()
pid = self.processdev.pid
p = psutil.Process(self.processdev.pid)
p.nice(psutil.HIGH_PRIORITY_CLASS)
print(str(pid) + "est le pid")
def get(self):
""" get byte from device """
return self.fifoin.get()
def quit(self):
""" quit device process """
self.processdev.terminate()
self.processdev.join()
def extract(self, size):
""" Extract 'size' bytes from 'fifo' and return a bytearray """
rec = bytearray([0]*size)
for i in range(0, size):
rec[i] = int.from_bytes(self.get(), 'big')
return rec
def readarray(self, size):
""" read a bytearray from device """
return bytearray(self.extract(size))
def readascii(self):
""" read data in ascii from serial port """
data = self.get()
return data.decode('ascii','backslashreplace')
def readsep(self, sep, size):
""" read data using Regexp """
data = self.readascii()
regexp = ""
for i in range(0,size):
regexp = regexp + r"([0-9]+(?:\.[0-9]+)?)(?:" + sep + ")"
#regexp = r"([0-9]+(?:\.[0-9]+)?)(?:\|)([0-9]+(?:\.[0-9]+)?)"
retour = re.findall(regexp, data)
try:
return retour[0]
except Exception:
return (0,0,0,0)
def incommingsize(self):
"""get the incomming buffer size"""
return self.fifoin.qsize()
def writeint(self, tosend):
"""write data to haptic device"""
bufenvoi = bytearray(4)
bufenvoi[0] = int(tosend) & int('0b00111111', 2)
bufenvoi[1] = ((int(tosend) >> 6) & int('0b00111111', 2)) | int('0b01000000', 2)
bufenvoi[2] = ((int(tosend) >> 12) & int('0b00111111', 2)) | int('0b10000000', 2)
bufenvoi[3] = int('0b11000000', 2)
self.fifoout.put(bufenvoi)
def write(self, tosend):
""" convert and send data to haptic device"""
forcenow = max(min(tosend, 130), -130)
forcenowint = 32767*(1+forcenow/130)
self.writeint(forcenowint)
from multiprocessing import Queue from time import sleep # 创建队列 q = Queue(3) q.put(1) sleep(0.5) print(q.empty()) q.put(2) sleep(0.5) print(q.qsize()) q.put(3) sleep(0.5) print(q.full()) # q.put(4,False,3) print(q.get()) q.close()
class OpenQuoteContext:
def __init__(self, host="127.0.0.1", sync_port=11111, async_port=11111):
"""
create a context to established a network connection
:param host:the address of the network connection
:param sync_port:network connection port for synchronous communication
:param async_port:network connection port for asynchronous communication,receiving client data push
"""
self.__host = host
self.__sync_port = sync_port
self.__async_port = async_port
self._req_queue = Queue()
self._handlers_ctx = HandlerContext()
self._async_ctx = _AsyncNetworkManager(self.__host, self.__async_port,
self._handlers_ctx)
self._proc_run = False
self._sync_net_ctx = _SyncNetworkQueryCtx(self.__host,
self.__sync_port,
long_conn=True)
self._net_proc = Thread(target=_net_proc,
args=(
self._async_ctx,
self._req_queue,
))
def __del__(self):
if self._proc_run:
self._proc_run = False
self._stop_net_proc()
self._net_proc.join(timeout=5)
def set_handler(self, handler):
return self._handlers_ctx.set_handler(handler)
def start(self):
"""
start the receiving thread,asynchronously receive the data pushed by the client
"""
self._net_proc = Thread(target=_net_proc,
args=(
self._async_ctx,
self._req_queue,
))
self._net_proc.start()
self._proc_run = True
def stop(self):
"""
stop the receiving thread, no longer receive the data pushed by the client
"""
if self._proc_run:
self._stop_net_proc()
self._net_proc.join(timeout=5)
self._proc_run = False
self._net_proc = Thread(target=_net_proc,
args=(
self._async_ctx,
self._req_queue,
))
def _send_sync_req(self, req_str):
"""
send a synchronous request
"""
ret, msg, content = self._sync_net_ctx.network_query(req_str)
if ret != RET_OK:
return RET_ERROR, msg, None
return RET_OK, msg, content
def _send_async_req(self, req_str):
"""
send a asynchronous request
"""
if self._req_queue.full() is False:
try:
self._req_queue.put((True, req_str), timeout=1)
return RET_OK, ''
except Exception as e:
_ = e
err = sys.exc_info()[1]
error_str = ERROR_STR_PREFIX + str(err)
return RET_ERROR, error_str
else:
error_str = ERROR_STR_PREFIX + "Request queue is full. The size: %s" % self._req_queue.qsize(
)
return RET_ERROR, error_str
def _get_sync_query_processor(self, pack_func, unpack_func):
"""
synchronize the query processor
:param pack_func: back
:param unpack_func: unpack
:return: sync_query_processor
"""
send_req = self._send_sync_req
def sync_query_processor(**kargs):
ret_code, msg, req_str = pack_func(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg, None
ret_code, msg, rsp_str = send_req(req_str)
if ret_code == RET_ERROR:
return ret_code, msg, None
ret_code, msg, content = unpack_func(rsp_str)
if ret_code == RET_ERROR:
return ret_code, msg, None
return RET_OK, msg, content
return sync_query_processor
def _stop_net_proc(self):
"""
stop the request of network
:return: (ret_error,error_str)
"""
if self._req_queue.full() is False:
try:
self._req_queue.put((False, None), timeout=1)
return RET_OK, ''
except Exception as e:
_ = e
err = sys.exc_info()[1]
error_str = ERROR_STR_PREFIX + str(err)
return RET_ERROR, error_str
else:
error_str = ERROR_STR_PREFIX + "Cannot send stop request. queue is full. The size: %s" \
% self._req_queue.qsize()
return RET_ERROR, error_str
def get_trading_days(self, market, start_date=None, end_date=None):
if market is None or isinstance(market, str) is False:
error_str = ERROR_STR_PREFIX + "the type of market param is wrong"
return RET_ERROR, error_str
if start_date is not None and isinstance(start_date, str) is False:
error_str = ERROR_STR_PREFIX + "the type of start_date param is wrong"
return RET_ERROR, error_str
if end_date is not None and isinstance(end_date, str) is False:
error_str = ERROR_STR_PREFIX + "the type of end_date param is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
TradeDayQuery.pack_req, TradeDayQuery.unpack_rsp)
# the keys of kargs should be corresponding to the actual function arguments
kargs = {
'market': market,
'start_date': start_date,
"end_date": end_date
}
ret_code, msg, trade_day_list = query_processor(**kargs)
if ret_code != RET_OK:
return RET_ERROR, msg
return RET_OK, trade_day_list
def get_stock_basicinfo(self, market, stock_type='STOCK'):
param_table = {'market': market, 'stock_type': stock_type}
for x in param_table:
param = param_table[x]
if param is None or isinstance(param, str) is False:
error_str = ERROR_STR_PREFIX + "the type of %s param is wrong" % x
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
StockBasicInfoQuery.pack_req, StockBasicInfoQuery.unpack_rsp)
kargs = {"market": market, 'stock_type': stock_type}
ret_code, msg, basic_info_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = ['code', 'name', 'lot_size', 'stock_type']
basic_info_table = pd.DataFrame(basic_info_list, columns=col_list)
return RET_OK, basic_info_table
def get_history_kline(self,
code,
start=None,
end=None,
ktype='K_DAY',
autype='qfq'):
if start is not None and isinstance(start, str) is False:
error_str = ERROR_STR_PREFIX + "the type of start param is wrong"
return RET_ERROR, error_str
if end is not None and isinstance(end, str) is False:
error_str = ERROR_STR_PREFIX + "the type of end param is wrong"
return RET_ERROR, error_str
param_table = {'code': code, 'ktype': ktype, 'autype': autype}
for x in param_table:
param = param_table[x]
if param is None or isinstance(param, str) is False:
error_str = ERROR_STR_PREFIX + "the type of %s param is wrong" % x
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
HistoryKlineQuery.pack_req, HistoryKlineQuery.unpack_rsp)
kargs = {
"stock_str": code,
"start_date": start,
"end_date": end,
"ktype": ktype,
"autype": autype
}
ret_code, msg, kline_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'code', 'time_key', 'open', 'close', 'high', 'low', 'volume',
'turnover'
]
kline_frame_table = pd.DataFrame(kline_list, columns=col_list)
return RET_OK, kline_frame_table
def get_autype_list(self, code_list):
if code_list is None or isinstance(code_list, list) is False:
error_str = ERROR_STR_PREFIX + "the type of code_list param is wrong"
return RET_ERROR, error_str
for code in code_list:
if code is None or isinstance(code, str) is False:
error_str = ERROR_STR_PREFIX + "the type of param in code_list is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
ExrightQuery.pack_req, ExrightQuery.unpack_rsp)
kargs = {"stock_list": code_list}
ret_code, msg, exr_record = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'code', 'ex_div_date', 'split_ratio', 'per_cash_div',
'per_share_div_ratio', 'per_share_trans_ratio', 'allotment_ratio',
'allotment_price', 'stk_spo_ratio', 'stk_spo_price',
'forward_adj_factorA', 'forward_adj_factorB',
'backward_adj_factorA', 'backward_adj_factorB'
]
exr_frame_table = pd.DataFrame(exr_record, columns=col_list)
return RET_OK, exr_frame_table
def get_market_snapshot(self, code_list):
if code_list is None or isinstance(code_list, list) is False:
error_str = ERROR_STR_PREFIX + "the type of code_list param is wrong"
return RET_ERROR, error_str
for code in code_list:
if code is None or isinstance(code, str) is False:
error_str = ERROR_STR_PREFIX + "the type of param in code_list is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
MarketSnapshotQuery.pack_req, MarketSnapshotQuery.unpack_rsp)
kargs = {"stock_list": code_list}
ret_code, msg, snapshot_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'code', 'data_date', 'data_time', 'last_price', 'open_price',
'high_price', 'low_price', 'prev_close_price', 'volume',
'turnover', 'turnover_rate', 'suspension', 'listing_date'
]
snapshot_frame_table = pd.DataFrame(snapshot_list, columns=col_list)
return RET_OK, snapshot_frame_table
def subscribe(self, stock_code, data_type, push=False):
"""
subcribe a sort of data for a stock
:param stock_code: string stock_code . For instance, "HK.00700", "US.AAPL"
:param data_type: string data type. For instance, "K_1M", "K_MON"
:param push: push option
:return: (ret_code, ret_data). ret_code: RET_OK or RET_ERROR.
"""
param_table = {'stock_code': stock_code, 'data_type': data_type}
for x in param_table:
param = param_table[x]
if param is None or isinstance(param, str) is False:
error_str = ERROR_STR_PREFIX + "the type of %s param is wrong" % x
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
SubscriptionQuery.pack_subscribe_req,
SubscriptionQuery.unpack_subscribe_rsp)
# the keys of kargs should be corresponding to the actual function arguments
kargs = {'stock_str': stock_code, 'data_type': data_type}
ret_code, msg, _ = query_processor(**kargs)
if ret_code != RET_OK:
return RET_ERROR, msg
if push:
ret_code, msg, push_req_str = SubscriptionQuery.pack_push_req(
stock_code, data_type)
if ret_code != RET_OK:
return RET_ERROR, msg
ret_code, msg = self._send_async_req(push_req_str)
if ret_code != RET_OK:
return RET_ERROR, msg
return RET_OK, None
def unsubscribe(self, stock_code, data_type):
"""
unsubcribe a sort of data for a stock
:param stock_code: string stock_code . For instance, "HK.00700", "US.AAPL"
:param data_type: string data type. For instance, "K_1M", "K_MON"
:return: (ret_code, ret_data). ret_code: RET_OK or RET_ERROR.
"""
param_table = {'stock_code': stock_code, 'data_type': data_type}
for x in param_table:
param = param_table[x]
if param is None or isinstance(param, str) is False:
error_str = ERROR_STR_PREFIX + "the type of %s param is wrong" % x
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
SubscriptionQuery.pack_unsubscribe_req,
SubscriptionQuery.unpack_unsubscribe_rsp)
# the keys of kargs should be corresponding to the actual function arguments
kargs = {'stock_str': stock_code, 'data_type': data_type}
ret_code, msg, _ = query_processor(**kargs)
if ret_code != RET_OK:
return RET_ERROR, msg
return RET_OK, None
def query_subscription(self):
"""
get the current subscription table
:return:
"""
query_processor = self._get_sync_query_processor(
SubscriptionQuery.pack_subscription_query_req,
SubscriptionQuery.unpack_subscription_query_rsp)
ret_code, msg, subscription_table = query_processor()
if ret_code == RET_ERROR:
return ret_code, msg
return RET_OK, subscription_table
def get_stock_quote(self, code_list):
"""
:param code_list:
:return: DataFrame of quote data
Usage:
After subcribe "QUOTE" type for given stock codes, invoke
get_stock_quote to obtain the data
"""
if code_list is None or isinstance(code_list, list) is False:
error_str = ERROR_STR_PREFIX + "the type of code_list param is wrong"
return RET_ERROR, error_str
for code in code_list:
if code is None or isinstance(code, str) is False:
error_str = ERROR_STR_PREFIX + "the type of param in code_list is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
StockQuoteQuery.pack_req,
StockQuoteQuery.unpack_rsp,
)
kargs = {"stock_list": code_list}
ret_code, msg, quote_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'code', 'data_date', 'data_time', 'last_price', 'open_price',
'high_price', 'low_price', 'prev_close_price', 'volume',
'turnover', 'turnover_rate', 'amplitude', 'suspension',
'listing_date'
]
quote_frame_table = pd.DataFrame(quote_list, columns=col_list)
return RET_OK, quote_frame_table
def get_rt_ticker(self, code, num=500):
"""
get transaction information
:param code: stock code
:param num: the default is 500
:return: (ret_ok, ticker_frame_table)
"""
if code is None or isinstance(code, str) is False:
error_str = ERROR_STR_PREFIX + "the type of code param is wrong"
return RET_ERROR, error_str
if num is None or isinstance(num, int) is False:
error_str = ERROR_STR_PREFIX + "the type of num param is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
TickerQuery.pack_req,
TickerQuery.unpack_rsp,
)
kargs = {"stock_str": code, "num": num}
ret_code, msg, ticker_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'stock_code', 'time', 'price', 'volume', 'turnover',
"ticker_direction", 'sequence'
]
ticker_frame_table = pd.DataFrame(ticker_list, columns=col_list)
return RET_OK, ticker_frame_table
def get_cur_kline(self, code, num, ktype='K_DAY', autype='qfq'):
"""
get current kline
:param code: stock code
:param num:
:param ktype: the type of kline
:param autype:
:return:
"""
param_table = {'code': code, 'ktype': ktype}
for x in param_table:
param = param_table[x]
if param is None or isinstance(param, str) is False:
error_str = ERROR_STR_PREFIX + "the type of %s param is wrong" % x
return RET_ERROR, error_str
if num is None or isinstance(num, int) is False:
error_str = ERROR_STR_PREFIX + "the type of num param is wrong"
return RET_ERROR, error_str
if autype is not None and isinstance(autype, str) is False:
error_str = ERROR_STR_PREFIX + "the type of autype param is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
CurKlineQuery.pack_req,
CurKlineQuery.unpack_rsp,
)
kargs = {
"stock_str": code,
"num": num,
"ktype": ktype,
"autype": autype
}
ret_code, msg, kline_list = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
col_list = [
'code', 'time_key', 'open', 'close', 'high', 'low', 'volume',
'turnover'
]
kline_frame_table = pd.DataFrame(kline_list, columns=col_list)
return RET_OK, kline_frame_table
def get_order_book(self, code):
if code is None or isinstance(code, str) is False:
error_str = ERROR_STR_PREFIX + "the type of code param is wrong"
return RET_ERROR, error_str
query_processor = self._get_sync_query_processor(
OrderBookQuery.pack_req,
OrderBookQuery.unpack_rsp,
)
kargs = {"stock_str": code}
ret_code, msg, orderbook = query_processor(**kargs)
if ret_code == RET_ERROR:
return ret_code, msg
return RET_OK, orderbook
class Publisher:
""" Publisher class in which the queue for publishing messages is defined and also a separated process is started.
It is important to have a new process, since the serialization/deserialization of messages from the QUEUE may be
a bottleneck for performance.
"""
def __init__(self, port=5555):
self.logger = get_logger(name=__name__)
self._port = port
self._queue = Queue(
) # The publisher will grab and broadcast the messages from this queue
self._event = Event() # This event is used to stop the process
self._process = Process(target=publisher,
args=[self._queue, self._event, self._port])
self.logger.info('Initialized published on port {}'.format(port))
def start(self):
""" Start a new process that will be responsible for broadcasting the messages.
"""
self._event.clear()
self._process.start()
sleep(1) # This forces the start to block until the publisher is ready
def stop(self):
self._event.set()
self.empty_queue()
def empty_queue(self):
""" If the publisher stops before broadcasting all the messages, the Queue may still be using some memory. This
method is simply getting all the elements in order to free memory. Can be useful for garbage collection or
better control of the downstream program.
"""
self.logger.info('Emptying the queue of the publisher')
self.logger.debug('Queue length: {}'.format(self._queue.qsize()))
self._queue.close()
def publish(self, topic, data):
""" Adapts the data to make it faster to broadcast
:param str topic: Topic in which to publish the data
:param data: Data to be published
:return: None
"""
self.logger.debug('Adding data of type {} to topic {}'.format(
type(data), topic))
try:
self._queue.put({'topic': topic, 'data': data})
except AssertionError:
# This means the queue has been closed already
pass
@property
def port(self):
return self._port
@port.setter
def port(self, new_port):
if new_port != self._port:
self._port = new_port
self.logger.warning(
'Changing the port requires restarting the publisher process')
self.logger.debug(
'Setting the new publisher port to {}'.format(new_port))
self.stop()
self._process.join()
self._process = Process(
target=publisher, args=[self._queue, self._event, self._port])
self.start()
else:
self.logger.warning(
'New port {} is the same as the old port'.format(new_port))
def join(self, timeout=0):
if self._process.is_alive():
self.logger.debug('Waiting for Publisher process to finish')
self._process.join(timeout)
def main(args):
number = ''
_img_count = 0
dsf = os.path.dirname(os.path.realpath(__file__))
js_path = os.path.join(dsf, args.config_file)
json_file = open(js_path).read()
js = json.loads(json_file)
config = js[str(args.config)]
x1, x2, y1, y2 = config[
'region'] # (125, 950, 100, 620)#src='http:///mjpg/video.mjpg',
cap = video_capture(config['videosource'], config['width'],
config['height'])
if cap is None:
return
server = config['server']
port = config['port']
qo = Queue()
qi = Queue()
p = Process(target=ocr, args=(qo, qi))
p.start()
atexit.register(ocr_kill, p)
md = motiondetect.MotionDetect()
while True:
try:
ret, image = cap.read()
if image is None:
cap.release()
cap = video_capture(config['videosource'], config['width'],
config['height'])
time.sleep(2)
continue
img = image[y1:y2, x1:x2]
img = cv2.cvtColor(img, cv2.COLOR_BGR2GRAY)
cv2.rectangle(image, (0, 0), (image.shape[1], 50), (255, 255, 255),
cv2.FILLED)
cv2.rectangle(image, (x1, y1), (x2, y2), (0, 0, 255), 2)
font = cv2.FONT_HERSHEY_SIMPLEX
evc_l, delta, count = md.evc_detect(img.copy())
cv2.putText(image,
str(evc_l) + ' ' + str(delta) + ' ' + str(count),
(620, 200), font, 1, (255, 100, 0), 2, cv2.LINE_AA)
if evc_l >= 3.0 and _img_count < 10:
_img_count += 1
cv2.rectangle(image, (x1, y1), (x2, y2), (0, 255, 0), 2)
qi.put(img)
if qo.qsize() > 0:
ocr_data = qo.get()
_img_count -= 1
if ocr_data[0]:
number = ocr_data[2] + '&' + ocr_data[1].strftime(
'%d.%m.%Y %H:%M:%S')
cv2.rectangle(image, (0, 0), (image.shape[1], 50),
(0, 255, 0), 2)
t = Thread(target=tcp_client,
args=(server, port, str(args.config) + '&' +
number + '&' + ocr_data[3] + '&'))
t.start()
else:
cv2.rectangle(image, (0, 0), (image.shape[1], 50),
(0, 0, 255), 2)
cv2.putText(image, number, (10, 35), font, 1, (255, 100, 0), 2,
cv2.LINE_AA)
# #image[y1:y2, x1:x2] = imgc
cv2.imshow('Video', image)
if cv2.waitKey(1) == 27:
cap.release()
cv2.destroyAllWindows()
exit(0)
if not p.is_alive():
p = Process(target=ocr, args=(qo, qi))
p.start()
except:
logging.exception('')
cap.release()
cv2.destroyAllWindows()
class KittiLoader(object):
# return:
# tag (N)
# label (N) (N')
# rgb (N, H, W, C)
# raw_lidar (N) (N', 4)
# vox_feature
# vox_number
# vox_coordinate
def __init__(self,
object_dir='.',
queue_size=20,
require_shuffle=False,
is_testset=True,
batch_size=1,
use_multi_process_num=0,
split_file='',
multi_gpu_sum=1,
aug=False):
assert (use_multi_process_num >= 0)
self.object_dir = object_dir
self.is_testset = is_testset
self.use_multi_process_num = use_multi_process_num if not self.is_testset else 1
self.require_shuffle = require_shuffle if not self.is_testset else False
self.batch_size = batch_size
self.split_file = split_file
self.multi_gpu_sum = multi_gpu_sum
self.aug = aug
if self.split_file != '':
# use split file
_tag = []
self.f_rgb, self.f_lidar, self.f_label = [], [], []
for line in open(self.split_file, 'r').readlines():
line = line[:-1] # remove '\n'
_tag.append(line)
self.f_rgb.append(
os.path.join(self.object_dir, 'image_2', line + '.png'))
self.f_lidar.append(
os.path.join(self.object_dir, 'velodyne', line + '.bin'))
self.f_label.append(
os.path.join(self.object_dir, 'label_2', line + '.txt'))
else:
self.f_rgb = glob.glob(
os.path.join(self.object_dir, 'image_2', '*.png'))
self.f_rgb.sort()
self.f_lidar = glob.glob(
os.path.join(self.object_dir, 'velodyne', '*.bin'))
self.f_lidar.sort()
self.f_label = glob.glob(
os.path.join(self.object_dir, 'label_2', '*.txt'))
self.f_label.sort()
self.data_tag = [
name.split('/')[-1].split('.')[-2] for name in self.f_rgb
]
assert (len(self.data_tag) == len(self.f_rgb) == len(self.f_lidar))
self.dataset_size = len(self.f_rgb)
self.already_extract_data = 0
self.cur_frame_info = ''
print("Dataset total length: {}".format(self.dataset_size))
if self.require_shuffle:
self.shuffle_dataset()
self.queue_size = queue_size
self.require_shuffle = require_shuffle
# must use the queue provided by multiprocessing module(only this can be shared)
self.dataset_queue = Queue()
self.load_index = 0
if self.use_multi_process_num == 0:
self.loader_worker = [
threading.Thread(target=self.loader_worker_main,
args=(self.batch_size, ))
]
else:
self.loader_worker = [
Process(target=self.loader_worker_main,
args=(self.batch_size, ))
for i in range(self.use_multi_process_num)
]
self.work_exit = Value('i', 0)
[i.start() for i in self.loader_worker]
# This operation is not thread-safe
self.rgb_shape = (cfg.IMAGE_HEIGHT, cfg.IMAGE_WIDTH, 3)
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.work_exit.value = True
def __len__(self):
return self.dataset_size
def fill_queue(self, batch_size=0):
load_index = self.load_index
self.load_index += batch_size
if self.load_index >= self.dataset_size:
if not self.is_testset: # test set just end
if self.require_shuffle:
self.shuffle_dataset()
load_index = 0
self.load_index = load_index + batch_size
else:
self.work_exit.value = True
labels, tag, voxel, rgb, raw_lidar = [], [], [], [], []
for _ in range(batch_size):
try:
if self.aug:
ret = aug_data(self.data_tag[load_index], self.object_dir)
tag.append(ret[0])
rgb.append(ret[1])
raw_lidar.append(ret[2])
voxel.append(ret[3])
labels.append(ret[4])
else:
rgb.append(
cv2.resize(cv2.imread(self.f_rgb[load_index]),
(cfg.IMAGE_WIDTH, cfg.IMAGE_HEIGHT)))
raw_lidar.append(
np.fromfile(self.f_lidar[load_index],
dtype=np.float32).reshape((-1, 4)))
if not self.is_testset:
labels.append([
line for line in open(self.f_label[load_index],
'r').readlines()
])
else:
labels.append([''])
tag.append(self.data_tag[load_index])
######################################################
# TODO:
# we need to change the preprocessing of point cloud for point fusion
voxel.append(process_pointcloud(raw_lidar[-1]))
load_index += 1
except:
if not self.is_testset: # test set just end
self.load_index = 0
if self.require_shuffle:
self.shuffle_dataset()
else:
self.work_exit.value = True
# only for voxel -> [gpu, k_single_batch, ...]
vox_feature, vox_number, vox_coordinate = [], [], []
single_batch_size = int(self.batch_size / self.multi_gpu_sum)
for idx in range(self.multi_gpu_sum):
_, per_vox_feature, per_vox_number, per_vox_coordinate = build_input(
voxel[idx * single_batch_size:(idx + 1) * single_batch_size])
vox_feature.append(per_vox_feature)
vox_number.append(per_vox_number)
vox_coordinate.append(per_vox_coordinate)
self.dataset_queue.put_nowait(
(labels, (vox_feature, vox_number, vox_coordinate), rgb, raw_lidar,
tag))
def load(self):
try:
if self.is_testset and self.already_extract_data >= self.dataset_size:
return None
buff = self.dataset_queue.get()
label = buff[0]
vox_feature = buff[1][0]
vox_number = buff[1][1]
vox_coordinate = buff[1][2]
rgb = buff[2]
raw_lidar = buff[3]
tag = buff[4]
self.cur_frame_info = buff[4]
self.already_extract_data += self.batch_size
ret = (np.array(tag), np.array(label), np.array(vox_feature),
np.array(vox_number), np.array(vox_coordinate),
np.array(rgb), np.array(raw_lidar))
except:
print("Dataset empty!")
ret = None
return ret
def load_specified(self, index=0):
rgb = cv2.resize(cv2.imread(self.f_rgb[index]),
(cfg.IMAGE_WIDTH, cfg.IMAGE_HEIGHT))
raw_lidar = np.fromfile(self.f_lidar[index], dtype=np.float32).reshape(
(-1, 4))
labels = [line for line in open(self.f_label[index], 'r').readlines()]
tag = self.data_tag[index]
if self.is_testset:
ret = (
np.array([tag]),
np.array([rgb]),
np.array([raw_lidar]),
)
else:
ret = (
np.array([tag]),
np.array([labels]),
np.array([rgb]),
np.array([raw_lidar]),
)
return ret
def loader_worker_main(self, batch_size):
if self.require_shuffle:
self.shuffle_dataset()
while not self.work_exit.value:
if self.dataset_queue.qsize() >= self.queue_size // 2:
time.sleep(1)
else:
# since we use multiprocessing, 1 is ok
self.fill_queue(batch_size)
def get_shape(self):
return self.rgb_shape
def shuffle_dataset(self):
# to prevent diff loader load same data
index = shuffle([i for i in range(len(self.f_label))],
random_state=random.randint(
0, self.use_multi_process_num**5))
self.f_label = [self.f_label[i] for i in index]
self.f_rgb = [self.f_rgb[i] for i in index]
self.f_lidar = [self.f_lidar[i] for i in index]
self.data_tag = [self.data_tag[i] for i in index]
def get_frame_info(self):
return self.cur_frame_info
class DemoGui(QtWidgets.QWidget):
def __init__(self):
super().__init__()
self._layout = QtWidgets.QVBoxLayout()
self._number1 = QtWidgets.QLineEdit('5')
self._number1.setToolTip('Number of Spheres per job')
self._number2 = QtWidgets.QLineEdit('4')
self._number2.setToolTip('Number of jobs to generate')
self._opacity = QtWidgets.QSlider()
self._opacity.setToolTip(
'Opacity of Spheres. Evaluated just before render, after polydata has returned from'
' the worker. Renderer gets pretty bogged down with opacity and 1000+ spheres.'
)
self._opacity.setMinimum(0)
self._opacity.setMaximum(100)
self._opacity.setValue(100)
self._send_button = QtWidgets.QPushButton('send')
self._stop_button = QtWidgets.QPushButton('stop worker')
self._clear_button = QtWidgets.QPushButton('clear outputs')
self._textbox = QtWidgets.QTextEdit()
self._vtk_plot = VTKPlot()
self._inputs_queue = Queue()
self._outputs_queue = Queue()
self._worker_process1 = Process(target=worker,
args=(self._inputs_queue,
self._outputs_queue, 1))
self._worker_process2 = Process(target=worker,
args=(self._inputs_queue,
self._outputs_queue, 2))
self._worker_process3 = Process(target=worker,
args=(self._inputs_queue,
self._outputs_queue, 3))
self._worker_process4 = Process(target=worker,
args=(self._inputs_queue,
self._outputs_queue, 4))
self._worker_process1.start()
self._worker_process2.start()
self._worker_process3.start()
self._worker_process4.start()
self._timer = QtCore.QTimer()
self._timer.setInterval(100)
self._timer.timeout.connect(self.display_results)
self._timer.start()
self._send_button.clicked.connect(self.send_numbers)
self._stop_button.clicked.connect(self.stop)
self._clear_button.clicked.connect(self.clear)
self._layout.addWidget(self._number1)
self._layout.addWidget(self._number2)
self._layout.addWidget(self._opacity)
self._layout.addWidget(self._send_button)
self._layout.addWidget(self._stop_button)
self._layout.addWidget(self._clear_button)
self._layout.addWidget(self._textbox)
self._layout.addWidget(self._vtk_plot)
self.setLayout(self._layout)
self._time_dict = {}
def closeEvent(self, event):
self.stop()
def clear(self, *args):
self._textbox.clear()
self._vtk_plot.ren.Clear()
for actor in self._vtk_plot.ren.GetActors():
self._vtk_plot.ren.RemoveActor(actor)
self._vtk_plot.ren.ResetCamera()
self._vtk_plot.ren.Render()
self._vtk_plot.renwin.Render()
def stop(self, *args):
print('sending stop')
for _ in range(4):
self._inputs_queue.put('STOP')
def send_numbers(self, *args):
print('sending numbers')
try:
num1 = float(self._number1.text())
num2 = float(self._number2.text())
except Exception as e:
import traceback
traceback.print_exc()
return
meta = {'t1': time()}
for _ in range(int(num2)):
self._inputs_queue.put((num1, num2, meta))
def display_results(self):
if not self._outputs_queue.empty():
# Not sure if this is necessary, but I don't want to think about what happens when the timer goes off while
# the method is in progress.
self._timer.blockSignals(True)
t0 = time()
n = self._outputs_queue.qsize()
# This is a compromize... I would keep emptying the queue until it's empty, which is more likely to lock
# the GUI because the workers may continue catching up while this is operating.
# If there is no loop here, then items are only read once per timer tick.
# If the worker puts each sphere in the queue individually, this can cause each sphere's appearance to
# be animated which is much slower because Render() is slow... however workers are currently appending all spheres
# of a job into an AppendPolyData
for _ in range(n):
# print('Displaying results')
r_message = self._outputs_queue.get()
result, meta = r_message
t1 = meta['t1']
t_work = meta['t_work']
t3 = time()
dt = t3 - t1
t_overhead = dt - t_work
proc_id = meta['proc_id']
# self._textbox.append(str(result))
self._textbox.append(
f'Took {dt:1.6f} s total. {t_work:1.6f} working. {t_overhead:1.6f} overhead. Proc: {proc_id}'
)
self.plot_shape(result, proc_id)
t1 = time()
self._timer.blockSignals(False)
print(f'Output Processing Time: {t1-t0:1.3f}s')
self._vtk_plot.ren.ResetCamera()
self._vtk_plot.ren.Render()
self._vtk_plot.renwin.Render()
def plot_shape(self, shape_data, proc_id):
t0 = time()
reader = vtk.vtkPolyDataReader()
reader.ReadFromInputStringOn()
reader.SetInputString(shape_data)
reader.Update()
shape_pd = reader.GetOutput()
mapper = vtk.vtkPolyDataMapper()
mapper.SetInputData(shape_pd)
actor = vtk.vtkActor()
actor.SetMapper(mapper)
color = {
1: (1, 0, 0),
2: (0, 1, 0),
3: (0, 0, 1),
4: (0, 1, 1)
}[proc_id]
actor.GetProperty().SetDiffuseColor(color)
actor.GetProperty().SetOpacity(self._opacity.value() / 100)
self._vtk_plot.ren.AddActor(actor)
t1 = time()
def run(args):
start = datetime.now()
#print args
#==#
verbose = args.verbose
force = args.force
#==#
inputFile = args.input
breakPointsFile = args.breakpoints
outputFile = args.output
numberOfBands = int(args.bands)
#==#
rows = int(args.rows)
numberOfThreads = int(args.threads)
#==#
print "=================================="
print "#==#"
print "CyberGIS Script / cybergis-script-ittc-stetch.py"
print "Apply stetch to raster image(s)."
print "#==#"
#==#
if numberOfBands != 1 and numberOfBands != 3:
print "The number of bands specified is not expected."
return 0
if rows <= 0:
print "You to process at least 1 row at a time."
return 0
if numberOfThreads <= 0:
print "You need at least 1 thread."
return 0
if not os.path.exists(inputFile):
print "Input file does not exist."
return 0
if not os.path.exists(breakPointsFile):
print "Breakpoints file does not exist."
return 0
if force == 0 and os.path.exists(outputFile):
print "Output file already exists."
return 0
#==#
#Load input image file and breakpoints file
inputDataset = gdal.Open(inputFile, GA_ReadOnly)
lookUpTables = LookUpTables(breakPointsFile, numberOfBands)
if inputDataset is None:
print "Error opening input image file."
return 0
if not lookUpTables.isValid():
print "Error building lookup tables. Are you sure the breakpoints file matches the right number of bands specified?"
return 0
#==#
#Create output file
outputFormat = "HFA"
w = inputDataset.RasterXSize
h = inputDataset.RasterYSize
if force > 0 and os.path.isfile(outputFile):
if verbose > 0:
print "Deleting existing output file at ", outputFile
os.remove(outputFile)
outputDataset = initDataset(outputFile, outputFormat, w, h, numberOfBands)
outputDataset.SetGeoTransform(list(inputDataset.GetGeoTransform()))
outputDataset.SetProjection(inputDataset.GetProjection())
#==#
#Core Process
if numberOfThreads == 1:
for b in range(numberOfBands):
print "Stretching Band " + str(b + 1)
lut = numpy.array(lookUpTables.tables[b].table)
if verbose > 0:
print "Lookup Table (lut): "
print lut
inBand = inputDataset.GetRasterBand(b + 1)
outBand = outputDataset.GetRasterBand(b + 1)
r = rows
if verbose > 0:
print "Processing", r, "rows at a time."
print "Processing", int(inBand.YSize / r), "batches of rows."
for y in range(int(inBand.YSize / r)):
outBand.WriteArray(
lut[inBand.ReadAsArray(0, y * r, inBand.XSize, r,
inBand.XSize, r)], 0, y * r)
if verbose > 0:
print "Processing", (inBand.YSize % r), "residual rows."
y0 = inBand.YSize / rows
for y in range(inBand.YSize % r):
outBand.WriteArray(
lut[inBand.ReadAsArray(0, y0 + y, inBand.XSize, 1,
inBand.XSize, 1)], 0, y0 + y)
elif numberOfThreads > 1:
print "not fully implemented yet"
global exitFlag
global queueLock
global writeLock
global workQueue
global tasks
#==#
exitFlag = 0
queueLock = Lock()
writeLock = Lock()
workQueue = Queue(0)
tasks = Tasks()
#==#
for b in range(numberOfBands):
print "Adding tasks for band " + str(b + 1)
lut = numpy.array(lookUpTables.tables[b].table)
inBand = inputDataset.GetRasterBand(b + 1)
outBand = outputDataset.GetRasterBand(b + 1)
y0 = int(inBand.YSize / r)
for y in range(int(inBand.YSize / r)):
task = b + 1, inBand, outBand, lut, y0, y, r, 1
tasks.add(task)
for y in range(inBand.YSize % r):
task = b + 1, inBand, outBand, lut, y0, y, r, 2
tasks.add(task)
print "tasks in main queue: " + str(len(tasks.tasks))
#Add tasks to queue
queueLock.acquire()
for taskID in range(len(tasks.tasks)):
workQueue.put(taskID)
queueLock.release()
processes = initProcesses(numberOfThreads)
print "Queue is full with " + str(workQueue.qsize()) + " tasks."
print "Rendering threads will now execute."
while not workQueue.empty():
pass
exitFlag = 1 #tell's threads it's time to quit
for process in processes:
process.join()
#==#
#Post Process
inputDataset = None
outputDataset = None
print datetime.now() - start
return 0
print "=================================="
