class Transcoder:
def __init__(self):
self.stopping = Event()
def stop(self):
logger.debug("Preventing new transcoding processes.")
self.stopping.set()
def transcode(self, path, format='mp3', bitrate=False):
if self.stopping.is_set():
return
try:
stop = Event()
start_time = time.time()
parent_conn, child_conn = Pipe()
process = Process(target=transcode_process,
args=(child_conn, path, stop, format, bitrate))
process.start()
while not (self.stopping.is_set() or stop.is_set()):
data = parent_conn.recv()
if not data:
break
yield data
logger.debug("Transcoded %s in %0.2f seconds." % (path.encode(cfg['ENCODING']), time.time() - start_time))
except GeneratorExit:
stop.set()
logger.debug("User canceled the request during transcoding.")
except:
stop.set()
logger.warn("Some type of error occured during transcoding.")
finally:
parent_conn.close()
process.join()
def single_output(stop_event: Event):
print("single output get queue:")
sum_limit = 1000
counter = 0
manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
while not stop_event.is_set():
try:
while not output_q.empty() or not stop_event.is_set():
result = output_q.get(False, 1)
counter += 1
if isinstance(result, list):
for item in result:
print("server queue output:", str(item), "count:", counter)
else:
# print(result)
pass
if counter/sum_limit > 0 and counter % sum_limit==0:
print("current output count is:", counter)
time.sleep(0.000001)
except Exception as ex:
pass
# manager, output_q = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Output)
finally:
print("going to sleep.")
time.sleep(1)
class Worker(Process):
def __init__(self, buffer, reorder_buffer, job):
Process.__init__(self)
self.buffer = buffer
self.reorder_buffer = reorder_buffer
self.job = job
self.event = Event()
def run(self):
self.event.set()
while self.event.is_set():
try:
block_number, data = self.buffer.get()
except IOError, e:
if e.errno == errno.EINTR:
data = EOF
if data == EOF:
self.stop()
break
worked_data = self.job(data)
while self.event.is_set():
try:
self.reorder_buffer.put(block_number, worked_data)
break
except IndexError:
# Block num bigger than expected,
# wait untill ReorderBuffer start
# processing blocks in this range
time.sleep(0.1)
except IOError, e:
if e.errno == errno.EINTR:
self.stop()
class Cluster(object):
def __init__(self, broker=None):
self.broker = broker or get_broker()
self.sentinel = None
self.stop_event = None
self.start_event = None
self.pid = current_process().pid
self.host = socket.gethostname()
self.timeout = Conf.TIMEOUT
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
def start(self):
# Start Sentinel
self.stop_event = Event()
self.start_event = Event()
self.sentinel = Process(target=Sentinel, args=(self.stop_event, self.start_event, self.broker, self.timeout))
self.sentinel.start()
logger.info(_("Q Cluster-{} starting.").format(self.pid))
while not self.start_event.is_set():
sleep(0.1)
return self.pid
def stop(self):
if not self.sentinel.is_alive():
return False
logger.info(_("Q Cluster-{} stopping.").format(self.pid))
self.stop_event.set()
self.sentinel.join()
logger.info(_("Q Cluster-{} has stopped.").format(self.pid))
self.start_event = None
self.stop_event = None
return True
def sig_handler(self, signum, frame):
logger.debug(_("{} got signal {}").format(current_process().name, Conf.SIGNAL_NAMES.get(signum, "UNKNOWN")))
self.stop()
@property
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
@property
def is_starting(self):
return self.stop_event and self.start_event and not self.start_event.is_set()
@property
def is_running(self):
return self.stop_event and self.start_event and self.start_event.is_set()
@property
def is_stopping(self):
return self.stop_event and self.start_event and self.start_event.is_set() and self.stop_event.is_set()
@property
def has_stopped(self):
return self.start_event is None and self.stop_event is None and self.sentinel
class CheckerProcess(Process):
def __init__(self, team_id, checks, db_host, db_port, db_name, check_delay):
super(CheckerProcess, self).__init__()
self.team_id = team_id
self.db = MongoDBWrapper(db_host, int(db_port), db_name)
self.check_delay = int(check_delay)
self.shutdown_event = Event()
self.checks = []
for check in checks:
check_db_data = self.db.get_specific_check(check.check_id, check.check_class.check_type)
if len(check_db_data) > 0:
check_data = check_db_data[0]
if issubclass(check.check_class, InjectCheck):
check_obj = check.check_class(check_data['machine'], team_id, db_host, db_port, db_name, check_data['time_to_check'])
self.checks.append(Checker(check.check_id, check_obj, check_obj.time_to_run))
elif issubclass(check.check_class, (ServiceCheck, AttackerCheck)):
check_obj = check.check_class(check_data['machine'], team_id, db_host, db_port, db_name)
self.checks.append(Checker(check.check_id, check_obj, datetime.now()))
random.shuffle(self.checks, random.random)
def run(self):
while not self.shutdown_event.is_set():
self.run_checks()
def run_checks(self):
indices_to_remove = []
for i in range(0, len(self.checks)):
check = self.checks[i]
check_obj = copy(check.object)
now = datetime.now()
print check.time_to_run, '<', now, '=', check.time_to_run < now
if check.time_to_run < now:
check_process = Process(target=check_obj.run_check)
check_process.start()
check_process.join(check_obj.timeout)
if check_process.is_alive():
check_process.terminate()
score = check_obj.score
if issubclass(type(check_obj), InjectCheck):
self.db.complete_inject_check(check.id, self.team_id, datetime.now(), score)
#self.checks[:] = [obj for obj in self.checks if not obj == check]
indices_to_remove.append(i)
elif issubclass(type(check_obj), ServiceCheck):
self.db.complete_service_check(check.id, self.team_id, datetime.now(), score)
check.timestamp = datetime.now() + timedelta(seconds=self.check_delay)
elif issubclass(type(check_obj), AttackerCheck):
self.db.complete_attacker_check(check.id, self.team_id, datetime.now(), score)
check.timestamp = datetime.now() + timedelta(seconds=self.check_delay)
self.db.calculate_scores_for_team(self.team_id)
if self.shutdown_event.is_set():
break
indices_to_remove.sort(reverse=True)
for i in indices_to_remove:
del self.checks[i]
class SharedEvent:
def __init__(self, timeout = 0.7):
self._rest = Event()
self._exit = Event()
self.timeout = timeout
self._rest.set()
def shutdown(self):
while self._rest.is_set():
time.sleep(self.timeout)
return self._exit.is_set()
class ParserThread(Process):
""" Class writes simulated data to navigation system and
receives results """
def __init__(self, device, baudrate, q):
Process.__init__(self)
self.device = device
self.baudrate = baudrate
self.q = q
self.__stop = Event()
def stop(self):
self.__stop.set()
def mav_dispatch(self, m):
try:
self.q.put_nowait(m)
except Full:
pass
def run(self):
signal.signal(signal.SIGINT, signal.SIG_IGN)
print("*** parser process started. Pid:", os.getpid())
master = mavutil.mavlink_connection(self.device, self.baudrate, dialect="lapwing")
while not self.__stop.is_set():
m = master.recv_msg()
if (m is not None):
dbg_print(m)
self.mav_dispatch(m)
master.close()
print("*** parser process stopped")
def test_recycle(broker, monkeypatch):
# set up the Sentinel
broker.list_key = 'test_recycle_test:q'
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
start_event = Event()
stop_event = Event()
# override settings
monkeypatch.setattr(Conf, 'RECYCLE', 2)
monkeypatch.setattr(Conf, 'WORKERS', 1)
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, broker=broker)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
async('django_q.tests.tasks.multiply', 2, 2, broker=broker)
task_queue = Queue()
result_queue = Queue()
# push two tasks
pusher(task_queue, stop_event, broker=broker)
pusher(task_queue, stop_event, broker=broker)
# worker should exit on recycle
worker(task_queue, result_queue, Value('f', -1))
# check if the work has been done
assert result_queue.qsize() == 2
# save_limit test
monkeypatch.setattr(Conf, 'SAVE_LIMIT', 1)
result_queue.put('STOP')
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
broker.delete_queue()
def pipelineDaemon(pipeline, returnEvent, options=None, programName=None):
"""Launches Pyro server and (if specified by options) pipeline executors"""
#check for valid pipeline
if pipeline.runnable.empty()==None:
print "Pipeline has no runnable stages. Exiting..."
sys.exit()
if options.urifile==None:
options.urifile = os.path.abspath(os.curdir + "/" + "uri")
e = Event()
process = Process(target=launchServer, args=(pipeline,options,e,))
process.start()
e.wait()
if options.num_exec != 0:
processes = [Process(target=launchPipelineExecutor, args=(options,programName,)) for i in range(options.num_exec)]
for p in processes:
p.start()
for p in processes:
p.join()
#Return to calling code if pipeline has no more runnable stages:
#Event will be cleared once clients are unregistered.
while e.is_set():
time.sleep(5)
returnEvent.set()
class Updater(Process):
def __init__(self, maxsize=15):
Process.__init__(self)
#self.queue = Queue(maxsize)
self.queue = Queue()
self.queue_lock = Lock()
self._exit = Event()
def run(self):
while not self._exit.is_set():
#with self.queue_lock:
self.queue.put(self.receive())
#self.queue.put_nowait(self.receive())
#if self.queue.full():
# try:
# self.queue.get_nowait()
# except:
# pass
def stop(self):
self._exit.set()
# This leaves the process hanging on Windows
#self.join(STOP_TIMEOUT)
if self.is_alive():
#TODO make a nicer warning
print 'Terminating updater:', self
self.terminate()
def receive(self):
raise NotImplementedError
class MistProcess(Process):
def __init__(self, gpio, sleep=1, name='MistProcess'):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.name = name
self.gpio = gpio
self.sleep = sleep
self.mist = mraa.Gpio(self.gpio)
self.mist.dir(mraa.DIR_OUT)
def _mist_on(self):
self.logger.debug('Mist on')
self.mist.write(1)
def _mist_off(self):
self.logger.debug('Mist off')
if self.mist:
self.mist.write(0)
def run(self):
self.event.set()
self.logger.debug('PID: %d' % multiprocessing.current_process().pid)
while self.event.is_set():
self._mist_on()
time.sleep(self.sleep)
def stop(self):
self.logger.debug('Process {} will halt.'.format(self.name))
self.event.clear()
self._mist_off()
def execute_action(self, action):
event = Event()
queue = Queue()
proc = Process(
target=execute_action_proc,
args=(self.execute, action, event, queue))
proc.start()
# Send heartbeat.
heartbeat_retry = 0
while not event.is_set():
event.wait(config.ACTIVITY_HEARTBEAT_INTERVAL)
try:
res = self.heartbeat(self.task_token)
if res['cancelRequested']:
proc.terminate()
proc.join()
return Result('cancelled', -1, '', '', '', -1)
except Exception as err:
if heartbeat_retry <= config.ACTIVITY_HEARTBEAT_MAX_RETRY:
heartbeat_retry += 1
continue
else:
proc.terminate()
proc.join()
raise
# Evaluate the result.
result = queue.get_nowait()
proc.join()
return result
def test_sentinel():
start_event = Event()
stop_event = Event()
stop_event.set()
s = Sentinel(stop_event, start_event, list_key='sentinel_test:q')
assert start_event.is_set()
assert s.status() == Conf.STOPPED
def test_sentinel():
start_event = Event()
stop_event = Event()
stop_event.set()
s = Sentinel(stop_event, start_event, broker=get_broker('sentinel_test:q'))
assert start_event.is_set()
assert s.status() == Conf.STOPPED
class World (Process):
""" A group of particles that can interact with each other """
def __init__(self, plane, send, update_rate=60):
Process.__init__(self)
self.plane = plane
self.send = send
self.update_interval = 1/update_rate
self.exit = Event()
logger.info('Initialised')
def run(self):
logger.info('Running')
previous_update = current_time()
while not self.exit.is_set():
# Update each particle
for particle in self.plane:
particle.update()
# stuff and things
self.send.send(self.plane)
# Sleep to maintain update rate
update_delay = self.update_interval - (current_time() - previous_update)
previous_update = current_time()
sleep(max(0, update_delay))
def terminate(self):
logger.info('Exiting')
self.exit.set()
class BroadcastClient(Process):
def __init__(self, port, datagram_size, name="BroadcastClient"):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.port = port
self.datagram_size = datagram_size
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM, socket.IPPROTO_UDP)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.settimeout(1)
self.sock.bind(("", self.port))
def run(self):
self.event.set()
self.logger.debug("PID: %d" % multiprocessing.current_process().pid)
while self.event.is_set():
try:
message, (ip, port) = self.sock.recvfrom(self.datagram_size)
teacher_discovered.send(sender=self)
self.logger.debug("Received: %s from: %s" % (message, ip))
except socket.timeout:
self.logger.debug("%s timeout" % multiprocessing.current_process().name)
time.sleep(1)
def stop(self):
self.logger.debug("Client will halt.")
self.event.clear()
self.sock.close()
self.terminate()
class Logger(object):
def __init__(self, filename):
self.qtag = Queue()
self.done = Event()
self.tag = None
self.filename = filename
self.file = None
def start(self):
self.file = open(self.filename, 'w')
print 'Opened',self.filename,'for writing.'
def set_tag(self, tag):
self.qtag.put(tag)
def set_done(self):
self.done.set()
def log(self, nodeid, msgid, data):
if not self.qtag.empty():
self.tag = self.qtag.get()
if self.done.is_set():
self.done.clear()
return True
L = ['%f'%time.time(), '%d'%nodeid, '%d'%msgid] + map(str,data)
if self.tag:
L.append(self.tag)
print >>self.file, ','.join(L)
self.file.flush()
def close(self):
if self.file:
self.file.close()
print 'File closed.'
def face_proc(self, child_face_recog: Pipe, e_new_person: Event):
"""
Parallel process of saving people for face recognition
Arguments:
child_face_recog {Pipe} -- pipe for communication with parent process,
recieve ROI ndarray type of recognized object
"""
if not os.path.exists('humans'):
print('created', os.getcwd())
os.mkdir('humans')
else:
print('exist')
os.chdir(os.path.join(os.getcwd(), 'humans'))
is_first = True
counter = 0
while True:
if e_new_person.is_set():
counter = 0
if not is_first:
os.chdir('..')
new_dir = datetime.datetime.now().strftime("%Y-%m-%d-%H-%M-%S")
os.mkdir(new_dir)
print('Created', os.getcwd() + new_dir)
os.chdir(os.path.join(os.getcwd(), new_dir))
e_new_person.clear()
is_first = False
image = child_face_recog.recv()
cv.imwrite(filename=str(counter) + '.jpg', img=image)
print('image saved:', os.getcwd() + str(counter) + '.jpg')
counter += 1
def test_recycle(r):
# set up the Sentinel
list_key = 'test_recycle_test:q'
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
start_event = Event()
stop_event = Event()
# override settings
Conf.RECYCLE = 2
Conf.WORKERS = 1
# set a timer to stop the Sentinel
threading.Timer(3, stop_event.set).start()
s = Sentinel(stop_event, start_event, list_key=list_key)
assert start_event.is_set()
assert s.status() == Conf.STOPPED
assert s.reincarnations == 1
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
async('django_q.tests.tasks.multiply', 2, 2, list_key=list_key, redis=r)
task_queue = Queue()
result_queue = Queue()
# push two tasks
pusher(task_queue, stop_event, list_key=list_key, r=r)
pusher(task_queue, stop_event, list_key=list_key, r=r)
# worker should exit on recycle
worker(task_queue, result_queue, Value('f', -1))
# check if the work has been done
assert result_queue.qsize() == 2
# save_limit test
Conf.SAVE_LIMIT = 1
result_queue.put('STOP')
# run monitor
monitor(result_queue)
assert Success.objects.count() == Conf.SAVE_LIMIT
r.delete(list_key)
class DeviceServer(ThreadedTCPServer, Process):
#causes handle_request to return
timeout = 1
def __init__(self, mux, muxdevice, server_address, RequestHandlerClass):
Process.__init__(self)
ThreadedTCPServer.__init__(self, server_address, RequestHandlerClass)
self.mux = mux
self.muxdev = muxdevice
self._stop = Event()
def stop(self):
self._stop.set()
def stopped(self):
return self._stop.is_set()
def run(self):
if self.stopped():
_LOGGER.warning("Thread already stopped")
while not self.stopped():
self.handle_request()
self.socket.close()
_LOGGER.debug("%s will exit now" % (str(self)))
class AMQPInput(BaseInput):
log = logging.getLogger("%s.AMQPInput" %(__name__))
def __init__(self, aggr_config, namespace="local", config={}):
super(AMQPInput, self).__init__(aggr_config, namespace, config)
self.exit = Event()
def run(self):
self.log.info("Connecting to (%s): %s" %(self.outputType, self.outputUri))
aggrConn = self.connectAggregator()
rabbitmqConsumer = RabbitMQConsumer(**self.config)
def callback(event):
if not event.has_key('namespace'):
event['namespace'] = self.namespace
aggrConn.send(event)
if self.exit.is_set():
aggrConn.close()
rabbitmqConsumer.stop()
rabbitmqConsumer.userCallbacks = [ callback ]
rabbitmqConsumer.start()
def shutdown(self):
self.exit.set()
def run_server_until_file_change(serve_forever_path, verbosity=0):
files_queue = Queue()
file_changed = Event()
worker = Worker(serve_forever_path, files_queue, os.getpid())
worker.daemon = True
monitor = Monitor(lambda: file_changed.set())
monitor.start()
worker.start()
print("started server with PID %s" % worker.pid)
while (not file_changed.is_set()) and worker.is_alive():
try:
path = files_queue.get(timeout=1)
except queue.Empty:
pass
else:
if path.endswith('testapp.py'):
print("adding %s to paths" % path)
monitor.add_path(path)
monitor.stop()
print("Waiting for monitor thread in %s to quit" % worker.pid)
monitor.join()
print("Waiting for server %s to quit" % worker.pid)
worker.terminate()
worker.join()
print("server with PID %s done" % worker.pid)
class DataProcess(Process):
def __init__(self, data_pipeline, **get_batch_kwargs):
super(DataProcess, self).__init__(name='neuralnilm-data-process')
self._stop = Event()
self._queue = Queue(maxsize=3)
self.data_pipeline = data_pipeline
self._get_batch_kwargs = get_batch_kwargs
def run(self):
batch = self.data_pipeline.get_batch(**self._get_batch_kwargs)
while not self._stop.is_set():
try:
self._queue.put(batch)
except AssertionError:
# queue is closed
break
batch = self.data_pipeline.get_batch(**self._get_batch_kwargs)
def get_batch(self, timeout=30):
if self.is_alive():
return self._queue.get(timeout=timeout)
else:
raise RuntimeError("Process is not running!")
def stop(self):
self._stop.set()
self._queue.close()
self.terminate()
self.join()
class CaptureProcess(Process):
"""A process that fills a queue with images as captured from
a camera feed"""
def __init__(self, capture, imageQueue):
Process.__init__(self, name="Capture")
self.imageQueue = imageQueue
self.capture = capture
self.keepGoing = Event()
self.keepGoing.set()
self.daemon = True
def run(self):
print "CaptureProcess pid: %s" % (self.pid,)
while self.keepGoing.is_set():
image = captureImage(self.capture)
# sys.stdout.write(".")
try:
self.imageQueue.put(serializeImage(image), block=True, timeout=0.25)
except FullException:
try:
_ = self.imageQueue.get_nowait()
except:
pass # Try to clear the queue, but don't worry if someone snatches it first
def stop(self):
self.keepGoing.clear()
def do_the_transfer(source, target, cmd, parameters, filename, size, runname, clean=False, logfile=None):
logger.debug("starting transfer")
if ("gsiftp" in source) or ("gsiftp" in target):
if not cmd:
cmd = "globus-url-copy"
transfer = Gridftp(source+filename, target+filename, cmd, parameters)
else:
if not cmd:
cmd = 'scp'
transfer = Scp(source+filename, target+filename, cmd, parameters)
results_sizes[runname] = size
results_transfer[runname] = transfer
file_temp = get_source_file(size, working_directory, clean)
transfer.prepare(file_temp, clean)
#logger.info("Starting transfer:\n\t"+str(gridftp))
transfer_cmd = transfer.transfer_command()
event = Event()
e = threading.Thread(target=execution, args=(event, transfer))
t = threading.Thread(target=timing, args=(event, size, results_time, results_speed, runname, transfer, logfile))
t.start()
e.start()
while not event.is_set():
time.sleep(1)
class BroadcastServer(Process):
def __init__(self, ip, port, message, name="BroadcastServer"):
Process.__init__(self, name=name)
self.logger = multiprocessing.get_logger()
self.event = Event()
self.message = message
self.ip = ip
self.port = port
self.sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
self.sock.bind(("", 0))
def run(self):
self.event.set()
self.logger.debug("PID: %d" % multiprocessing.current_process().pid)
while self.event.is_set():
self.logger.debug("Sending: %s" % self.message)
self.sock.sendto(self.message, (self.ip, self.port))
time.sleep(1)
def stop(self):
self.logger.debug("Server will halt.")
self.event.clear()
self.terminate()
class StoppableProcess(Process):
""" Base class for Processes which require the ability
to be stopped by a process-safe method call
"""
def __init__(self):
self._should_stop = Event()
self._should_stop.clear()
super(StoppableProcess, self).__init__()
def join(self, timeout=0):
""" Joins the current process and forces it to stop after
the timeout if necessary
:param timeout: Timeout duration in seconds
"""
self._should_stop.wait(timeout)
if not self.should_stop():
self.stop()
super(StoppableProcess, self).join(0)
def stop(self):
self._should_stop.set()
def should_stop(self):
return self._should_stop.is_set()
def __repr__(self):
return "<%s(should_stop=%s)>" % (
self.__class__.__name__, self.should_stop())
def start_spider(spider_cls):
try:
result_queue = Queue()
network_response_queue = Queue()
shutdown_event = Event()
generator_done_event = Event()
taskq = QueueBackend('ng')
#from grab.spider.base import logger_verbose
#logger_verbose.setLevel(logging.DEBUG)
kwargs = {
'taskq': taskq,
'result_queue': result_queue,
'network_response_queue': network_response_queue,
'shutdown_event': shutdown_event,
'generator_done_event': generator_done_event,
'ng': True,
}
# Generator: OK
generator_waiting_shutdown_event = Event()
bot = spider_cls(waiting_shutdown_event=generator_waiting_shutdown_event, **kwargs)
generator = Process(target=bot.run_generator)
generator.start()
# Downloader: OK
downloader_waiting_shutdown_event = Event()
bot = spider_cls(waiting_shutdown_event=downloader_waiting_shutdown_event,
**kwargs)
downloader = Process(target=bot.run)
downloader.start()
# Parser: OK
events = []
for x in xrange(2):
parser_waiting_shutdown_event = Event()
events.append(parser_waiting_shutdown_event)
bot = spider_cls(waiting_shutdown_event=parser_waiting_shutdown_event,
**kwargs)
parser = Process(target=bot.run_parser)
parser.start()
while True:
time.sleep(2)
print('task size', taskq.size())
print('response size', network_response_queue.qsize())
if (downloader_waiting_shutdown_event.is_set() and
all(x.is_set() for x in events)):
shutdown_event.set()
break
time.sleep(1)
print('done')
finally:
for child in active_children():
logging.debug('Killing child process (pid=%d)' % child.pid)
child.terminate()
def _start_async_server(self):
self.server = LiveSyncSocketServer(port=self.liveport)
server_started_event = Event()
self.server_process = Process(target=self.server.start, args=(server_started_event,))
self.server_process.daemon = True
self.server_process.start()
server_started_event.wait(timeout=0.1)
return server_started_event.is_set()
class Connector(object):
def __init__(self, reply_generator: ConnectorReplyGenerator, connectors_event: Event):
self._reply_generator = reply_generator
self._scheduler = None
self._thread = Thread(target=self.run)
self._write_queue = Queue()
self._read_queue = Queue()
self._frontends_event = connectors_event
self._shutdown_event = Event()
self._muted = True
def give_nlp(self, nlp):
self._reply_generator.give_nlp(nlp)
def start(self):
self._scheduler.start()
self._thread.start()
def run(self):
while not self._shutdown_event.is_set():
message = self._scheduler.recv(timeout=0.2)
if self._muted:
self._scheduler.send(None)
elif message is not None:
# Receive the message and put it in a queue
self._read_queue.put(message)
# Notify main program to wakeup and check for messages
self._frontends_event.set()
# Send the reply
reply = self._write_queue.get()
self._scheduler.send(reply)
def send(self, message: str):
self._write_queue.put(message)
def recv(self) -> Optional[ConnectorRecvMessage]:
if not self._read_queue.empty():
return self._read_queue.get()
return None
def shutdown(self):
# Shutdown event signals both our thread and process to shutdown
self._shutdown_event.set()
self._scheduler.shutdown()
self._thread.join()
def generate(self, message: str, doc: Doc=None) -> str:
return self._reply_generator.generate(message, doc)
def mute(self):
self._muted = True
def unmute(self):
self._muted = False
def empty(self):
return self._read_queue.empty()
class MainProcess(Process):
## Constructor
#
# Inits the process, the CSV file, the PCA9547 Multiplexor, the settings
# file for each sensor (with calibration)
# @param self The object pointer
def __init__(self):
Process.__init__(self)
self.exit = Event()
self.csv = CSVExport()
self.i2cMux = PCA9547()
self.settings0 = RTIMU.Settings(SETTINGS_FILE_0)
self.settings1 = RTIMU.Settings(SETTINGS_FILE_1)
self.settings2 = RTIMU.Settings(SETTINGS_FILE_2)
self.settings3 = RTIMU.Settings(SETTINGS_FILE_3)
self.settings4 = RTIMU.Settings(SETTINGS_FILE_4)
self.settings5 = RTIMU.Settings(SETTINGS_FILE_5)
self.imu0 = RTIMU.RTIMU(self.settings0)
self.imu1 = RTIMU.RTIMU(self.settings1)
self.imu2 = RTIMU.RTIMU(self.settings2)
self.imu3 = RTIMU.RTIMU(self.settings3)
self.imu4 = RTIMU.RTIMU(self.settings4)
self.imu5 = RTIMU.RTIMU(self.settings5)
self.settings = [
self.settings0, self.settings1, self.settings2, self.settings3,
self.settings4, self.settings5
]
self.imus = [
self.imu0, self.imu1, self.imu2, self.imu3, self.imu4, self.imu5
]
self.detectedIMU = [False] * 6
self.detectImu()
self.pollInterval = 2
## Detectes the number of IMUs avaliable in each channel
#
# @param self The object pointer
# @return number of detected IMUs
def detectImu(self):
try:
for n, imu in enumerate(self.imus):
self.i2cMux.setChannel(n)
if imu.IMUInit():
self.detectedIMU[n] = True
else:
self.detectedIMU[n] = False
except:
print("Error in detection")
return self.detectedIMU
def setPollInterval(self, pollInterval):
self.pollInterval = pollInterval
## Sets file name for the CSV file
#
# @param self The object pointer
# @param txt file name
def setFileName(self, txt):
self.csv.setTitle(txt)
## gets the current data from the current sensor
#
# @param self The object pointer
# @return all sensor data as a vector
def getData(self):
return self.allData
## stops the thread and signals to exit
#
# @param self The object pointer
def stop(self):
self.exit.set()
def saveSettings(self):
try:
for s in self.setting:
s.save()
except:
print("Error in detection")
return self.detectedIMU
## Thread loop
#
# Get data in the polling interval definid time, and stores
# data form all the avaliable sensor with his identificator and timestamp
# @param self The object pointer
# @return finalized job
def run(self):
self.csv.createFile()
init_time = time.time()
while not self.exit.is_set():
try:
for n, imu in enumerate(self.imus):
if self.detectedIMU[n]:
self.i2cMux.setChannel(n)
if imu.IMURead():
data0 = imu.getIMUData()
quaternion = data0["fusionQPose"]
acceleration = data0["accel"]
gyro = data0["gyro"]
compass = data0["compass"]
nowTime = time.time() - init_time
self.allData = [n] + \
[nowTime] + \
[acceleration[0]] + \
[acceleration[1]] + \
[acceleration[2]] + \
[gyro[0]] + \
[gyro[1]] + \
[gyro[2]] + \
[compass[0]] + \
[compass[1]] + \
[compass[2]] + \
[quaternion[0]] + \
[quaternion[1]] + \
[quaternion[2]] + \
[quaternion[3]]
self.csv.csvWrite(self.allData)
#print(self.allData)
time.sleep(self.pollInterval * 1.0 / 1000.0)
except (KeyboardInterrupt, SystemExit):
print("Finishing Thread bad")
self.exit.set()
print("Finished Adquisition")
return
class Cluster:
def __init__(self, broker: Broker = None):
self.broker = broker or get_broker()
self.sentinel = None
self.stop_event = None
self.start_event = None
self.pid = current_process().pid
self.cluster_id = uuid.uuid4()
self.host = socket.gethostname()
self.timeout = Conf.TIMEOUT
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
def start(self) -> int:
# Start Sentinel
self.stop_event = Event()
self.start_event = Event()
self.sentinel = Process(
target=Sentinel,
args=(
self.stop_event,
self.start_event,
self.cluster_id,
self.broker,
self.timeout,
),
)
self.sentinel.start()
logger.info(_(f"Q Cluster {self.name} starting."))
while not self.start_event.is_set():
sleep(0.1)
return self.pid
def stop(self) -> bool:
if not self.sentinel.is_alive():
return False
logger.info(_(f"Q Cluster {self.name} stopping."))
self.stop_event.set()
self.sentinel.join()
logger.info(_(f"Q Cluster {self.name} has stopped."))
self.start_event = None
self.stop_event = None
return True
def sig_handler(self, signum, frame):
logger.debug(
_(f'{current_process().name} got signal {Conf.SIGNAL_NAMES.get(signum, "UNKNOWN")}'
))
self.stop()
@property
def stat(self) -> Status:
if self.sentinel:
return Stat.get(pid=self.pid, cluster_id=self.cluster_id)
return Status(pid=self.pid, cluster_id=self.cluster_id)
@property
def name(self) -> str:
return humanize(self.cluster_id.hex)
@property
def is_starting(self) -> bool:
return self.stop_event and self.start_event and not self.start_event.is_set(
)
@property
def is_running(self) -> bool:
return self.stop_event and self.start_event and self.start_event.is_set(
)
@property
def is_stopping(self) -> bool:
return (self.stop_event and self.start_event
and self.start_event.is_set() and self.stop_event.is_set())
@property
def has_stopped(self) -> bool:
return self.start_event is None and self.stop_event is None and self.sentinel
class Thread():
""" Leverages multiprocessing(Process, Event)
Wraps a provided function in a thread that can run independently until stopped
Functions provided should be designed to run a finite number of times so that
a loop doesn't cause the function to never release control to the thread.
Setting executions to anything other than exactly int() > 0 will cause the thread
to run until stopped.
For functions that modify class attributes, the attribute must be set up to use
a shared memory variable, otherwise the attribute change won't persist the thread.
See camera.Camera().capture() for an example
"""
def __init__(self,
target,
*args,
executions: int = 0,
start=False,
**kwargs):
self.inputs = locals() # For posterity
self._target = target
self._args = args
self._kwargs = kwargs
self._stop_event = Event()
self.executions = executions if isinstance(executions, int) else 0
self.results = list() # So results are retrievable
self._setup()
if start:
self.start()
def _setup(self):
# Spawns a process
self.process = Process(target=self._target,
args=self._args,
kwargs=self._kwargs)
# override the default Process().run() method with Thread()._run()
self.process.run = self._run
def start(self):
# Start the process in this thread
self.process.start()
def stop(self, *args, **kwargs):
# Shortcut for stop_gracefully()
return self.stop_gracefully(*args, **kwargs)
def stop_gracefully(self, *args, **kwargs):
""" Allow the thread to come to the end of an iteration, and stop.
"""
self._stop_event.set()
self.process.join(*args, **kwargs)
while self.is_alive():
time.sleep(0.001) # check every millisecond
self.process.close()
return True
def stop_immediately(self, *args, **kwargs):
""" Send a SIGKILL to the thread, halting it without allowing the thread
to come to the end of an iteration.
"""
self._stop_event.set()
self.process.kill(*args, **kwargs)
while self.is_alive():
time.sleep(0.001) # check every millisecond
self.process.close()
return True
def restart(self):
""" Restart the thread if stopped, or stop_immediately, reset, and start()
"""
if not self._stop_event.is_set():
self.stop_immediately()
self._stop_event.clear()
self._setup()
self.start()
def is_alive(self):
return self.process.is_alive()
def _run(self):
""" This method is used to replace the default Process().run() method,
which is executed on Process().start()
"""
if self.executions > 0:
# Run n times
remaining = self.executions
while remaining > 0:
self.results.append(self._target(*self._args, **self._kwargs))
remaining -= 1
# Cleanup
self.process.close()
else:
# Run until stopped
while not self._stop_event.is_set():
self.results.append(self._target(*self._args, **self._kwargs))
'''
if self.con.acquire():
while self.last_cnt >= self.counter:
self.con.wait()
index = self.last_cnt
self.last_cnt += 1
self.con.release()
return self.container[index]
'''
def close(self):
self.socket.close()
lock = threading.Lock()#互斥锁
con = threading.Condition()#为了轮流读取两个服务器的数据,不需要互斥锁了
event = Event()
print(" is start receive sensor data ? ",end = ":")
print(event.is_set())
mythread1 = myThread(host1,port1,con,event)
mythread1.start()
event.set()
print(" is start receive sensor data ? ",end = ":")
def showData(data):
for item in data:
print(np.array(item))
print("================")
i = 0
thresh = 40
def saveImageData(sensor1,path,avgtemp):
np.save(path+"/imagedata.npy",np.array(sensor1))
np.save(path+"/avgtemp.npy",avgtemp)
i = 0
class WalletRPCManager(ProcessManager):
def __init__(self,
resources_path,
wallet_file_path,
wallet_password,
app,
log_level=1,
enable_ssl=False):
self.user_agent = str(uuid4().hex)
enable_ssl = False
wallet_log_path = os.path.join(os.path.dirname(wallet_file_path),
"ryo-wallet-rpc.log")
log_level = 2
wallet_rpc_args = u'%s/bin/ryo-wallet-rpc --disable-rpc-login --prompt-for-password --daemon-address %s --wallet-file %s --log-file %s --rpc-bind-port %d --log-level %d' \
% (resources_path, REMOTE_DAEMON_ADDRESS, wallet_file_path, wallet_log_path, WALLET_RPC_PORT, log_level)
print(wallet_rpc_args)
ProcessManager.__init__(self, wallet_rpc_args, "ryo-wallet-rpc")
sleep(0.2)
self.send_command(wallet_password)
self.rpc_request = WalletRPCRequest(app, self.user_agent, enable_ssl)
# self.rpc_request.start()
self._stopped = False
self._ready = Event()
self.block_height = 0
self.is_password_invalid = Event()
self.last_log_lines = []
self.last_error = ""
def run(self):
rpc_ready_strs = [
"Binding on 127.0.0.1:%d" % WALLET_RPC_PORT,
"Starting wallet RPC server", "Run net_service loop",
"Refresh done", "RPC server ready"
]
err_str = "ERROR"
invalid_password_str = "invalid password"
height_regex = re.compile(
r"Processed block: \<([a-z0-9]+)\>, height (\d+)")
height_regex2 = re.compile(r"Skipped block by height: (\d+)")
height_regex3 = re.compile(
r"Skipped block by timestamp, height: (\d+)")
for line in iter(self.proc.stdout.readline, b''):
if self._stopped: break
m_height = height_regex.search(line)
if m_height: self.block_height = m_height.group(2)
if not m_height:
m_height = height_regex2.search(line)
if m_height: self.block_height = m_height.group(1)
if not m_height:
m_height = height_regex3.search(line)
if m_height: self.block_height = m_height.group(1)
if not self._ready.is_set() and any(s in line
for s in rpc_ready_strs):
self._ready.set()
log("RPC server ready!", LEVEL_INFO, self.proc_name)
if err_str in line:
self.last_error = line.rstrip()
if not self.is_password_invalid.is_set(
) and invalid_password_str in line:
self.is_password_invalid.set()
log("ERROR: Invalid wallet password", LEVEL_ERROR,
self.proc_name)
else:
log(self.last_error, LEVEL_ERROR, self.proc_name)
elif m_height:
log(line.rstrip(), LEVEL_INFO, self.proc_name)
else:
log(line.rstrip(), LEVEL_DEBUG, self.proc_name)
if len(self.last_log_lines) > 1:
self.last_log_lines.pop(0)
self.last_log_lines.append(line[:120])
if not self.proc.stdout.closed:
self.proc.stdout.close()
def is_ready(self):
return self._ready.is_set()
def is_invalid_password(self):
return self.is_password_invalid.is_set()
def stop(self, force=False):
if not force: self.rpc_request.stop_wallet()
if self.is_proc_running():
counter = 0
while True:
if self.is_proc_running():
if counter < 60:
sleep(1)
counter += 1
else:
self.proc.kill()
log("[%s] killed" % self.proc_name, LEVEL_INFO,
self.proc_name)
break
else:
break
self._stopped = True
self._ready = Event()
self.block_height = 0
self.is_password_invalid = Event()
self.last_log_lines = []
self.last_error = ""
log("[%s] stopped" % self.proc_name, LEVEL_INFO, self.proc_name)
class DataProcess(Process):
def __init__(self, data_queue, data_paths, repeat=True):
'''
data_queue : Multiprocessing queue
data_paths : list of data and label pair used to load data
repeat : if set True, return data until exit is set
'''
super(DataProcess, self).__init__()
# Queue to transfer the loaded mini batches
self.data_queue = data_queue
self.data_paths = data_paths
self.num_data = len(data_paths)
self.repeat = repeat
# Tuple of data shape
self.batch_size = cfg.CONST.BATCH_SIZE
self.exit = Event()
self.shuffle_db_inds()
def shuffle_db_inds(self):
# Randomly permute the training roidb
if self.repeat:
self.perm = np.random.permutation(np.arange(self.num_data))
else:
self.perm = np.arange(self.num_data)
self.cur = 0
def get_next_minibatch(self):
if (self.cur + self.batch_size) >= self.num_data and self.repeat:
self.shuffle_db_inds()
db_inds = self.perm[self.cur:min(self.cur +
self.batch_size, self.num_data)]
self.cur += self.batch_size
return db_inds
def shutdown(self):
self.exit.set()
@print_error
def run(self):
iteration = 0
# Run the loop until exit flag is set
while not self.exit.is_set() and self.cur <= self.num_data:
# Ensure that the network sees (almost) all data per epoch
db_inds = self.get_next_minibatch()
data_list = []
label_list = []
for batch_id, db_ind in enumerate(db_inds):
datum = self.load_datum(self.data_paths[db_ind])
label = self.load_label(self.data_paths[db_ind])
data_list.append(datum)
label_list.append(label)
batch_data = np.array(data_list).astype(np.float32)
batch_label = np.array(label_list).astype(np.float32)
# The following will wait until the queue frees
self.data_queue.put((batch_data, batch_label), block=True)
iteration += 1
def load_datum(self, path):
pass
def load_label(self, path):
pass
class ClockBaseInterruptBehavior(ClockBaseBehavior):
'''A kivy clock which can be interrupted during a frame to execute events.
'''
interupt_next_only = False
_event = None
_get_min_timeout_func = None
def __init__(self, interupt_next_only=False, **kwargs):
super(ClockBaseInterruptBehavior, self).__init__(**kwargs)
self._event = MultiprocessingEvent() if PY2 else ThreadingEvent()
self.interupt_next_only = interupt_next_only
self._get_min_timeout_func = self.get_min_timeout
def usleep(self, microseconds):
self._event.clear()
self._event.wait(microseconds / 1000000.)
def on_schedule(self, event):
fps = self._max_fps
if not fps:
return
if not event.timeout or (
not self.interupt_next_only
and event.timeout <= 1 / fps - # remaining time
(self.time() - self._last_tick) + # elapsed time
4 / 5. * self.get_resolution()): # resolution fudge factor
self._event.set()
def idle(self):
fps = self._max_fps
event = self._event
resolution = self.get_resolution()
if fps > 0:
done, sleeptime = self._check_ready(fps, resolution,
4 / 5. * resolution)
if not done:
event.wait(sleeptime)
current = self.time()
self._dt = current - self._last_tick
self._last_tick = current
event.clear()
# anything scheduled from now on, if scheduled for the upcoming frame
# will cause a timeout of the event on the next idle due to on_schedule
# `self._last_tick = current` must happen before clear, otherwise the
# on_schedule computation is wrong when exec between the clear and
# the `self._last_tick = current` bytecode.
return current
def _check_ready(self, fps, min_sleep, undershoot):
if self._event.is_set():
return True, 0
t = self._get_min_timeout_func()
if not t:
return True, 0
if not self.interupt_next_only:
curr_t = self.time()
sleeptime = min(1 / fps - (curr_t - self._last_tick), t - curr_t)
else:
sleeptime = 1 / fps - (self.time() - self._last_tick)
return sleeptime - undershoot <= min_sleep, sleeptime - undershoot
class MyApp(QMainWindow):
def __init__(self):
super(MyApp, self).__init__()
self.ui = Ui_MainWindow()
self.ui.closeEvent = self.closeEvent
self.ui.setupUi(self)
self.botinstances = []
self.cookies = []
self.newrow = []
self.killevents = []
self.count = 0
self.ui.addRow.clicked.connect(self.AddRow)
self.ui.verify.clicked.connect(self.verification)
self.ui.runAll.hide()
self.ui.runAll.clicked.connect(self.runall)
self.ui.actionOpen.setShortcut("Ctrl+R")
self.ui.actionOpen.triggered.connect(self.open)
self.ui.actionOpen.setShortcut("Ctrl+O")
self.ui.actionSave.triggered.connect(self.save)
self.ui.actionSave.setShortcut("Ctrl+S")
# Start cookie listening
self.qcookies = Queue()
self.endcookies = Event()
self.cookieplacer = threading.Thread(target=self.CookieListener)
self.cookieplacer.start()
def save(self):
Qname = QFileDialog.getSaveFileName(self, 'Save file', '',
"HolyCopBot Files (*.hcb)")
if Qname:
name = Qname[0]
linklist = []
proxylist = []
sizelist = []
for i in range(self.count):
linklist.append(self.newrow[i].urlInput.text())
proxylist.append(self.newrow[i].proxy.text() + ":" +
self.newrow[i].proxyport.text())
sizelist.append(str(self.newrow[i].sizeBox.currentText()))
hcbexport.all(name, linklist, proxylist, sizelist)
def open(self):
Qname = QFileDialog.getOpenFileName(self, 'Open file', '',
"HolyCopBot Files (*.hcb)")[0]
if Qname:
name = Qname
filecontents = hcbimport.all(name)
linklist = filecontents[0]
proxylist = filecontents[1]
proxyportlist = filecontents[2]
sizelist = filecontents[3]
for i in range(len(linklist)):
newest = self.count
self.AddRow()
self.newrow[newest].urlInput.setText(linklist[i])
self.newrow[newest].proxy.setText(proxylist[i])
self.newrow[newest].proxyport.setText(proxyportlist[i])
index = self.newrow[newest].sizeBox.findText(
sizelist[i], QtCore.Qt.MatchFixedString)
if index >= 0:
self.newrow[newest].sizeBox.setCurrentIndex(index)
def verification(self):
# api verification check
key = self.ui.verifyKey.text()
verified = apicall.verify(key)
if verified:
self.ui.verify.hide()
self.ui.verifyKey.hide()
self.ui.verifyLabel.hide()
self.ui.runAll.show()
self.ui.addRow.setEnabled(True)
self.ui.actionOpen.setEnabled(True)
self.ui.actionSave.setEnabled(True)
else:
reply = QMessageBox.question(
self, 'Not Registered', "Your verification key was incorrect",
QMessageBox.Ok)
if reply == QMessageBox.Ok:
self.close()
def runall(self):
if self.count == 0:
return
else:
self.ui.runAll.setEnabled(False)
for i in range(self.count):
self.RunBot(i)
def AddRow(self):
#limit of concurrent bots
if self.count > 20:
return
self.newrow.append(RowWidget(self.ui, self.count))
somenum = int(self.newrow[self.count].countLabel.text())
self.ui.verticalLayout_2.addWidget(self.newrow[self.count].widget)
self.newrow[self.count].run.clicked.connect(
lambda: self.RunBot(somenum))
self.newrow[self.count].openCart.clicked.connect(
lambda: self.OpenCart(somenum))
self.newrow[self.count].stop.clicked.connect(
lambda: self.StopProcess(somenum))
self.count += 1
def RunBot(self, somenum):
link = self.newrow[somenum].urlInput.text()
if link == "":
self.newrow[somenum].invalidUrl.setText("Invalid Url!")
else:
self.newrow[somenum].invalidUrl.setText("")
# get and validate proxy if exists
self.proxy = self.newrow[somenum].proxy.text(
) + ":" + self.newrow[somenum].proxyport.text()
ipcheck = validateIP.check(self.proxy)
if ipcheck == False or self.proxy == '':
self.proxy = 0
rawsize = str(self.newrow[somenum].sizeBox.currentText())
if rawsize == "Size":
self.newrow[somenum].invalidUrl.setText("Invalid Size!")
if not rawsize == "Size":
size = rawsize.replace(".", "")
self.newrow[somenum].run.setText("Running")
self.newrow[somenum].stop.setEnabled(True)
self.newrow[somenum].run.setEnabled(False)
self.newrow[somenum].sizeBox.setEnabled(False)
imagepreview = threading.Thread(target=self.GetPreviewImage,
args=(link, somenum))
imagepreview.start()
self.BotInstance(link, size, somenum)
def BotInstance(self, link, size, row):
kill = Event()
self.killevents.insert(row, kill)
botprocess = Process(target=launchbot.runbot,
args=(link, size, self.proxy, self.qcookies, row,
self.killevents[row]))
botprocess.start()
self.botinstances.insert(row, botprocess)
def CookieListener(self):
while not self.endcookies.is_set():
if self.qcookies.qsize() != 0:
self.ui.runAll.setEnabled(True)
result = self.qcookies.get()
placeinrow = result[0]
# error handling
if result[1] == "oos":
error = "Size Out of Stock!"
self.newrow[placeinrow].invalidUrl.setText(error)
if result[1] == 'forbidden':
error = "IP Ban! (temporary)"
self.newrow[placeinrow].invalidUrl.setText(error)
else:
newcookies = result[1]
self.cookies.insert(placeinrow, newcookies)
self.newrow[placeinrow].run.setText("Copped!")
self.newrow[placeinrow].openCart.setEnabled(True)
self.newrow[placeinrow].stop.setEnabled(False)
def OpenCart(self, somenum):
self.newrow[somenum].run.setText("Run")
self.newrow[somenum].run.setEnabled(True)
self.newrow[somenum].sizeBox.setEnabled(True)
link = self.newrow[somenum].urlInput.text()
global site
if "footlocker" in link:
site = "https://www.footlocker.com/"
if "footaction" in link:
site = "https://www.footaction.com/"
if "eastbay" in link:
site = "https://www.eastbay.com/"
if "champssports" in link:
site = "https://www.champssports.com/"
cartbrowser = Process(target=launchcart.opencart,
args=(site, self.cookies[somenum], self.proxy))
cartbrowser.start()
def GetPreviewImage(self, link, rownum):
img = getpreview.req(link)
# set pixmap qt object to image returned
pixmap = QPixmap()
pixmap.loadFromData(img)
self.newrow[rownum].previewImage.setPixmap(pixmap)
def StopProcess(self, rownum):
# set event for botinstance to kill, then wait for event to be unset, reset gui
killevent = self.killevents[rownum]
killevent.set()
while killevent.is_set():
time.sleep(.1)
self.botinstances[rownum].terminate()
self.newrow[rownum].run.setText("Run")
self.newrow[rownum].stop.setEnabled(False)
self.newrow[rownum].run.setEnabled(True)
self.newrow[rownum].sizeBox.setEnabled(True)
def closeEvent(self, event):
if self.count == 0:
self.endcookies.set()
sys.exit()
reply = QMessageBox.question(self, 'Confirm Exit',
"Are you sure you want to quit?",
QMessageBox.Yes | QMessageBox.No,
QMessageBox.No)
if reply == QMessageBox.Yes:
self.endcookies.set()
time.sleep(0.15)
event.accept()
else:
event.ignore()
class WalletCliManager(ProcessManager):
fail_to_connect_str = "wallet failed to connect to daemon"
def __init__(self,
resources_path,
wallet_file_path,
wallet_log_path,
restore_wallet=False,
restore_height=0):
if not restore_wallet:
wallet_args = u'%s/bin/ryo-wallet-cli --daemon-address %s --generate-new-wallet=%s --log-file=%s ' \
% (resources_path, REMOTE_DAEMON_ADDRESS, wallet_file_path, wallet_log_path)
else:
restore_height = 0
wallet_args = u'%s/bin/ryo-wallet-cli --daemon-address %s --log-file=%s --restore-deterministic-wallet --create-address-file --restore-height %d' \
% (resources_path, "fakehost", wallet_log_path, restore_height)
ProcessManager.__init__(self, wallet_args, "ryo-wallet-cli")
self.ready = Event()
self.last_error = ""
def run(self):
is_ready_str = "Background refresh thread started"
err_str = "Error:"
for line in iter(self.proc.stdout.readline, b''):
if not self.ready.is_set() and is_ready_str in line:
self.ready.set()
log("Wallet ready!", LEVEL_INFO, self.proc_name)
elif err_str in line:
self.last_error = line.rstrip()
log("[%s]>>> %s" % (self.proc_name, line.rstrip()),
LEVEL_ERROR, self.proc_name)
else:
log("[%s]>>> %s" % (self.proc_name, line.rstrip()),
LEVEL_DEBUG, self.proc_name)
if not self.proc.stdout.closed:
self.proc.stdout.close()
def is_ready(self):
return self.ready.is_set()
def is_connected(self):
self.send_command("refresh")
if self.fail_to_connect_str in self.last_error:
return False
return True
def stop(self):
if self.is_proc_running():
self.send_command('exit')
#self.proc.stdin.close()
counter = 0
while True:
if self.is_proc_running():
if counter < 10:
if counter == 2:
try:
self.send_command('exit')
except:
pass
sleep(1)
counter += 1
else:
self.proc.kill()
log("[%s] killed" % self.proc_name, LEVEL_INFO,
self.proc_name)
break
else:
break
log("[%s] stopped" % self.proc_name, LEVEL_INFO, self.proc_name)
class MainWindow(QMainWindow):
""" sets up the main window"""
def __init__(self, parent=None):
super(MainWindow, self).__init__(parent)
self.setWindowTitle("Spectrum Sense")
self.view = view = ViewWidget()
self.grid = grid = GridWidget()
self.gps = gps = GPSWidget()
self.textUpdate = textUpdate = QTextBrowser()
tabwidget = QTabWidget()
tabwidget.addTab(view, 'View')
tabwidget.addTab(grid, 'Grid')
tabwidget.addTab(gps, 'GPS')
splitter = QSplitter(Qt.Vertical)
splitter.addWidget(tabwidget)
splitter.addWidget(textUpdate)
splitter.setStretchFactor(0, 3)
splitter.setStretchFactor(1, 1)
self.setCentralWidget(splitter)
view.startRecord.connect(self.record)
grid.startRecord.connect(self.record)
gps.startRecord.connect(self.record)
gps.noGpsdevice.connect(
lambda: self.textUpdate.append('[GPS] No device found.'))
gps.noPositionLock.connect(lambda: self.textUpdate.append(
'[GPS] Position lock not obtained.'))
view.startPlot.connect(self.plot)
self.radio = radio = RadioStreamer()
self.plotEvent = Event()
self.threadpool = QThreadPool()
self.dataqueue = Queue()
self.threadpool.setMaxThreadCount(2)
def plot(self, params):
""" method for plotting"""
if not self.plotEvent.is_set():
self.plotEvent.set()
self.threadpool.waitForDone()
self.plotEvent.clear()
self.radio.setup(params, self.plotEvent)
self.view.canvas.initPlot(
np.linspace(params['centerFrequency'] - params['bandwidth'] / 2,
params['centerFrequency'] + params['bandwidth'] / 2,
params['fftsize']), np.zeros(params['fftsize']))
task1 = Worker(self.radio.stream, self.dataqueue, params['fftsize'],
params['window'], 0.3)
task2 = Worker(self.view.canvas.updatePlot, self.dataqueue)
self.threadpool.start(task1)
self.threadpool.start(task2)
def record(self, params, id):
""" method for recording"""
if not self.plotEvent.is_set():
self.plotEvent.set()
self.threadpool.waitForDone()
self.plotEvent.clear()
if id == 'VIEW':
self.textUpdate.append('[VIEW] Recording requested.')
filename = 'data/static/capture.bin'
self.radio.setup(params, self.plotEvent)
try:
self.radio.record(
filename, int(params['recordTime'] * params['bandwidth']))
self.textUpdate.append(
'[VIEW] Recording finished. Data Saved to:{}'.format(
filename))
except Exception:
self.textUpdate.append(
'[VIEW] Overflow occured. Captured data may not be valid: {}'
.format(filename))
self.plot(params)
if id == 'MAP':
self.textUpdate.append('[MAP] Recording requested.')
lat = params['position'][0]
lng = params['position'][1]
filename = 'data/geo/lat_' + str(lat) + 'lng_' + str(lng) + '.bin'
self.radio.setup(params, self.plotEvent)
try:
self.radio.record(
filename, int(params['recordTime'] * params['bandwidth']))
self.textUpdate.append(
'[MAP] Recording finished. Data Saved to:{}'.format(
filename))
except Exception:
self.textUpdate.append(
'[MAP] Overflow occured. Captured data may not be valid: {}'
.format(filename))
if id == 'GRID':
self.textUpdate.append('[GRID] Recording requested.')
x = params['position'][0]
y = params['position'][1]
step = min(params['step'])
filename = 'data/grid/x_' + str(x) + 'y_' + str(y) + '.bin'
self.radio.setup(params, self.plotEvent)
try:
self.radio.record(
filename, int(params['recordTime'] * params['bandwidth']))
self.textUpdate.append(
'[GRID] Recording finished. Data Saved to:{}'.format(
filename))
except Exception:
self.textUpdate.append(
'[GRID] Overflow occured. Captured data may not be valid: {}'
.format(filename))
task = Worker(self.grid.canvas.updatePlot, x, y, step,
params['recordTime'], params['bandwidth'], filename)
self.threadpool.start(task)
def closeEvent(self, event):
""" To handle the event close"""
event.ignore()
self.plotEvent.set()
self.threadpool.waitForDone()
event.accept()
class MultiThreadedAugmenter(object):
""" Makes your pipeline multi threaded. Yeah!
If seeded we guarantee that batches are retunred in the same order and with the same augmentation every time this
is run. This is realized internally by using une queue per worker and querying the queues one ofter the other.
Args:
data_loader (generator or DataLoaderBase instance): Your data loader. Must have a .next() function and return
a dict that complies with our data structure
transform (Transform instance): Any of our transformations. If you want to use multiple transformations then
use our Compose transform! Can be None (in that case no transform will be applied)
num_processes (int): number of processes
num_cached_per_queue (int): number of batches cached per process (each process has its own
multiprocessing.Queue). We found 2 to be ideal.
seeds (list of int): one seed for each worker. Must have len(num_processes).
If None then seeds = range(num_processes)
pin_memory (bool): set to True if all torch tensors in data_dict are to be pinned. Pytorch only.
"""
def __init__(self,
data_loader,
transform,
num_processes,
num_cached_per_queue=2,
seeds=None,
pin_memory=False):
self.pin_memory = pin_memory
self.transform = transform
if seeds is not None:
assert len(seeds) == num_processes
else:
seeds = [None] * num_processes
self.seeds = seeds
self.generator = data_loader
self.num_processes = num_processes
self.num_cached_per_queue = num_cached_per_queue
self._queues = []
self._processes = []
self._end_ctr = 0
self._queue_loop = 0
self.pin_memory_thread = None
self.pin_memory_queue = None
self.abort_event = Event()
def __iter__(self):
return self
def next(self):
return self.__next__()
def _next_queue(self):
r = self._queue_loop
self._queue_loop += 1
if self._queue_loop == self.num_processes:
self._queue_loop = 0
return r
def __get_next_item(self):
success = False
item = None
use_this_queue = self._next_queue()
while not success:
try:
if self.abort_event.is_set():
self._finish()
raise RuntimeError(
"MultiThreadedAugmenter.abort_event was set, something went wrong. Maybe one of "
"your workers crashed")
else:
if not self.pin_memory:
item = self._queues[use_this_queue].get(timeout=2)
success = True
else:
item = self.pin_memory_queue.get(timeout=2)
success = True
except Empty:
pass
return item
def __next__(self):
if len(self._queues) == 0:
self._start()
try:
item = self.__get_next_item()
while isinstance(item, str) and (item == "end"):
self._end_ctr += 1
if self._end_ctr == self.num_processes:
self._end_ctr = 0
self._queue_loop = 0
logging.debug(
"MultiThreadedGenerator: finished data generation")
raise StopIteration
item = self.__get_next_item()
return item
except KeyboardInterrupt:
logging.error(
"MultiThreadedGenerator: caught exception: {}".format(
sys.exc_info()))
self._finish()
raise KeyboardInterrupt
def _start(self):
if len(self._processes) == 0:
self.abort_event.clear()
logging.debug("starting workers")
self._queue_loop = 0
self._end_ctr = 0
if hasattr(self.generator, 'was_initialized'):
self.generator.was_initialized = False
for i in range(self.num_processes):
self._queues.append(Queue(self.num_cached_per_queue))
self._processes.append(
Process(target=producer,
args=(self._queues[i], self.generator,
self.transform, i, self.seeds[i],
self.abort_event)))
self._processes[-1].daemon = True
self._processes[-1].start()
if self.pin_memory:
import torch
self.pin_memory_queue = thrQueue(2)
self.pin_memory_thread = threading.Thread(
target=pin_memory_loop,
args=(self._queues, self.pin_memory_queue,
self.abort_event, torch.cuda.current_device()))
self.pin_memory_thread.daemon = True
self.pin_memory_thread.start()
else:
logging.debug(
"MultiThreadedGenerator Warning: start() has been called but workers are already running"
)
def _finish(self):
self.abort_event.set()
sleep(0.2) # allow pin memory thread to finish
if len(self._processes) != 0:
logging.debug("MultiThreadedGenerator: workers terminated")
for i, p in enumerate(self._processes):
p.terminate()
self._queues[i].close()
self._queues[i].join_thread()
self._queues = []
self._processes = []
self._queue = None
self._end_ctr = 0
self._queue_loop = 0
def restart(self):
self._finish()
self._start()
def __del__(self):
logging.debug("MultiThreadedGenerator: destructor was called")
self._finish()
class TankWorker():
SECTION = 'core'
FINISH_FILENAME = 'finish_status.yaml'
DEFAULT_CONFIG = 'load.yaml'
def __init__(self,
configs,
cli_options=None,
cfg_patches=None,
cli_args=None,
no_local=False,
log_handlers=None,
wait_lock=False,
files=None,
ammo_file=None,
api_start=False):
self.api_start = api_start
self.wait_lock = wait_lock
self.log_handlers = log_handlers if log_handlers is not None else []
self.files = [] if files is None else files
self.ammo_file = ammo_file
self.interrupted = ProcessEvent() if api_start else ThreadEvent()
self.info = TankInfo(Manager().dict()) if api_start else TankInfo(
dict())
self.config_list = self._combine_configs(configs, cli_options,
cfg_patches, cli_args,
no_local)
self.core = TankCore(self.config_list, self.interrupted, self.info)
self.folder = self.init_folder()
self.init_logging(debug=True)
is_locked = Lock.is_locked(self.core.lock_dir)
if is_locked and not self.core.config.get_option(
self.SECTION, 'ignore_lock'):
raise LockError(is_locked)
@staticmethod
def _combine_configs(run_cfgs,
cli_options=None,
cfg_patches=None,
cli_args=None,
no_local=False):
if cli_options is None:
cli_options = []
if cfg_patches is None:
cfg_patches = []
if cli_args is None:
cli_args = []
run_cfgs = run_cfgs if len(run_cfgs) > 0 else [
TankWorker.DEFAULT_CONFIG
]
if no_local:
configs = [load_cfg(cfg) for cfg in run_cfgs] + \
parse_options(cli_options) + \
parse_and_check_patches(cfg_patches) + \
cli_args
else:
configs = [load_core_base_cfg()] + \
load_local_base_cfgs() + \
[load_cfg(cfg) for cfg in run_cfgs] + \
parse_options(cli_options) + \
parse_and_check_patches(cfg_patches) + \
cli_args
return configs
def init_folder(self):
folder = self.core.artifacts_dir
if self.api_start > 0:
for f in self.files:
shutil.move(f, folder)
if self.ammo_file:
shutil.move(self.ammo_file, folder)
os.chdir(folder)
return folder
def stop(self):
self.interrupted.set()
logger.warning('Interrupting')
def get_status(self):
return {
'status_code': self.status,
'left_time': None,
'exit_code': self.retcode,
'lunapark_id': self.get_info('uploader', 'job_no'),
'tank_msg': self.msg,
'lunapark_url': self.get_info('uploader', 'web_link'),
'luna_id': self.get_info('neuploader', 'job_no'),
'luna_url': self.get_info('neuploader', 'web_link')
}
def save_finish_status(self):
with open(os.path.join(self.folder, self.FINISH_FILENAME), 'w') as f:
yaml.safe_dump(self.get_status(),
f,
encoding='utf-8',
allow_unicode=True)
def get_info(self, section_name, key_name):
return self.info.get_value([section_name, key_name])
def init_logging(self, debug=False):
filename = os.path.join(self.core.artifacts_dir, 'tank.log')
open(filename, 'a').close()
current_file_mode = os.stat(filename).st_mode
os.chmod(
filename,
current_file_mode | stat.S_IRUSR | stat.S_IRGRP | stat.S_IROTH)
logger.handlers = []
logger.setLevel(logging.DEBUG if debug else logging.INFO)
file_handler = logging.FileHandler(filename)
file_handler.setLevel(logging.DEBUG)
file_handler.setFormatter(
logging.Formatter(
"%(asctime)s [%(levelname)s] %(name)s %(filename)s:%(lineno)d\t%(message)s"
))
file_handler.addFilter(TankapiLogFilter())
logger.addHandler(file_handler)
logger.info("Log file created")
for handler in self.log_handlers:
logger.addHandler(handler)
logger.info("Logging handler {} added".format(handler))
def get_lock(self):
while not self.interrupted.is_set():
try:
lock = Lock(self.test_id, self.folder).acquire(
self.core.lock_dir,
self.core.config.get_option(self.SECTION, 'ignore_lock'))
break
except LockError as e:
self.upd_msg(e.message)
if not self.wait_lock:
raise RuntimeError("Lock file present, cannot continue")
logger.warning("Couldn't get lock. Will retry in 5 seconds...")
time.sleep(5)
else:
raise KeyboardInterrupt
return lock
def upd_msg(self, msg):
if msg:
self.msg = self.msg + '\n' + msg
class SiteChecker(FeedbackInterface, SiteTempDataSrcRefInterface, ProgressLogInterface, ExternalTempInterface):
full_link_key = "full_link"
datasource_key = "data_source"
controller_ley = "controller"
max_level_key = "max_level"
max_page_key = "max_page"
output_queue_key = "output_queue"
_use_lxml_parser = False
def __init__(self, full_link: str="", data_source: SiteTempDataSrcInterface=None,
controller: SiteCheckerController=None,
max_level=10, max_page=1000, delegate=None, output_buff_size=2000,
output_queue=None, output_all_external=False, result_delegate=None,
memory_control_terminate_event=None, check_robot_text=True,
**kwargs):
"""
:param full_link: The full link of a domain, e.g: https://www.google.co.uk
:param domain: domain to crawl
:param max_level: stop crawling if it reaches this level
:param max_page: maximum pages to check within a site, also stop crawling
:param delegate: if this is not None, then it will send the latest result of external domain of ResponseCode==404 or 999
:param result_delegate: send site_info upon finish
:param memory_control_terminate_event: if this is not None and being set, it will be able to terminate an external memory controlled process.
:return:
"""
FeedbackInterface.__init__(self, **kwargs)
#super(SiteChecker, self).__init__(**kwargs)
if full_link is None or len(full_link) == 0:
raise ValueError()
original_path = ""
try:
paras = urlsplit(full_link)
self.scheme, self.domain, original_path = paras[0], paras[1], paras[2]
except:
pass
domain_data = LinkChecker.get_root_domain(full_link, False)
self.root_domain = domain_data[1]
self.sub_domain = domain_data[4]
self.domain_suffix = domain_data[5]
self.sub_domain_no_local = self.sub_domain.strip(self.domain_suffix)
if self.scheme == "":
self.scheme = "http"
if self.domain == "":
self.domain = self.root_domain
self.orginal_link = full_link
self.domain_link = LinkChecker.get_valid_link(self.root_domain, full_link, self.scheme)
self.max_level = max_level
self.max_page = max_page
self.page_count = 0 # keep track page done
self._page_count_shadow = 0 # track previous count
self._all_page_count_shadow = 0 #track previous count in datasource
self.internal_page_count = 0
self.internal_page_last_count = 0
self.page_allocated = 0
self.current_level = 0 # if this = 0, it is root domain/home_page
self._stop_event = Event()
valid_file_name = SiteTempDataSrcInterface.get_valid_file_name(self.domain_link)
self._external_db_buffer = ExternalTempDataDiskBuffer(valid_file_name+".ext.db", self,
stop_event=self._stop_event,
buf_size=int(output_buff_size/2),
dir_path=get_db_buffer_default_dir(),
convert_output=False)
self._external_db_buffer.append_to_buffer([(self.root_domain, ResponseCode.DNSError),], convert_tuple=False)
self._memory_control_terminate_event = memory_control_terminate_event
self.task_control_lock = threading.RLock()
if data_source is None:
#self.data_source = SiteTempDataDisk(self.root_domain, ref_obj=self)
self.data_source = SiteTempDataDiskWithBuff(ref=self.domain_link, output_buff_size=output_buff_size, ref_obj=self)
else:
self.data_source = data_source # a list of OnSiteLink
self.delegate = delegate
if LinkChecker.might_be_link_html_page(original_path):
self.data_source.append(OnSiteLink(self.domain_link, response_code=ResponseCode.LinkOK, link_level=1)) # add the root domain as a starting point
self.data_source.append(OnSiteLink(self.scheme + "://www."+self.sub_domain, ResponseCode.LinkOK, link_level=1))
self.data_source.append(OnSiteLink(self.scheme + "://" + self.domain, ResponseCode.LinkOK, link_level=1))
self.cache_list = [] # internal page cache
self.page_need_look_up_temp = 0
self.cache_list.append(self.domain_link)
if "www." not in self.sub_domain:
self.cache_list.append(self.scheme + "://www."+self.sub_domain)
self.cache_list.append(self.scheme + "://" + self.domain)
self.page_need_look_up = self.data_source.count_all()
self.cache_size = 500 # create a small cache list to avoid going to check link in file system with lots of read and write
self._double_check_cache_lock = threading.RLock()
self._double_check_cache = deque(maxlen=self.cache_size)
self.external_cache_list = []
self.external_cache_size = 500 # cache that hold external sites
self.external_links_checked = 0
self.add_internal_page_OK_only = True
self.output_queue = output_queue
self.output_all_external = output_all_external
self.controller = controller
self.result_delegate = result_delegate
self.page_count_lock = threading.RLock()
self.internal_page_count_lock = threading.RLock()
self.level_lock = threading.RLock()
self.page_look_up_lock = threading.RLock()
self.external_link_check_lock = threading.RLock()
self._finihsed = False
self.task_control_max = 1
self.agent = "VegeBot (we follow your robots.txt settings before crawling, you can slow down the bot by change the Crawl-Delay parameter in the settings." \
"if you have an enquiry, please email to: [email protected])"
self.agent_from = "*****@*****.**"
if check_robot_text:
self.robot_agent = LinkChecker.get_robot_agent(self.sub_domain, protocol=self.scheme)
else:
self.robot_agent = None
self.site_crawl_delay = 0.60
if isinstance(self.robot_agent, Rules):
delay_temp = self.robot_agent.delay(self.agent)
if delay_temp is not None and delay_temp != self.site_crawl_delay:
self.site_crawl_delay = delay_temp
self.task_control_counter = 1
self._speed_penalty_count = 0
self._speed_penalty_threshold = 10
self._progress_logging_speed = 120
self._output_period = 120
self._output_batch_size = 100
self._death_wish_sent = False
SiteChecker._is_lxml_parser_exist()
self._output_thread = None
self._output_queue = None
self.progress_logger = ProgressLogger(self._progress_logging_speed, self, self._stop_event)
self._status = "Start"
self._populate_with_state() # restore laste known state
# self.data_source.additional_startup_procedures() # use the data set in self._populate_with_state() to start
# def _empty_external_links_db(self):
# if self.output_queue is not None:
def _put_result_in_output_queue_loop(self, item_list: list):
if not self._stop_event.is_set():
try:
self._output_queue.put(item_list, True, 2)
except Exception as ex:
if self._output_queue is None:
manager, self._output_queue = get_queue_client(QueueManager.MachineSettingCrawler,
QueueManager.Method_Whois_Input)
time.sleep(0.1)
ErrorLogger.log_error("SiteChecker._get_external_links_to_queue", self.sub_domain+" "+str(ex))
self._put_result_in_output_queue_loop(item_list)
def _get_external_links_to_queue(self):
ref_time = time.time()
manager, self._output_queue = get_queue_client(QueueManager.MachineSettingCrawler, QueueManager.Method_Whois_Input)
self.output_queue = self._output_queue # override output_queue
# if result_queue is None:
# ErrorLogger.log_error("SiteChecker._get_external_links_to_queue()", ValueError("result queue is none, cannot put item in queue."))
# else:
batch = list()
counter = 0
for item in self._external_db_buffer:
if self._stop_event.is_set() or self.external_links_checked >= self._external_db_buffer.count_all():
try:
manager.shutdown()
except:
pass
finally:
# print("exist _get_external_links_to_queue")
# if self._stop_event.is_set() and self.external_links_checked >= self._external_db_buffer.count_all():
break
elif isinstance(item, tuple):
# print("outputting item: ", str(item))
batch.append((item[0], item[1]))
counter += 1
if len(batch) > 0:
current_time = time.time()
if current_time - ref_time or len(batch) >= self._output_batch_size:
self._put_result_in_output_queue_loop(batch)
self.external_links_checked += len(batch)
ref_time = time.time()
batch.clear()
time.sleep(0.0001)
@staticmethod
def _is_lxml_parser_exist():
try:
import lxml
except ImportError:
SiteChecker._use_lxml_parser = False
else:
SiteChecker._use_lxml_parser = True
def use_lxml_parser(self):
return SiteChecker._use_lxml_parser
@staticmethod
def get_input_parameter_base(full_link: str, max_page: int, max_level: int, output_queue) -> dict:
return {SiteChecker.full_link_key: full_link, SiteChecker.max_page_key: max_page,
SiteChecker.max_level_key: max_level, SiteChecker.output_queue_key: output_queue}
def get_external_count_finished(self) -> int:
"""
ExternalTempInterface, get the number of job done in ExternalTempDataDiskBuffer
:return:
"""
return self.external_links_checked
def set_internal_count(self, count: int):
"""
ExternalTempInterface, set the number of job done in ExternalTempDataDiskBuffer
:param count:
:return:
"""
self.external_links_checked = count
def _set_task_control_max(self, concurrent_task: int):
if concurrent_task <= 0:
raise ValueError
self.task_control_max = concurrent_task
self.task_control_counter = concurrent_task
min_page_per_s = concurrent_task/20
self._speed_penalty_threshold = self._progress_logging_speed * min_page_per_s
if self.site_crawl_delay > 1/min_page_per_s:
ErrorLogger.log_error("SiteChecker._set_task_control_max()",
ValueError("site has crawl delay greater than mas delay."), self.domain_link)
self._status = "Stopped"
self.sudden_death()
def get_site_feedback(self) -> SeedSiteFeedback:
return SeedSiteFeedback(self.orginal_link, page_count=self.get_page_need_look_up())
def get_site_info(self) -> SiteInfo: # keep the original reference when sending back the site infomation
info = SiteInfo(self.orginal_link, self.data_source)
return info
def populate_with_state(self, state):
if state is not None and isinstance(state, SiteCheckerState):
self._status = "Restarted"
self.page_count = state.page_count
self.page_allocated = state.page_count
self.internal_page_count = state.internal_page_count
self.internal_page_last_count = state.internal_page_count
self.external_links_checked = state.external_page_count
self._external_db_buffer.set_progress(state.external_page_count)
self.page_need_look_up = state.page_need_look_up
self.current_level = state.current_level
self.progress_logger.set_reference(state.log_sample_index, state.log_started_time)
counter = 0
if self.data_source is not None:
try:
for item in self.data_source.get_next():
if counter >= self.cache_size:
break
if isinstance(item, OnSiteLink) and not LinkChecker.is_external_link(self.root_domain, item.link):
self.cache_list.append(item.link)
# print("--restore: ", item)
counter += 1
except Exception as ex:
msg = "error in SiteChecker.populate_with_state(), trying to populate cache, " + self.root_domain
ErrorLogger.log_error("SiteChecker", ex, msg)
self.data_source.ref = state.datasource_ref
self.data_source.output_c = state.datasource_output_c
self.data_source.set_progress(state.datasource_index if state.datasource_index < state.page_count else state.page_count)
self.data_source.set_continue_lock(True)
def get_file_name(self):
return self.data_source.ref
def get_limit(self):
return 100000
def get_column_names(self):
return ["Page Index", "External", "All", "Status"]
def get_progress(self):
data_source_count = self.data_source.count_all()
if self.page_count - self._page_count_shadow <= self._speed_penalty_threshold: # determine if site is slow
self._speed_penalty_count += 1
if self._speed_penalty_count > 2:
self._status = "Stopped"
self.sudden_death()
else:
self._speed_penalty_count = 0
if self.page_count == self._page_count_shadow and data_source_count == self._all_page_count_shadow: # determine if site is stucked
self._status = "Stopped"
self.sudden_death()
self._page_count_shadow = self.page_count
self._all_page_count_shadow = data_source_count
return [self.page_count, self.external_links_checked, data_source_count, self._status]
def is_programme_finshed(self):
return self._finihsed
def get_callback_data(self):
with self.page_count_lock:
gap = self.internal_page_count - self.internal_page_last_count
self.internal_page_last_count = self.internal_page_count
seed_feedback = None
if self._finihsed:
seed_feedback = self.get_site_feedback()
return SiteFeedback(gap, self._finihsed, seed_feedback=seed_feedback, datasource_ref=self.data_source.ref)
def get_state(self):
return SiteCheckerState(page_count=self.page_count, page_need_look_up=self.page_need_look_up,
current_level=self.current_level, internal_page_count=self.internal_page_count,
external_page_count= self.external_links_checked,
datasource_index=self.data_source.temp_counter,
datasource_output_c=self.data_source.output_c,
datasource_ref=self.data_source.ref, log_started_time=self.progress_logger.begin_time,
log_sample_index=self.progress_logger.limit_counter,)
def additional_reset(self):
pass
def addtional_clear(self):
pass
def stop(self):
# natural stop
self._status = "Stopped"
self.progress_logger.report_progress()
self._stop_event.set()
if self.progress_logger.is_alive():
self.progress_logger.join()
def clear(self):
self.cache_list.clear()
self.addtional_clear()
def acquire_task(self, level: int, link: str):
tasked_acquired = True
if link.endswith('/'):
temp = link
else:
temp = link + '/'
with self.task_control_lock:
if len(self._double_check_cache) > 0:
if temp in self._double_check_cache:
print("duplicate link found:", link)
tasked_acquired = False
else:
if len(self._double_check_cache) >= self.cache_size:
self._double_check_cache.popleft()
self._double_check_cache.append(temp)
self.task_control_counter -= 1
self.page_allocated += 1
if tasked_acquired:
if level > self.current_level:
self.current_level = level
# time.sleep(self.site_crawl_delay)
return tasked_acquired
def release_task(self, new_page: int):
with self.task_control_lock:
if self.page_need_look_up == 1 and new_page == 0:
PrintLogger.print("set to stop data source")
self.data_source.set_continue_lock(False)
else:
self.page_count += 1
self.page_need_look_up += new_page
#self.external_links_checked += external_page_count
self.task_control_counter += 1
# was determine if it is internal or external page
self.internal_page_count += 1
if self.internal_page_count > self.max_page or self.current_level > self.max_level:
if self.data_source.can_continue():
PrintLogger.print("set stop: " + str(self.internal_page_count)+" level: "+str(self.current_level))
self.data_source.set_continue_lock(False)
def get_page_count(self):
with self.page_count_lock:
page_count = self.page_count
return page_count
def set_page_count(self, page_count: int):
with self.page_count_lock:
self.page_count = page_count
def set_internal_page_count(self, count: int):
with self.internal_page_count_lock:
self.internal_page_count += count
def get_internal_page_count(self):
with self.internal_page_count_lock:
count = self.internal_page_count
return count
def get_current_level(self):
with self.level_lock:
current_level = self.current_level
return current_level
def set_current_level(self, level):
with self.level_lock:
self.current_level = level
def get_page_need_look_up(self):
with self.page_look_up_lock:
page_look_up = self.page_need_look_up
#self.page_look_up_lock.release()
return page_look_up
def set_page_need_look_up(self, page_count):
with self.page_look_up_lock:
#time.sleep(0.1)
self.page_need_look_up = page_count
# self.page_look_up_lock.release()
def set_page_need_look_up_plus_more(self, count: int):
with self.page_look_up_lock:
self.page_need_look_up += count
def get_internal_page_progress_index(self)->int:
return self.get_page_count()
def set_internal_page_progress_index(self, index: int):
self.page_count = index
self.page_allocated = index
def is_idle(self):
idle = False
with self.task_control_lock:
page_need_look_up = self.get_page_need_look_up()
new_task_added = page_need_look_up - self.page_need_look_up_temp
has_new_task = True if new_task_added > 0 else False
#page_count = self.get_page_count()
if has_new_task:
self.page_need_look_up_temp = page_need_look_up
else:
if self.task_control_counter >= self.task_control_max:
idle = True
# print("is idle")
# else:
# print("is working")
return idle
def add_link_to_cache(self, link):
if len(self.cache_list) > self.cache_size:
return
else:
if link.endswith('/'):
self.cache_list.append(link)
else:
self.cache_list.append(link+'/')
def is_link_in_cache(self, link):
if link.endswith('/'):
temp = link
else:
temp = link + '/'
return True if temp in self.cache_list else False
def reset_as(self, domain: str, link: str=""): # reset the target domain
PrintLogger.print("crawl reset as: "+domain)
self.domain = domain
self.domain_link = self.scheme + "://" + self.domain
self.page_count = 0
self.current_level = 0
self.set_page_need_look_up(1)
# self.set_page_looked_up(0)
self.clear()
if len(link) == 0:
self.cache_list.append(self.domain_link)
self.data_source.re_target(self.domain_link, OnSiteLink(self.domain_link, response_code=ResponseCode.LinkOK, link_level=1))
#self.data_source.append(OnSiteLink(self.domain_link, response_code=ResponseCode.LinkOK, link_level=1))
else:
self.cache_list.append(link)
self.data_source.re_target(link, OnSiteLink(link, response_code=ResponseCode.LinkOK, link_level=1))
#self.data_source.append(OnSiteLink(link, response_code=ResponseCode.LinkOK, link_level=1))
self.additional_reset()
self.data_source.additional_startup_procedures()
def crawling(self): # call this method to start operation
self._start_sending_feedback()
self._output_thread = threading.Thread(target=self._get_external_links_to_queue)
if self.data_source.can_continue():
self.data_source.additional_startup_procedures() # use the data set in self._populate_with_state() to start
self._external_db_buffer.start_input_output_cycle()
self._output_thread.start()
self.progress_logger.start()
self.progress_logger.report_progress() # log first row
self._status = "Work"
self.begin_crawl()
# prefix = "www."
# page_count_limit = 2
# if self.page_count <= page_count_limit and prefix not in self.domain_link:
# new_domain = prefix + self.sub_domain
# self.reset_as(new_domain)
# self._status = "Work"
# self.begin_crawl()
# print("going to stop all.")
self.stop()
self.clear()
self.data_source.additional_finish_procedures()
# print("going to finish output buffer.")
self._external_db_buffer.terminate()
# print("going to stop output_thread.")
if self._output_thread.is_alive():
self._output_thread.join()
PrintLogger.print("finished naturally: "+self.domain_link)
# print("finished naturally.")
self._finihsed = True
#calling this at the end of operation
PrintLogger.print("send last response")
# print("send last response")
# print("send last response.")
self._end_sending_feedback()
if self._memory_control_terminate_event is not None:
self._memory_control_terminate_event.set()
def sudden_death(self):
if not self._finihsed:
self._finihsed = True
PrintLogger.print("start sudden death: "+self.orginal_link)
#self.stop()
self.stop()
self.clear()
self.data_source.set_continue_lock(False)
self.data_source.additional_finish_procedures()
self._external_db_buffer.terminate()
if isinstance(self._output_thread, threading.Thread):
if self._output_thread.is_alive():
self._output_thread.join()
#calling this at the end of operation
PrintLogger.print("send last response")
self._end_sending_feedback()
if self._memory_control_terminate_event is not None:
ErrorLogger.log_error("SiteChecker", TimeoutError("slow processing speed, terminated."), self.orginal_link)
self._memory_control_terminate_event.set()
def begin_crawl(self, level=0): # subclass this to make different behaviour
pass
class GpsReader(Process):
def __init__(self, queue, log_file_path, test=False):
super(Process, self).__init__()
self.queue = queue
self.test = test
self.exit = Event()
self.speed = -1
self.log_file_path = log_file_path
if not test:
self._setup_gps()
self.last_timestamp = time.monotonic()
def _setup_gps(self):
uart = serial.Serial("/dev/ttyUSB0", baudrate=9600, timeout=3000)
# Create a GPS module instance.
self.gps = adafruit_gps.GPS(uart, debug=False)
self.gps.send_command(b'PMTK314,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0')
self.gps.send_command(b'PMTK220,500')
def run(self):
print("Running GPS Reader")
with open(self.log_file_path, "w") as log_file:
self.csv_writer = csv.writer(log_file)
self.csv_writer.writerow(["Speed"]) #write header
while not self.exit.is_set():
if not self.test:
self._update_speed()
else:
self._update_fake_speed()
if self.speed >= 0: #speed has been set
self._log_speed()
def _log_speed(self):
self.csv_writer.writerow([self.speed])
try:
self.queue.put(self.speed, True, 0.1)
except Q.Full:
pass #Don't let a full queue cause an issue
def _update_speed(self):
# Make sure to call gps.update() every loop iteration and at least twice
# as fast as data comes from the GPS unit (usually every second).
# This returns a bool that's true if it parsed new data (you can ignore it
# though if you don't care and instead look at the has_fix property).
self.gps.update()
current = time.monotonic()
if current - self.last_timestamp >= 1.0:
self.last_timestamp = current
if not self.gps.has_fix:
# Try again if we don't have a fix yet
print('Waiting for fix...')
self.speed = -1
return
# We have a fix! (gps.has_fix is true)
if self.gps.track_angle_deg is not None:
#convert knots to mph
self.speed = self.gps.speed_knots * 1.15078
def _update_fake_speed(self):
current = time.monotonic()
if current - self.last_timestamp >= 0.1:
self.last_timestamp = current
self.speed = random.randint(0, 50)
def shutdown(self):
print("Shutting Down GPS Reader")
self.exit.set()
class IPRPCChannel(Process):
"""
An IPRPC (InterPlanetary Remote Procedure Call) channel is the base unit
of communication in pillar. It allows two nodes to pass IPRPCMessages
between each other using IPFS pubsub queues. It generates random channel
IDs based on the peer fingerprints, time, and an optional pre-shared-key.
The queue IDs will rotate every hour, and the channel listens on both
the previous hour and next hours ID so no messages should be lost during
the transition.
This class is designed to be run as a subprocess, as such it is started
with the .start() method, which calls the .run() method in a subprocess.
Messages are received from and passed to this thread using the pipe
endpoints returned by the .get_pipe_endpoints() method.
"""
def __init__(self,
id: str,
peer_fingerprint: str = None,
encryption_helper: EncryptionHelper = None,
ipfs_instance: IPFSClient = None,
keepalive_send_interval: int = 30,
keepalive_timeout_interval: int = 60,
pre_shared_key: str = ''):
self.id = id
self.peer_id = peer_fingerprint
self.pre_shared_key = pre_shared_key
self.queues = []
self.encryption_helper = encryption_helper
self.ipfs = ipfs_instance or IPFSClient()
self.our_ipfs_peer_id = None
self.tx_input, self.tx_output = Pipe()
self.rx_input, self.rx_output = Pipe()
self.status = PeeringStatus.IDLE
self.logger = logging.getLogger(self.__repr__())
self.keepalive_send_interval = keepalive_send_interval
self.keepalive_timeout_interval = keepalive_timeout_interval
self.timeout = None
self.keepalive_send_timeout = None
self._establish_and_rotate_queues()
self.shutdown_callback = Event()
super().__init__()
self.logger.info(
f"Spawned channel between {self.id} and {self.peer_id}")
self.logger.info(
f"Initial channel window: {self.queues}"
)
def get_pipe_endpoints(self):
"""
Returns the endpoints that allow messages to be sent and received
from this thread.
:return:
tx_pipe, rx_pipe
"""
return self.tx_input, self.rx_output
def run(self) -> None:
"""
This method should not be called directly, .start() will start a
subprocess thread where this method will be called.
:return:
"""
while True:
self.timeout = time.time() + self.keepalive_timeout_interval
self.keepalive_send_timeout = time.time() + \
self.keepalive_send_interval
rx_workers = []
asyncio.ensure_future(handler_loop(
self._handle_establish_connection, sleep=5)
)
rx_workers.append(asyncio.ensure_future(
handler_loop(
self._handle_messages_current_window,
sleep=.01
)
))
rx_workers.append(asyncio.ensure_future(
handler_loop(
self._handle_messages_previous_window,
sleep=.01
)
))
rx_workers.append(asyncio.ensure_future(
handler_loop(
self._handle_messages_next_window,
sleep=.01
)
))
asyncio.ensure_future(
handler_loop(
self._handle_tx_queue_messages,
sleep=.01
)
)
asyncio.ensure_future(
handler_loop(
self._handle_keepalive,
sleep=5,
)
)
asyncio.ensure_future(
handler_loop(
self._handle_timeout,
sleep=5,
)
)
asyncio.ensure_future(handler_loop(
self._async_rotate_queues_wrapper,
sleep=5
))
asyncio.ensure_future(handler_loop(
self._stop_on_shutdown_event,
sleep=1
))
loop = asyncio.get_event_loop()
loop.run_forever()
print(f"Cancelling rx workers: {rx_workers}")
for rx_worker in rx_workers:
rx_worker.cancel()
if self.shutdown_callback.is_set():
break
def _change_peering_status(self, new_status: PeeringStatus):
if self.status != new_status:
self.logger.info(f"Peering status change from {self.status} to "
f"{new_status}")
self.status = new_status
async def _handle_establish_connection(self) -> None:
"""
Sends a PeeringHello message to wake up the other side of the
connection.
"""
if not self.status == PeeringStatus.ESTABLISHED:
self._change_peering_status(PeeringStatus.ESTABLISHING)
await self._send_message(PeeringHello(initiator_id=self.id))
async def _handle_tx_queue_messages(self) -> None:
"""
If state == ESTABLISHED, this will pull messages from the tx queue pipe
and send them to the other peer.
"""
if self.status == PeeringStatus.ESTABLISHED:
if self.tx_output.poll():
message = self.tx_output.recv()
await self._send_message(message)
async def _handle_timeout(self) -> None:
if time.time() > self.timeout:
if not self.status == PeeringStatus.ESTABLISHING:
self._change_peering_status(PeeringStatus.IDLE)
async def _stop_on_shutdown_event(self) -> None:
loop = asyncio.get_event_loop()
if self.shutdown_callback.is_set():
loop.stop()
async def _async_rotate_queues_wrapper(self) -> None:
"""
If the _establish_and_rotate_queues() returns true, this resets the
event loop and stops all worker asyncio events so they don't wait
for messages on retired channels.
"""
loop = asyncio.get_event_loop()
result = self._establish_and_rotate_queues()
if result:
self.logger.info("Event loop reset")
loop.stop()
def _establish_and_rotate_queues(self) -> bool:
"""
Handles rotating the sliding queue_id window.
:return:
True on window change
False on no change
"""
previous_queue_id = self._get_queue_id(time_delta=timedelta(hours=-1))
current_queue_id = self._get_queue_id()
next_queue_id = self._get_queue_id(time_delta=timedelta(hours=1))
current_list = [previous_queue_id, current_queue_id, next_queue_id]
if not self.queues:
self.logger.info(f"Current Queues: {self.queues}")
self.queues = current_list
return True
elif self.queues != current_list:
self.logger.info(f"Window slide occured, Old Queues: {self.queues}"
f"New Queues: {current_list}")
self.queues = current_list
return True
else:
return False
async def _handle_messages_previous_window(self) -> None:
await self._handle_incoming_messages(self.queues[0])
async def _handle_messages_next_window(self) -> None:
await self._handle_incoming_messages(self.queues[2])
async def _handle_messages_current_window(self) -> None:
await self._handle_incoming_messages(self.queues[1])
def _get_queue_id(self, time_delta=None):
if time_delta:
time = datetime.utcnow() + time_delta
else:
time = datetime.utcnow()
return generate_queue_id(
self.id,
self.peer_id,
preshared_key=self.pre_shared_key,
datetime=time
)
async def _handle_incoming_messages(self, queue_id: str) -> None:
"""
Handles messages received from the other peer.
Some messages are needed by the class to establish a connection or keep
it alive. All others are output over the rx pipe so they can be passed
to other processes.
:param queue_id:
ID of the pubsub queue to handle messages on.
"""
async for rx_message in self._get_message(queue_id):
if type(rx_message) is PeeringHello:
await self._send_message(
PeeringHelloResponse(responder_id=self.id)
)
elif type(rx_message) is PeeringHelloResponse:
self.peer_id = rx_message.responder_id
self._change_peering_status(PeeringStatus.ESTABLISHED)
elif type(rx_message) is PeeringKeepalive:
self.timeout = time.time() + \
self.keepalive_timeout_interval
else:
self.rx_input.send(rx_message)
async def _handle_keepalive(self):
if self.status == PeeringStatus.ESTABLISHED:
if time.time() > self.keepalive_send_timeout:
await self._send_message(PeeringKeepalive())
self.keepalive_send_timeout = time.time() + \
self.keepalive_send_interval
async def _set_our_ipfs_peer_id(self) -> None:
"""This sets self.our_ipfs_peer_id so we can ignore messages we sent"""
id_info = await self.ipfs.get_id()
self.our_ipfs_peer_id = id_info.get('ID')
self.logger.info(f"Set our ipfs peer id to {self.our_ipfs_peer_id}")
async def _send_message(self, call: IPRPCMessage):
message = call.serialize_to_json()
if self.encryption_helper:
message = self.encryption_helper.\
sign_and_encrypt_string_to_peer_fingerprint(
message,
self.peer_id
)
await self._send_ipfs(message)
self.logger.info(f"Sent message: {call}")
async def _send_ipfs(self, message: str) -> None:
await self.ipfs.send_pubsub_message(self.queues[1], message)
async def _get_from_ipfs(self, queue_id: str):
if not self.our_ipfs_peer_id:
await self._set_our_ipfs_peer_id()
async for message in self.ipfs.get_pubsub_message(queue_id):
if not message['from'].decode() == self.our_ipfs_peer_id:
raw_message = unquote(message['data'].decode('utf-8'))
yield raw_message
async def _get_message(self, queue_id: str):
async for message in self._get_from_ipfs(queue_id):
if self.encryption_helper:
try:
message = self._decrypt_message(message)
except Exception as e:
self.logger.warning(f"Failed to decrypt message on"
f" encrypted channel: {e}")
try:
message = IPRPCRegistry.deserialize_from_json(message)
self.logger.info(f"Got message from peer: {message}")
yield message
except Exception as e:
self.logger.warning(f"Failed to decodde message: {e}")
def _decrypt_message(self, message: str):
return self.encryption_helper.\
decrypt_and_verify_encrypted_message(message)
def __repr__(self) -> str:
return f"<{self.__class__.__name__}:" \
f"peer_id={self.peer_id}>"
class Cluster(object):
def __init__(self, list_key=Conf.Q_LIST):
self.sentinel = None
self.stop_event = None
self.start_event = None
self.pid = current_process().pid
self.host = socket.gethostname()
self.list_key = list_key
self.timeout = Conf.TIMEOUT
signal.signal(signal.SIGTERM, self.sig_handler)
signal.signal(signal.SIGINT, self.sig_handler)
def start(self):
# This is just for PyCharm to not crash. Ignore it.
if not hasattr(sys.stdin, 'close'):
def dummy_close():
pass
sys.stdin.close = dummy_close
# Start Sentinel
self.stop_event = Event()
self.start_event = Event()
self.sentinel = Process(target=Sentinel,
args=(self.stop_event, self.start_event,
self.list_key, self.timeout))
self.sentinel.start()
logger.info(_('Q Cluster-{} starting.').format(self.pid))
while not self.start_event.is_set():
sleep(0.1)
return self.pid
def stop(self):
if not self.sentinel.is_alive():
return False
logger.info(_('Q Cluster-{} stopping.').format(self.pid))
self.stop_event.set()
self.sentinel.join()
logger.info(_('Q Cluster-{} has stopped.').format(self.pid))
self.start_event = None
self.stop_event = None
return True
def sig_handler(self, signum, frame):
logger.debug(
_('{} got signal {}').format(
current_process().name,
Conf.SIGNAL_NAMES.get(signum, 'UNKNOWN')))
self.stop()
@property
def stat(self):
if self.sentinel:
return Stat.get(self.pid)
return Status(self.pid)
@property
def is_starting(self):
return self.stop_event and self.start_event and not self.start_event.is_set(
)
@property
def is_running(self):
return self.stop_event and self.start_event and self.start_event.is_set(
)
@property
def is_stopping(self):
return self.stop_event and self.start_event and self.start_event.is_set(
) and self.stop_event.is_set()
@property
def has_stopped(self):
return self.start_event is None and self.stop_event is None and self.sentinel
class SlaPrinterApp(Flask, Observable, Observer, Process):
def __init__(self, import_name, db_controller = None, bus = None):
#super(SlaPrinterApp, self).__init__(import_name, template_folder=TEMPLATE_DIR, static_url_path=STATIC_DIR)
Flask.__init__(self,import_name, static_folder=STATIC_DIR, template_folder=TEMPLATE_DIR)
Observable.__init__(self, bus)
Observer.__init__(self, bus)
Process.__init__(self)
self.exit = Event()
self.endpoint_prefix = None
for name in dir(self):
if hasattr(getattr(self, name), ("_routing_data")):
fn = getattr(self, name)
rds = fn._routing_data
for rd in rds:
self.route(*rd.args, **rd.kwargs)(fn)
self.register_error_handler(404, self.page_not_found)
self.db_controller = db_controller
print("Flask Server initializing")
#func = lambda: self.run(host='0.0.0.0',debug=True, port=4242, use_evalex=False, use_reloader=False)
#self.runner_process = Process(target=func)
@route("/")
@route("/index")
def index(self):
if self.db_controller is not None:
tasks = self.db_controller.printing_tasks()
print("tasks: " + str(tasks))
active_job = self.db_controller.active_job()
return render_template("index.html", printing_tasks = tasks, active_job= active_job)
else:
return render_template("index.html")
@route("/enqueue")
def enqueue(self):
return render_template("enqueue.html")
@route("/info")
def info(self):
return render_template("info.html")
@route("/quit", methods = ['POST', 'GET'])
def quit(self):
msg = QuitMessage(SlaPrinterAppProto(),"sla printer shutting down")
self.put_message(msg)
return "quitting"
@route("/post/raw/", methods = ['POST'])
def post_data(self):
'''
function used to send simple raw string data to 3D printer
:return: according success / fail message
'''
# get current data pool to store new objects
#data_pool = cntrl.DataPool()
# load data
data = json.loads(request.data)
# if "data" in data:
# d = RawData(data["data"])
# data_pool.add(d)
# return str(d)
# else:
# return "no data received"
@route("/post/task/", methods = ['POST'])
def post_task(self):
'''
Function used to send printing task to 3D printer
:return: according success / fail message
'''
# load data
data = json.loads(request.data)
# create empty printing task
task = PrintingTaskData()
# if received data parseable to task object store new printing task
if task.parse(data):
if self.db_controller is not None:
jid = self.db_controller.save_printing_task(task)
return json.dumps({"id": jid})
else:
return "no valid printing task received"
else:
return "invalid data"
@route("/download/<jid>/", methods = ['POST', 'GET'])
def download_zip(self, jid):
# data_pool = cntrl.DataPool()
#
#
task = self.db_controller.get_by_id(jid)
if task is not None:
zip = task.stl_file
zip = base64.decodestring(zip)
filename = task.file_name
else:
zip = ''
filename = ''
return Response(zip,
mimetype='application/zip',
headers={'Content-Disposition':'attachment;filename=' + str(filename) + '.zip'})
#headers={'Content-Disposition':'attachment;filename=' + str(file_name) + '.zip'})
@route("/stepper/down/<steps>/", methods = ['POST', 'GET'])
def steps_down(self, steps):
print(" received " + str(steps) + " down message")
def page_not_found(self, e):
return render_template('404.html'), 404
def notify(self, msg):
print("[" + str(now()) + "] Server :: " + str(msg))
def start(self):
print("[" + str(now()) + "] Server :: starting ")
#self.runner_process.start()
#self.running = True
Process.start(self)
def stop(self):
self.exit.set()
self.terminate()
#self.join()
def run(self):
# run server
Flask.run(self, host='0.0.0.0',debug=True, port=4242, use_evalex=False, use_reloader=False)
while not self.exit.is_set():
print("[" + str(now()) + "] server heartbeat ")
time.sleep(0.5)
print("[" + str(now()) + "] server shutting down ")
return
class RestoreVMsWindow(Ui_Restore, QWizard):
__pyqtSignals__ = ("restore_progress(int)","backup_progress(int)")
def __init__(self, app, qvm_collection, blk_manager, parent=None):
super(RestoreVMsWindow, self).__init__(parent)
self.app = app
self.qvm_collection = qvm_collection
self.blk_manager = blk_manager
self.restore_options = None
self.vms_to_restore = None
self.func_output = []
self.feedback_queue = Queue()
self.canceled = False
self.tmpdir_to_remove = None
self.error_detected = Event()
self.excluded = {}
self.vm = self.qvm_collection[0]
assert self.vm != None
self.setupUi(self)
self.select_vms_widget = MultiSelectWidget(self)
self.select_vms_layout.insertWidget(1, self.select_vms_widget)
self.connect(self, SIGNAL("currentIdChanged(int)"), self.current_page_changed)
self.connect(self, SIGNAL("restore_progress(QString)"), self.commit_text_edit.append)
self.connect(self, SIGNAL("backup_progress(int)"), self.progress_bar.setValue)
self.dir_line_edit.connect(self.dir_line_edit, SIGNAL("textChanged(QString)"), self.backup_location_changed)
self.connect(self.verify_only, SIGNAL("stateChanged(int)"),
self.on_verify_only_toogled)
self.select_dir_page.isComplete = self.has_selected_dir
self.select_vms_page.isComplete = self.has_selected_vms
self.confirm_page.isComplete = self.all_vms_good
#FIXME
#this causes to run isComplete() twice, I don't know why
self.select_vms_page.connect(self.select_vms_widget, SIGNAL("selected_changed()"), SIGNAL("completeChanged()"))
fill_appvms_list(self)
self.__init_restore_options__()
@pyqtSlot(name='on_select_path_button_clicked')
def select_path_button_clicked(self):
select_path_button_clicked(self, True)
def on_ignore_missing_toggled(self, checked):
self.restore_options['use-default-template'] = checked
self.restore_options['use-default-netvm'] = checked
def on_ignore_uname_mismatch_toggled(self, checked):
self.restore_options['ignore-username-mismatch'] = checked
def on_verify_only_toogled(self, checked):
self.restore_options['verify-only'] = bool(checked)
def cleanupPage(self, p_int):
if self.page(p_int) is self.select_vms_page:
self.vms_to_restore = None
else:
super(RestoreVMsWindow, self).cleanupPage(p_int)
def __fill_vms_list__(self):
if self.vms_to_restore is not None:
return
self.select_vms_widget.selected_list.clear()
self.select_vms_widget.available_list.clear()
self.target_appvm = None
if self.appvm_combobox.currentIndex() != 0: #An existing appvm chosen
self.target_appvm = self.qvm_collection.get_vm_by_name(
str(self.appvm_combobox.currentText()))
try:
self.vms_to_restore = backup.backup_restore_prepare(
unicode(self.dir_line_edit.text()),
unicode(self.passphrase_line_edit.text()),
options=self.restore_options,
host_collection=self.qvm_collection,
encrypted=self.encryption_checkbox.isChecked(),
appvm=self.target_appvm)
for vmname in self.vms_to_restore:
if vmname.startswith('$'):
# Internal info
continue
self.select_vms_widget.available_list.addItem(vmname)
except QubesException as ex:
QMessageBox.warning (None, "Restore error!", str(ex))
def __init_restore_options__(self):
if not self.restore_options:
self.restore_options = {}
backup.backup_restore_set_defaults(self.restore_options)
if 'use-default-template' in self.restore_options and 'use-default-netvm' in self.restore_options:
val = self.restore_options['use-default-template'] and self.restore_options['use-default-netvm']
self.ignore_missing.setChecked(val)
else:
self.ignore_missing.setChecked(False)
if 'ignore-username-mismatch' in self.restore_options:
self.ignore_uname_mismatch.setChecked(self.restore_options['ignore-username-mismatch'])
def gather_output(self, s):
self.func_output.append(s)
def restore_error_output(self, s):
self.error_detected.set()
self.feedback_queue.put((SIGNAL("restore_progress(QString)"),
u'<font color="red">{0}</font>'.format(s)))
def restore_output(self, s):
self.feedback_queue.put((SIGNAL("restore_progress(QString)"),
u'<font color="black">{0}</font>'.format(s)))
def update_progress_bar(self, value):
self.feedback_queue.put((SIGNAL("backup_progress(int)"), value))
def __do_restore__(self, thread_monitor):
err_msg = []
self.qvm_collection.lock_db_for_writing()
try:
backup.backup_restore_do(self.vms_to_restore,
self.qvm_collection,
print_callback=self.restore_output,
error_callback=self.restore_error_output,
progress_callback=self.update_progress_bar)
except backup.BackupCanceledError as ex:
self.canceled = True
self.tmpdir_to_remove = ex.tmpdir
err_msg.append(unicode(ex))
except Exception as ex:
print "Exception:", ex
err_msg.append(unicode(ex))
err_msg.append("Partially restored files left in "
"/var/tmp/restore_*, investigate them and/or clean them up")
self.qvm_collection.unlock_db()
if self.canceled:
self.emit(SIGNAL("restore_progress(QString)"),
'<b><font color="red">{0}</font></b>'
.format("Restore aborted!"))
elif len(err_msg) > 0 or self.error_detected.is_set():
if len(err_msg) > 0:
thread_monitor.set_error_msg('\n'.join(err_msg))
self.emit(SIGNAL("restore_progress(QString)"),
'<b><font color="red">{0}</font></b>'
.format("Finished with errors!"))
else:
self.emit(SIGNAL("restore_progress(QString)"),
'<font color="green">{0}</font>'
.format("Finished successfully!"))
thread_monitor.set_finished()
def current_page_changed(self, id):
old_sigchld_handler = signal.signal(signal.SIGCHLD, signal.SIG_DFL)
if self.currentPage() is self.select_vms_page:
self.__fill_vms_list__()
elif self.currentPage() is self.confirm_page:
for v in self.excluded:
self.vms_to_restore[v] = self.excluded[v]
self.excluded = {}
for i in range(self.select_vms_widget.available_list.count()):
vmname = self.select_vms_widget.available_list.item(i).text()
self.excluded[str(vmname)] = self.vms_to_restore[str(vmname)]
del self.vms_to_restore[str(vmname)]
del self.func_output[:]
self.vms_to_restore = backup.restore_info_verify(self.vms_to_restore,
self.qvm_collection)
backup.backup_restore_print_summary(
self.vms_to_restore, print_callback = self.gather_output)
self.confirm_text_edit.setReadOnly(True)
self.confirm_text_edit.setFontFamily("Monospace")
self.confirm_text_edit.setText("\n".join(self.func_output))
self.confirm_page.emit(SIGNAL("completeChanged()"))
elif self.currentPage() is self.commit_page:
self.button(self.FinishButton).setDisabled(True)
self.showFileDialog.setEnabled(True)
self.showFileDialog.setChecked(self.showFileDialog.isEnabled()
and str(self.dir_line_edit.text())
.count("media/") > 0)
self.thread_monitor = ThreadMonitor()
thread = threading.Thread (target= self.__do_restore__ , args=(self.thread_monitor,))
thread.daemon = True
thread.start()
while not self.thread_monitor.is_finished():
self.app.processEvents()
time.sleep (0.1)
try:
for (signal_to_emit,data) in iter(self.feedback_queue.get_nowait,None):
self.emit(signal_to_emit,data)
except Empty:
pass
if not self.thread_monitor.success:
if self.canceled:
if self.tmpdir_to_remove and \
QMessageBox.warning(None, "Restore aborted",
"Do you want to remove temporary "
"files from %s?" % self
.tmpdir_to_remove,
QMessageBox.Yes, QMessageBox.No) == \
QMessageBox.Yes:
shutil.rmtree(self.tmpdir_to_remove)
else:
QMessageBox.warning (None, "Backup error!", "ERROR: {1}"
.format(self.vm.name, self.thread_monitor.error_msg))
if self.showFileDialog.isChecked():
self.emit(SIGNAL("restore_progress(QString)"),
'<b><font color="black">{0}</font></b>'.format(
"Please unmount your backup volume and cancel "
"the file selection dialog."))
if self.target_appvm:
self.target_appvm.run("QUBESRPC %s dom0" % "qubes"
".SelectDirectory")
else:
file_dialog = QFileDialog()
file_dialog.setReadOnly(True)
file_dialog.getExistingDirectory(
self, "Detach backup device",
os.path.dirname(unicode(self.dir_line_edit.text())))
self.progress_bar.setValue(100)
self.button(self.FinishButton).setEnabled(True)
self.button(self.CancelButton).setEnabled(False)
self.showFileDialog.setEnabled(False)
signal.signal(signal.SIGCHLD, old_sigchld_handler)
def all_vms_good(self):
for vminfo in self.vms_to_restore.values():
if not vminfo.has_key('vm'):
continue
if not vminfo['good-to-go']:
return False
return True
def reject(self):
if self.currentPage() is self.commit_page:
if backup.backup_cancel():
self.emit(SIGNAL("restore_progress(QString)"),
'<font color="red">{0}</font>'
.format("Aborting the operation..."))
self.button(self.CancelButton).setDisabled(True)
else:
self.done(0)
def has_selected_dir(self):
backup_location = unicode(self.dir_line_edit.text())
if not backup_location:
return False
if self.appvm_combobox.currentIndex() == 0:
if os.path.isfile(backup_location) or \
os.path.isfile(os.path.join(backup_location, 'qubes.xml')):
return True
else:
return True
return False
def has_selected_vms(self):
return self.select_vms_widget.selected_list.count() > 0
def backup_location_changed(self, new_dir = None):
self.select_dir_page.emit(SIGNAL("completeChanged()"))
class ProxyServer:
"""Extremely basic proxy server that simply forwards all traffic along.
http://luugiathuy.com/2011/03/simple-web-proxy-python/ used as a reference
Methods:
start(): Start the proxy server subprocess.
stop(): Cleanly stop the proxy server subprocess.
"""
def __init__(self):
self.e = Event() # Event signalling used for stopping the subprocess
self.server = Process(target=self._run, name='Proxy_Server')
self.e.set()
self.sock = None
attempts = 0
while (attempts < 5):
if attempts > 0:
print(
f"Attempting to open socket again in 10 seconds. ({attempts+1}/5)"
)
sleep(10)
try:
# create the socket
self.sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
# associate the socket to a host and port number
self.sock.bind(('', PROXY_PORT_NUMBER))
# release the socket immediately
self.sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
# listen for up to 50 connections
MAX_PENDING_CONNECTIONS = 50
self.sock.listen(MAX_PENDING_CONNECTIONS)
break
except socket.error as e:
if self.sock:
self.sock.close()
print("Could not open socket: " + str(e))
attempts += 1
def start(self):
"""Starts the proxy server."""
self.server.start()
def stop(self):
"""Stops the proxy server."""
self.e.clear()
requests.get(SERVER_URL, proxies={'http': '127.0.0.1:9003'})
self.e.set()
def split_url(self, orig_url):
"""Extracts url and port from a given url."""
# Default values if not specified in request
protocol = 'http://'
path = ''
# separate the fragment if necessary
url = orig_url.decode('utf-8')
url = url.split('#')
if len(url) > 1:
url, fragment = url[0], url[1]
else:
url, fragment = url[0], None
# determine the protocol
url_pos = url.find('://')
if url_pos != -1:
temp = url
protocol = f"{url[:url_pos]}://"
url = url[url_pos + 3:]
# determine the path, if any
path_pos = url.find('/')
if path_pos != -1:
path = url[path_pos:]
url = url[:path_pos]
# determine the port, if any
port_pos = url.find(':')
if port_pos != -1:
port = url[port_pos + 1:]
url = url[:port_pos]
return protocol, url, int(port), path
def _run(self):
while (self.e.is_set()):
# Handle requests
conn, client_addr = self.sock.accept()
handler = Process(target=self._proxy,
name='Request_Handler',
args=(conn, client_addr))
# Since this is only used for testing, we want this to be synchronous.
handler.start()
conn.close()
# Close the socket when we're done.
self.sock.close()
def _proxy(self, conn, client_addr):
# read the request
request = conn.recv(MAX_DATA_RECV)
# we need to extract the URL from the first line of the request
all_lines = request.split(bytes('\n', 'utf-8'))
first_line = all_lines[0]
# method is [0], url is [1]
orig_url = first_line.split(bytes(' ', 'utf-8'))[1]
# Split the url into its components.
protocol, base_url, port, path = self.split_url(orig_url)
try:
# Create a socket to connect to the server
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
s.connect((base_url, port))
s.send(request)
while True:
# Receive data from web server
data = s.recv(MAX_DATA_RECV)
if len(data) > 0:
# send to browser
conn.send(data)
else:
break
except socket.error as e:
print("Runtime Error: " + str(e))
finally:
if s:
s.close()
if conn:
conn.close()
class ValkkaProcess(Process):
"""
Semantics:
Frontend: the part of the forked process that keeps running in the current, user virtual memory space
Backend : the part of the forked process that runs in its own virtual memory space (e.g. "in the background")
This class has both backend and frontend methods:
- Backend methods should only be called from backend. They are designated with "_".
- Frontend methods should only be called from frontend
To avoid confusion, backend methods are designated with "_", except for the "run()" method, that's always in the backend
Frontend methods use a pipe to send a signal to backend that then handles the signal with a backend method having the same name (but with "_" in the end)
Backend methods can, in a similar fashion, send signals to the frontend using a pipe. In frontend, a listening thread is needed. That thread can then call
the handleSignal method that chooses the correct frontend method to call
TODO: add the possibility to bind the process to a certain processor
"""
# incoming signals : from frontend to backend
incoming_signal_defs = { # each key corresponds to a front- and backend methods
"test_": {
"test_int": int,
"test_str": str
},
"stop_": []
}
# outgoing signals : from back to frontend. Don't use same names as for
# incoming signals ..
outgoing_signal_defs = {
"test_o": {
"test_int": int,
"test_str": str
},
}
def __init__(self, name, affinity=-1, **kwargs):
super().__init__()
self.pre = self.__class__.__name__ + " : " + name + \
" : " # auxiliary string for debugging output
self.name = name
self.affinity = affinity
self.signal_in = Event()
self.signal_out = Event()
# communications pipe. Frontend uses self.pipe, backend self.childpipe
self.pipe, self.childpipe = Pipe()
self.signal_in.clear()
self.signal_out.clear()
# print(self.pre, "init")
def getPipe(self):
"""Returns communication pipe for front-end
"""
return self.pipe
def preRun_(self):
"""After the fork, but before starting the process loop
"""
if (self.affinity > -1):
os.system("taskset -p -c %d %d" % (self.affinity, os.getpid()))
def postRun_(self):
"""Just before process exit
"""
print(self.pre, "post: bye!")
def cycle_(self):
# Do whatever your process should be doing, remember timeout every now
# and then
time.sleep(5)
print(self.pre, "hello!")
def startAsThread(self):
from threading import Thread
t = Thread(target=self.run)
t.start()
def run(self
): # No "_" in the name, but nevertheless, running in the backed
"""After the fork. Now the process starts running
"""
# print(self.pre," ==> run")
self.preRun_()
self.running = True
while (self.running):
self.cycle_()
self.handleSignal_()
self.postRun_()
def handleSignal_(self):
"""Signals handling in the backend
"""
if (self.signal_in.is_set()):
signal_dic = self.childpipe.recv()
method_name = signal_dic.pop("name")
method = getattr(self, method_name)
method(**signal_dic)
self.signal_in.clear()
self.signal_out.set()
def sendSignal(self, **kwargs
): # sendSignal(name="test",test_int=1,test_str="kokkelis")
"""Incoming signals: this is used by frontend methods to send signals to the backend
"""
try:
name = kwargs.pop("name")
except KeyError:
raise (AttributeError("Signal name missing"))
# a dictionary: {"parameter_name" : parameter_type}
model = self.incoming_signal_defs[name]
for key in kwargs:
# raises error if user is using undefined signal
model_type = model[key]
parameter_type = kwargs[key].__class__
if (model_type == parameter_type):
pass
else:
raise (AttributeError("Wrong type for parameter " + str(key)))
kwargs["name"] = name
self.pipe.send(kwargs)
self.signal_out.clear()
self.signal_in.set() # indicate that there is a signal
self.signal_out.wait() # wait for the backend to clear the signal
def handleSignal(self, signal_dic):
"""Signal handling in the frontend
"""
method_name = signal_dic.pop("name")
method = getattr(self, method_name)
method(**signal_dic)
def sendSignal_(self, **kwargs): # sendSignal_(name="test_out",..)
"""Outgoing signals: signals from backend to frontend
"""
try:
name = kwargs.pop("name")
except KeyError:
raise (AttributeError("Signal name missing"))
# a dictionary: {"parameter_name" : parameter_type}
model = self.outgoing_signal_defs[name]
for key in kwargs:
# raises error if user is using undefined signal
try:
model_type = model[key]
except KeyError:
print("your outgoing_signal_defs for", name, "is:", model)
print("you requested key:", key)
raise
parameter_type = kwargs[key].__class__
if (model_type == parameter_type):
pass
else:
raise (AttributeError("Wrong type for parameter " + str(key)))
kwargs["name"] = name
self.childpipe.send(kwargs)
# *** backend methods corresponding to each incoming signals ***
def stop_(self):
self.running = False
def test_(self, test_int=0, test_str="nada"):
print(self.pre, "test_ signal received with", test_int, test_str)
# ** frontend methods corresponding to each incoming signal: these communicate with the backend via pipes **
def stop(self):
self.sendSignal(name="stop_")
def test(self, **kwargs):
dictionaryCheck(self.incoming_signal_defs["test_"], kwargs)
kwargs["name"] = "test_"
self.sendSignal(**kwargs)
# ** frontend methods corresponding to each outgoing signal **
# typically, there is a QThread in the frontend-side reading the process pipe
# the QThread reads kwargs dictionary from the pipe, say
# {"name":"test_o", "test_str":"eka", "test_int":1}
# And calls handleSignal(kwargs)
def test_o(self, **kwargs):
pass
class DataProcessingMultiprocessing(DataProcessing):
def __init__(self, options, log=None):
DataProcessing.__init__(self, options)
self.log = log
self.exit = Event()
utils.startup(self.options)
# First queue - Send the jobs
self.tasks = multiprocessing.JoinableQueue(maxsize=10000)
# Second one - Send the number of images to process
self.N_queue = multiprocessing.Queue()
# Third one - the results
self.results = multiprocessing.Queue()
# Process to look after the files
self.FS = FileSentinel(self.tasks, self.N_queue, self.options)
# Real workers...
if 'h5' in self.options['file_extension'].lower():
from .MultiProcess import MProcessEiger as MProcess
else:
from .MultiProcess import MProcess
self.consumers = [
MProcess(self.tasks,
self.results,
self.options,
self.ai,
self.detector,
name=str(i)) for i in range(self.options['cpus'])
]
# Some stats
self.statsManager = StatsManager(self.options, self.results)
def run(self):
self.startMP()
self.startFS()
i = 0
if self.options["live"]:
try:
while not self.exit.is_set():
self.getStats()
i += 1
except KeyboardInterrupt:
print(
"\n\nCtrl-c received! --- Aborting and trying not to compromising results..."
)
else:
while (self.statsManager.processed != self.statsManager.total
or self.statsManager.processed
== 0) and not self.exit.is_set():
try:
self.getStats()
except KeyboardInterrupt:
print(
"\n\nCtrl-c received --- Aborting and trying not to compromising results..."
)
break
self.statsManager.getFinalResults()
def shutDown(self):
self.exit.set()
def getStats(self):
while True or self.statsManager.processed != self.statsManager.total:
try:
self.statsManager.total = self.N_queue.get(block=False,
timeout=None)
self.statsManager.chunk = max(
int(self.statsManager.total / 1000.), 20)
except Queue.Empty:
pass
try:
self.statsManager.getResults()
except Queue.Empty:
break
def startFS(self):
self.FS.daemon = True
self.FS.start()
def startMP(self):
""" Start as many processes as set by the user
"""
for w in self.consumers:
w.start()
class SerialPort(Process):
"""
A multiprocessing based wrapper for an instance of pyserial Serial.
A RobotInterface class manages all instances of this class.
This class is for internal use only
"""
port_updates_per_second = 1000
def __init__(self, port_address):
"""
:param queue: a multiprocessing queue to which packets are passed
:param lock: a shared lock to prevent multiple sources accessing the queue
:param counter: a queue size counter. Keeps track of the number of packets in the queue
:param updates_per_second: How often the port should update. This is passed to a Clock instance
"""
self.address = port_address
self.packet_queue = Queue()
self.counter = Value('i', 0)
self.lock = Lock()
self.queue_len = 0
# status variables
self.configured = True
self.abides_protocols = True
self.port_assigned = False
self.message_lock = Lock()
self.error_message = Queue()
self.print_out_lock = Lock()
self.debug_print_outs = Queue()
# time variables
self.start_time = 0.0
self.loop_time = 0.0
# whoiam ID info
self.whoiam = None # ID tag of the microcontroller
self.whoiam_header = "iam" # whoiam packets start with "iam"
self.whoiam_ask = "whoareyou"
# first packet info
self.first_packet = None
self.first_packet_ask = "init?"
self.first_packet_header = "init:"
self.stop_packet_ask = "stop"
self.stop_packet_header = "stopping"
self.protocol_timeout = 3 # seconds
self.protocol_packets = [
self.whoiam_header, self.first_packet_header,
self.stop_packet_header
]
# misc. serial protocol
self.packet_end = "\n" # what this microcontroller's packets end with
self.default_rate = 115200
self.baud_rate = Value('i', self.default_rate)
# buffer for putting packets into
self.buffer = ""
self.prev_read_packets = []
self.prev_write_packet = ""
# leaves tabs, letters, numbers, spaces, newlines, and carriage returns
self.buffer_pattern = re.compile("([^\r\n\t\x20-\x7e]|_)+")
# events and locks
self.exit_event_lock = Lock()
self.start_event_lock = Lock()
self.exit_event = Event()
self.start_event = Event()
self.stop_event = Event()
self.serial_lock = Lock()
self.serial_ref = None
super(SerialPort, self).__init__(target=self.update)
# ----- initialization methods -----
def initialize(self):
# attempt to open the serial port
try:
self.serial_ref = serial.Serial(port=self.address,
baudrate=self.baud_rate.value)
except SerialException as error:
self.handle_error(error, traceback.format_stack())
self.configured = False
time.sleep(2) # wait for microcontroller to wake up
if self.configured:
# Find the ID of this port. The ports will be matched up to the correct RobotObject later
self.find_whoiam()
if self.whoiam is not None:
self.find_first_packet()
else:
self.debug_print(
"whoiam ID was None, skipping find_first_packet")
else:
self.debug_print("Port not configured. Skipping find_whoiam")
def send_start(self):
"""
Send the start flag
For external use
:return: None
"""
self.debug_print("sending start")
return self.write_packet("start")
def find_whoiam(self):
"""
Get the whoiam packet from the microcontroller. This method will wait 1 second
until the packet is received
example:
sent: "whoareyou\n"
received: "iamlidar\n"
When the packet is found, parse_whoiam_packet is called and whoiam is assigned
For initialization
:return: whoiam packet and first_packet
"""
self.whoiam = self.check_protocol(self.whoiam_ask, self.whoiam_header)
if self.whoiam is not None:
self.debug_print("%s has ID '%s'" % (self.address, self.whoiam))
else:
# self.configured = False
self.abides_protocols = False
self.debug_print("Failed to obtain whoiam ID!", ignore_flag=True)
def find_first_packet(self):
"""
Get the first packet from the microcontroller. This method will wait 1 second
until the packet is received
example:
sent: "init?\n"
received: "init:\n" (if nothing to init, initialization methods not called)
received: "init:something interesting\t01\t23\n"
'something interesting\t01\t23' would be the first packet
When the packet is found, parse_whoiam_packet is called and whoiam is assigned
For initialization
:return: whoiam packet and first_packet
"""
self.first_packet = self.check_protocol(self.first_packet_ask,
self.first_packet_header)
if self.first_packet is not None:
self.debug_print("sent initialization data: %s" %
repr(self.first_packet))
else:
# self.configured = False
self.abides_protocols = False
self.debug_print("Failed to obtain first packet!",
ignore_flag=True)
def check_protocol(self, ask_packet, recv_packet_header):
"""
A call and response method. After an "ask packet" is sent, the process waits for
a packet with the expected header for 2 seconds
For initialization
:param ask_packet: packet to send
:param recv_packet_header: what the received packet should start with
:return: the packet received without the header and packet end
"""
self.debug_print("Checking '%s' protocol" % ask_packet)
if not self.write_packet(ask_packet):
return None # return None if write failed
start_time = time.time()
abides_protocol = False
answer_packet = ""
attempts = 0
rounded_time = 0
# wait for the correct response
while not abides_protocol:
in_waiting = self.in_waiting()
if in_waiting > 0:
packets = self.read_packets(in_waiting)
if packets is None:
return None
self.print_packets(packets)
# return None if read failed
if packets is None:
self.handle_error(
"Serial read failed... Board never signalled ready",
traceback.format_stack())
return None
# parse received packets
for packet in packets:
if len(packet) == 0:
self.debug_print("Empty packet! Contained only \\n")
continue
if packet[0:len(
recv_packet_header
)] == recv_packet_header: # if the packet starts with the header,
self.debug_print("received packet: " + repr(packet))
answer_packet = packet[len(
recv_packet_header):] # record it and return it
abides_protocol = True
prev_rounded_time = rounded_time
rounded_time = int((time.time() - start_time) * 10)
if rounded_time > 5 and rounded_time % 3 == 0 and prev_rounded_time != rounded_time:
attempts += 1
self.debug_print("Writing '%s' again" % ask_packet)
if not self.write_packet(ask_packet):
return None
# return None if operation timed out
if (time.time() - start_time) > self.protocol_timeout:
self.handle_error(
"Didn't receive response for packet '%s'. Operation timed out."
% ask_packet, traceback.format_stack())
return None
return answer_packet # when the while loop exits, abides_protocol must be True
def handle_error(self, error, stack_trace):
"""
When errors occur in a RobotSerialPort, the process doesn't crash. The error is recorded,
self.update is stopped, and the main process is notified so all other ports can close safely
For initialization and process use
:param error: The error message to record
:return: None
"""
with self.exit_event_lock:
self.exit_event.set()
# if self.error_message.empty():
full_message = ""
for line in stack_trace:
full_message += str(line)
if type(error) == str:
full_message += error
else:
full_message += "%s: %s\n" % (error.__class__.__name__, str(error))
full_message += "Previous read: %s, write: %s" % (
self.prev_read_packets, self.prev_write_packet)
with self.message_lock:
# queue will always be size of one. Easiest way to share strings and avoid race conditions.
# (sometimes the error message would have arrived incomplete because
# it gets printed before it gets formed...)
self.error_message.put(full_message)
# ----- run methods -----
def update(self):
"""
Called when RobotSerialPort.start is called
:return: None
"""
self.start_time = time.time()
clock = Clock(SerialPort.port_updates_per_second)
clock.start(self.start_time)
# with self.start_event_lock:
# self.start_event.set()
time.sleep(0.01)
with self.baud_rate.get_lock():
if self.baud_rate.value != self.default_rate: # if changed externally
self.serial_ref.baudrate = self.baud_rate.value
self.debug_print("Baud is now", self.serial_ref.baudrate)
else:
self.debug_print("Baud rate unchanged")
with self.start_event_lock:
self.start_event.set()
try:
while True:
with self.exit_event_lock:
if self.exit_event.is_set():
break
# close the process if the serial port isn't open
with self.serial_lock:
if not self.serial_ref.isOpen():
self.stop()
raise RobotSerialPortClosedPrematurelyError(
"Serial port isn't open for some reason...", self)
in_waiting = self.in_waiting()
if in_waiting is None:
self.stop()
raise RobotSerialPortClosedPrematurelyError(
"Failed to check serial. Is there a loose connection?",
self)
elif in_waiting > 0:
# read every possible character available and split them into packets
packets = self.read_packets(in_waiting)
if packets is None: # if the read failed
self.stop()
raise RobotSerialPortReadPacketError(
"Failed to read packets", self)
# put data found into the queue
with self.lock:
for packet in packets:
put_on_queue = True
for header in self.protocol_packets:
if len(packet) >= len(
header
) and packet[:len(header)] == header:
if header == self.stop_packet_header:
self.stop()
raise RobotSerialPortClosedPrematurelyError(
"Port signalled to exit (stop flag was found)",
self)
else:
self.debug_print(
"Misplaced protocol packet:",
repr(packet))
put_on_queue = False
if put_on_queue:
self.packet_queue.put(
(time.time(), packet))
# start_time isn't used. The main process has its own initial time reference
self.counter.value += len(packets)
clock.update() # maintain a constant loop speed
except KeyboardInterrupt:
self.debug_print("KeyboardInterrupt in port loop")
self.debug_print("Current buffer:", repr(self.buffer))
self.debug_print("While loop exited. Exit event triggered.")
if not self.send_stop_events():
self.handle_error("Stop flag failed to send!",
traceback.format_stack())
def in_waiting(self):
try:
return self.serial_ref.inWaiting()
except OSError as error:
self.debug_print(
"Failed to check serial. Is there a loose connection?")
self.handle_error(error, traceback.format_stack())
return None
def read_packets(self, in_waiting):
"""
Read all available data on serial and split them into packets as
indicated by packet_end.
For initialization and process use
:return: None indicates the serial read failed and that the communicator thread should be stopped.
returns the received packets otherwise
"""
try:
# read every available character
if self.serial_ref.isOpen():
incoming = self.serial_ref.read(in_waiting)
else:
self.handle_error("Serial port wasn't open for reading...",
traceback.format_stack())
return None
except BaseException as error:
self.handle_error(error, traceback.format_stack())
return None
if len(incoming) > 0:
# append to the buffer
try:
self.buffer += incoming.decode("utf-8", "ignore")
except UnicodeDecodeError as error:
self.handle_error(error, traceback.format_stack())
return None
buf = self.buffer_pattern.sub('', self.buffer)
if len(self.buffer) != len(buf):
self.debug_print("Invalid characters found:",
repr(self.buffer))
self.buffer = buf
if len(self.buffer) > len(self.packet_end):
# split based on user defined packet end
packets = self.buffer.split(self.packet_end)
self.prev_read_packets = packets
# reset the buffer
self.buffer = packets.pop(-1)
return packets
return []
def write_packet(self, packet):
"""
Safely write a packet over serial. Automatically appends packet_end to the input.
For initialization and process use
:param packet: an arbitrary string without packet_end in it
:return: True or False if the write was successful
"""
self.prev_write_packet = str(packet)
try:
data = bytearray(str(packet) + self.packet_end, 'ascii')
except TypeError as error:
self.handle_error(error, traceback.format_stack())
return False
try:
if self.serial_ref.isOpen():
with self.serial_lock:
self.serial_ref.write(data)
else:
self.handle_error("Serial port wasn't open for writing...",
traceback.format_stack())
return False
except BaseException as error:
self.handle_error(error, traceback.format_stack())
return False
return True
def print_packets(self, packets):
"""
If debug_prints is True, print repr of all incoming packets
:param packets: a list of received packets
:return: None
"""
for packet in packets:
self.debug_print("> %s" % repr(packet))
def flush(self):
self.debug_print("Flushing serial")
self.serial_ref.reset_input_buffer()
self.serial_ref.reset_output_buffer()
self.debug_print("Serial content:", self.in_waiting())
# ----- external and status methods -----
def debug_print(self, *strings, ignore_flag=False):
"""
For initialization and process use
:param strings:
:param ignore_flag:
:return:
"""
string = "[%s] %s" % (self.whoiam, " ".join(map(str, strings)))
with self.print_out_lock:
self.debug_print_outs.put(string)
def change_rate(self, new_baud_rate):
"""
For external use
:param new_baud_rate:
:return:
"""
self.debug_print("Setting baud to", new_baud_rate)
with self.baud_rate.get_lock():
self.baud_rate.value = new_baud_rate
self.debug_print("Set baud to", self.baud_rate.value)
def is_running(self):
"""
Check if the port's thread is running correctly
For external use
:return:
-1: event event thrown
0: self.configured is False
1: process hasn't started or everything is fine
"""
with self.start_event_lock:
if not self.start_event.is_set(): # process hasn't started
return 1
if not self.configured:
return 0
with self.exit_event_lock:
if self.exit_event.is_set():
return -1
return 1
def send_stop_events(self):
"""
For process use
:return:
"""
# with self.exit_event_lock:
# if self.exit_event.is_set():
# self.debug_print("Exit event already set. Stop was sent internally")
# return True
if self.start_time > 0 and time.time(
) - self.start_time <= 2: # wait for arduino to listen
time.sleep(2)
if not self.stop_event.is_set():
if self.check_protocol(self.stop_packet_ask,
self.stop_packet_header) is None:
self.debug_print("Failed to send stop flag!!!")
return False
else:
self.debug_print("Sent stop flag")
self.stop_event.set()
self.debug_print("Acquiring start lock")
with self.start_event_lock:
if not self.start_event.is_set():
self.debug_print("start_event not set! Closing serial")
self.close_serial()
self.debug_print("Releasing start lock")
else:
self.debug_print("Stop event already set!")
return True
def close_serial(self):
"""
For external use
:return:
"""
self.debug_print("Acquiring serial lock")
with self.serial_lock:
if self.configured:
if self.serial_ref.isOpen():
self.serial_ref.stop()
self.debug_print("Closing serial")
else:
self.debug_print("Serial port was already closed!")
else:
self.debug_print("Port wasn't configured!!")
self.debug_print("Releasing serial lock")
def stop(self):
"""
Send stop packet, close the serial port.
For external use
:return: None
"""
self.debug_print("Acquiring exit lock")
if not self.exit_event.is_set():
self.exit_event.set()
else:
self.debug_print("Exit event already set! Error was likely thrown")
self.debug_print("Releasing exit lock")
def has_exited(self):
"""
For external use
:return:
"""
return self.exit_event.is_set()
def __str__(self):
return "%s(port_address=%s)" % (self.__class__.__name__, self.address)
def __repr__(self):
return self.__str__()
class ProcessServer(threading.Thread):
'''
This type manages the creation of subprocesses and the despatch of computational tasks to them. Typically the global
instance is created at startup through createGlobalServer() at which point the subprocesses are created. Tasks are
enqueued to be executed through callProcessFunc() or indirectly if a routine decorated with @concurrent is called.
'''
globalServer=None # global instance of the server
@staticmethod
def createGlobalServer(realnumprocs=cpu_count()):
'''Creates the global instance of the server, `realnumprocs' being the number of processes to create.'''
ProcessServer.globalServer=ProcessServer(realnumprocs)
ProcessServer.globalServer.start()
def __init__(self,realnumprocs=cpu_count()):
threading.Thread.__init__(self)
self.daemon=True
self.realnumprocs=clamp(realnumprocs,1,cpu_count())
self.procs=[]
self.sharer=ObjectSharer()
self.syncEvent=Event()
self.syncEvent2=Event()
self.syncCounter=Value('i',0)
self.syncLock=Lock()
self.jobqueue=queue.Queue()
self.progress=Array('l',self.realnumprocs)
self.objsrv=ObjectServer()
self.stopEvent=Event()
def callProcessFunc(self,valrange,numprocs,task,target,*args,**kwargs):
'''
Sends the request to execute `target' in parallel with the given `args' and `kwargs' argument values. The
`valrange' value is used to set the value range members of the AlgorithmProcess objects, `numprocs' is the
number of processes to execute `target' on (1 will cause `target' to be executed in the main process, a value
<=0 will mean all the processes will be used), and `task' is the (possibly None) Task object to report
progress to. The return value is a Future object which will contain the results or exceptions thrown.
The `target' routine must exist in a module scope at runtime so inner routines or dynamically created routines
cannot be used here. When called, the first argument will be the AlgorithmProcess object followed by those in
`args' and `kwargs'. The AlgorithmProcess instance will be different for each process `target' is called in,
its members will state which process and the subset of `valrange' assigned to it. The expectation is that
`valrange' is the number of elements `target' is used to operate on, when called each process is assigned a
subset of the range [0,`valrange') and the internal algorithm of `target' is expected to iterate over those
values only.
The optional argument "partitionArgs" in `kwargs' may contain a tuple containing the members in `args' which
are iterable and which should be partitioned amongst the processes. Values in `args' and `kwargs' are normally
copied to each process, so this allows large iterables to be partitioned and not needlessly duplicated.
The @concurrent can be used to wrap the invocation of callProcessFunc() within the function definition itself.
The first argument of the routine must still be the AlgorithmProcess object but when called three arguments
representing `valrange', `numprocs', and `task' must be provided instead. The routine will also no block until
the processing is complete and return the results instead of a Future object.
Example:
def testfunc(process):
return (process.index,int(process.startval),int(process.endval))
result=ProcessServer.globalServer.callProcessFunc(50,4,None,testfunc)
printFlush(listResults(result()))
Output: [(0, 0, 12), (1, 12, 25), (2, 25, 37), (3, 37, 50)]
Example:
@concurrent
def testfunc(process,values):
return (process.index,values)
values=range(20)
result=testfunc(len(values),3,None,values,partitionArgs=(values,))
printFlush(listResults(result))
Output: [(0, [0, 1, 2]), (1, [3, 4, 5]), (2, [6, 7, 8, 9])]
'''
result=Future()
self.jobqueue.put((valrange,numprocs,task,target,args,kwargs,result))
return result
def prepareArgs(self,index,numprocs,args,partArgs):
'''Prepare arguments by dividing lists/tuples present in both `args' and `partArgs' into the slice for proc `index'.'''
pargs=[]
for a in args: # construct an argument list `pargs' to be passed to the process object
if a in partArgs:
astart,aend=partitionSequence(len(a),index,numprocs)
pargs.append(a[astart:aend]) # replace `a' with a per-process slice of `a'
else:
pargs.append(a) # use `a' directly
return pargs
def run(self):
'''Run the server, reading messages from the job queue and sending them to the work processes.'''
atexit.register(self.stop)
# do not create processes if the number of procs is 1, this forces single process mode
if self.realnumprocs>1:
# start all the processes
for i in range(self.realnumprocs):
psharer=self.objsrv.getProxy(self.sharer)
p=AlgorithmProcess(i,self.realnumprocs,self.syncEvent,self.syncEvent2,self.syncCounter,self.syncLock,psharer,self.progress,self.stopEvent,os.getpid())
p.start()
self.procs.append(p)
# continually read items from `self.jobqueue' and send the execution requests to the processes
while not self.stopEvent.is_set():
valrange,numprocs,task,target,args,kwargs,result=self.jobqueue.get(True) # get message
partArgs=kwargs.pop('partitionArgs',()) # get list of objects to partition between processes
numprocs=min(valrange,self.realnumprocs if numprocs<=0 or numprocs>self.realnumprocs else numprocs) # get number of processes to use,
self.sharer.clear()
for i in range(self.realnumprocs): # reset the progress counting shared array
self.progress[i]=0
if task: # set the task's progress value
task.setMaxProgress(valrange)
with result:
if numprocs==1 or not self.procs: # if we are to use only one process execute locally instead of 1 concurrent process
try:
# construct a local process, passing None for parameters clues it in to not try using concurrency features like syncing
localproc=AlgorithmProcess(0,1,None,None,None,None,None,task,None,0)
localproc.endval=valrange
localproc.maxval=valrange
tresult=target(localproc,*args,**kwargs)
result.setObject({0:tresult})
except Exception as e:
printFlush('LOCALPROC',e)
traceback.print_exc()
result.setObject({0:e})
else: # otherwise use work processes
self.syncCounter.value=0 # reset the sync counter, the processes can't do this themselves cleanly without race condition
# for each process prepare the arguments to the target and send the request through its `send' pipe
for i in range(numprocs):
start,end=partitionSequence(valrange,i,numprocs)
pargs=self.prepareArgs(i,numprocs,args,partArgs)
self.procs[i].send.send((target,pargs,kwargs,start,end,valrange,numprocs)) # send the job
try:
# wait for each process being used to finish, updating the task object's progress if it's present
while any(not p.rrecv.poll(0.01) for p in self.procs[:numprocs]): # a process is done when it has sent its result
if task:
task.setProgress(sum(self.progress))
if task: # do a final update
task.setProgress(sum(self.progress))
result.setObject(dict((p.index,p.rrecv.recv()) for p in self.procs[:numprocs])) # map results to process index
except Exception as e:
# some error occurred, send e as the result for each process even though it wasn't actually thrown by the processes
result.setObject(dict((p.index,e) for p in self.procs[:numprocs]))
def stop(self):
'''Stops the processes and object server, no execution after this is possible.'''
self.objsrv.stop()
self.stopEvent.set()
class PlantDraw(object):
def __init__(self, plant, refresh_period=(1.0/240),
name='PlantDraw', *args, **kwargs):
super(PlantDraw, self).__init__()
self.name = name
self.plant = plant
self.drawing_thread = None
self.polling_thread = None
self.dt = refresh_period
self.exec_time = time()
self.scale = 150 # pixels per meter
self.center_x = 0
self.center_y = 0
self.running = Event()
self.polling_pipe, self.drawing_pipe = Pipe()
def init_ui(self):
plt.close(self.name)
self.fig = plt.figure(self.name)
self.ax = plt.gca()
self.ax.set_xlim([-1.5, 1.5])
self.ax.set_ylim([-1.5, 1.5])
self.ax.set_aspect('equal', 'datalim')
self.ax.grid(True)
self.fig.canvas.draw()
self.bg = self.fig.canvas.copy_from_bbox(self.ax.bbox)
self.cursor = Cursor(self.ax, useblit=True, color='red', linewidth=2)
self.init_artists()
# plt.ion()
plt.show(False)
def drawing_loop(self, drawing_pipe):
# start the matplotlib plotting
self.init_ui()
while self.running.is_set():
exec_time = time()
# get any data from the polling loop
updts = None
while drawing_pipe.poll():
data_from_plant = drawing_pipe.recv()
if data_from_plant is None:
self.running.clear()
break
# get the visuzlization updates from the latest state
state, t = data_from_plant
updts = self.update(state, t)
self.update_canvas(updts)
# sleep to guarantee the desired frame rate
exec_time = time() - exec_time
plt.waitforbuttonpress(max(self.dt-exec_time, 1e-9))
self.close()
def close(self):
# close the matplotlib windows, clean up
# plt.ioff()
plt.close(self.fig)
def update(self, *args, **kwargs):
plt.figure(self.name)
updts = self._update(*args, **kwargs)
self.update_canvas(updts)
def _update(self, *args, **kwargs):
msg = "You need to implement the self._update() method in your\
PlantDraw class."
raise NotImplementedError(msg)
def init_artists(self, *args, **kwargs):
msg = "You need to implement the self.init_artists() method in your\
PlantDraw class."
raise NotImplementedError(msg)
def update_canvas(self, updts):
if updts is not None:
# update the drawing from the plant state
self.fig.canvas.restore_region(self.bg)
for artist in updts:
self.ax.draw_artist(artist)
self.fig.canvas.update()
# sleep to guarantee the desired frame rate
exec_time = time() - self.exec_time
plt.waitforbuttonpress(max(self.dt-exec_time, 1e-9))
self.exec_time = time()
def polling_loop(self, polling_pipe):
current_t = -1
while self.running.is_set():
exec_time = time()
state, t = self.plant.get_state(noisy=False)
if t != current_t:
polling_pipe.send((state, t))
# sleep to guarantee the desired frame rate
exec_time = time() - exec_time
sleep(max(self.dt-exec_time, 0))
def start(self):
print_with_stamp('Starting drawing loop', self.name)
self.drawing_thread = Process(target=self.drawing_loop,
args=(self.drawing_pipe, ))
self.drawing_thread.daemon = True
self.polling_thread = Thread(target=self.polling_loop,
args=(self.polling_pipe, ))
self.polling_thread.daemon = True
# self.drawing_thread = Process(target=self.run)
self.running.set()
self.polling_thread.start()
self.drawing_thread.start()
def stop(self):
self.running.clear()
if self.drawing_thread is not None and self.drawing_thread.is_alive():
# wait until thread stops
self.drawing_thread.join(10)
if self.polling_thread is not None and self.polling_thread.is_alive():
# wait until thread stops
self.polling_thread.join(10)
print_with_stamp('Stopped drawing loop', self.name)
class AppProcess(Process):
"""
Process with some additional functionality and fixes
* Support for a multiprocessing logger
* Removes signals to prevent join problems
* Propagates exceptions to owner process
* Safe terminate with timeout, followed by force terminate
"""
# Timeout before process is force terminated
__DEFAULT_TERMINATE_TIMEOUT_MS = 1000
def __init__(self, name: str):
self.__name = name
super().__init__(name=self.__name)
self.mp_logger = None
self.logger = logging.getLogger(self.__name)
self.__exception_queue = Queue()
self._terminate = Event()
def set_multiprocessing_logger(self, mp_logger: MultiprocessingLogger):
self.mp_logger = mp_logger
@overrides(Process)
def run(self):
# Replace the signal handlers that may have been set by main process to
# default handlers. Having non-default handlers in subprocesses causes
# a deadlock when attempting to join the process
# Info: https://stackoverflow.com/a/631605
# NOTE: There is a minuscule chance of deadlock if a signal is received
# between start of the method and these resets.
# The ideal solution is to remove the signal before the process is
# started. Unfortunately that's difficult to do here because the
# subprocess is started from a job thread, and python doesn't
# allow setting signals from outside the main thread.
# So we accept this risk for the quick and easy solution here
signal.signal(signal.SIGTERM, signal.SIG_DFL)
signal.signal(signal.SIGINT, signal.SIG_DFL)
# Set the thread name for convenience
threading.current_thread().name = self.__name
# Configure the logger for this process
if self.mp_logger:
self.logger = self.mp_logger.get_process_safe_logger().getChild(self.__name)
self.logger.debug("Started process")
self.run_init()
try:
while not self._terminate.is_set():
self.run_loop()
self.logger.debug("Process received terminate flag")
except ServiceExit:
self.logger.debug("Process received a ServiceExit")
except Exception as e:
self.logger.debug("Process caught an exception")
self.__exception_queue.put(ExceptionWrapper(e))
raise
finally:
self.run_cleanup()
self.logger.debug("Exiting process")
@overrides(Process)
def terminate(self):
# Send a terminate signal, and force terminate after a timeout
self._terminate.set()
def elapsed_ms(start):
delta_in_s = (datetime.now() - start).total_seconds()
delta_in_ms = int(delta_in_s * 1000)
return delta_in_ms
timestamp_start = datetime.now()
while self.is_alive() and \
elapsed_ms(timestamp_start) < AppProcess.__DEFAULT_TERMINATE_TIMEOUT_MS:
pass
super().terminate()
def propagate_exception(self):
"""
Raises any exception that was caught by the process
:return:
"""
try:
exc = self.__exception_queue.get(block=False)
raise exc.re_raise()
except queue.Empty:
pass
@abstractmethod
def run_init(self):
"""
Called once before the run loop
:return:
"""
pass
@abstractmethod
def run_cleanup(self):
"""
Called once before cleanup
:return:
"""
pass
@abstractmethod
def run_loop(self):
"""
Process behaviour should be implemented here.
This function is repeatedly called until process exits.
The check for graceful shutdown is performed between the loop iterations,
so try to limit the run time for this method.
:return:
"""
pass
class DAMSComm(Process):
def __init__(self, ip_address, port, data_queue, message_length,
socket_timeout, socket_log_conf):
Process.__init__(self)
self._ip_address = ip_address
self._port = port
self._data_queue = data_queue
self._message_length = message_length
self._socket_timeout = socket_timeout
self._socket_log_config = socket_log_conf
self._shutdown_event = Event()
def close(self):
self._shutdown_event.set()
return
def connect(self):
try:
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sock.settimeout(self._socket_timeout)
sock.connect((self._ip_address, self._port))
return sock
except Exception as e:
raise e
return None
def run(self):
reconnect_attempts = -1
reconnect_cnt = 0
process_data = True
logger = None
try:
logging.config.fileConfig(self._socket_log_config)
logger = logging.getLogger("DAMSNTSocket")
logger.info('Socket logging file opened.')
logger.info("Connecting to ip: %s port: %d" %
(self._ip_address, self._port))
try:
sock = self.connect()
except Exception as e:
logger.error("Failed to connect to ip: %s port: %d" %
(self._ip_address, self._port))
logger.exception(e)
while process_data:
if not self._shutdown_event.is_set():
if sock is not None:
try:
data = sock.recv(self._message_length)
if len(data) > 0:
self._data_queue.put(data)
else:
sock.close()
logger.error(
"Disconnected, attempted reconnect.")
sock = None
reconnect_cnt = 0
except socket.timeout as e:
logger.error(
"Socket timed out. Closing for reconnect.")
sock.close()
sock = None
reconnect_cnt = 0
except Exception as e:
logger.exception(e)
sock.close()
sock = None
reconnect_cnt = 0
else:
if reconnect_attempts == -1 or reconnect_cnt <= reconnect_attempts:
logger.error("Reconnect: %d to ip: %s port: %d"\
% (reconnect_cnt, self._ip_address, self._port))
try:
sock = self.connect()
except Exception as e:
logger.error(
"Failed to connect to ip: %s port: %d" %
(self._ip_address, self._port))
logger.exception(e)
time.sleep(5)
reconnect_cnt += 1
else:
logger.error(
"Exceeded reconnect attempts, exiting.")
process_data = False
else:
logger.info("Shutdown event signalled.")
process_data = False
sock.close()
except Exception as e:
if logger is not None:
logger.debug(e)
else:
traceback.print_exc()
class Forwarder(Process):
"""
+-------------------------+
| Redis |
+-----------||------------+
RedisPubSub + hashset
||
+-----------vv------------+
| |
forwarder-service ---reg_ep---|-> Forwarder |
| |
+-------------------------+
/ / / | ^ |
/ / / | | |
/ / / | | |
/ / / Tasks Results Commands
/ / / | | |
V V V V | V
+-------------+ +--------------------+
| Endpoint 1 | ... | Endpoint N |
+-------------+ +--------------------+
Endpoint States
---------------
- Registered: Endpoint has been registered with the service but has not yet
connected
- Connected: Endpoint has connected, meaning ZMQ messages can be sent and received
- Disconnected: ZMQ message sending is failing for this endpoint, making it
disconnected
* All endpoints and endpoint states are forgotten when a forwarder restarts
* Endpoints always start in the Registered state. Before that, the forwarder does
not know they exist * Endpoints can send a registration message at any time while
in Connected/Disconnected state and it will not impact the current state
* Results can arrive on the forwarder from an endpoint when it is in any state. See
the NOTE below at handle_results
Endpoint State Transitions
--------------------------
Registered -> Connected =>
Everything is working as expected for first time connection on forwarder
(Endpoint is registered and ZMQ is working)
Registered -> nothing =>
Connectivity issue over the 5500* ports using ZMQ
Connected -> Disconnected =>
ZMQ network issues or endpoint is down
Disconnected -> Connected =>
Endpoint connection request received. This is either sent when a user manually
restarts a Disconnected endpoint, or when an endpoint recognizes it has lost
connection and automatically reconnects
"""
def __init__(
self,
command_queue,
response_queue,
address: str,
redis_address: str,
rabbitmq_conn_params,
endpoint_ports=(55001, 55002, 55003),
redis_port: int = 6379,
logging_level=logging.INFO,
heartbeat_period=30,
result_ttl: int = RESULT_TTL,
keys_dir=None,
):
"""
Parameters
----------
command_queue: Queue
Queue used by the service to send commands such as 'REGISTER_ENDPOINT'
Forwarder expects dicts of the form
{'command':<TERMINATE/REGISTER_ENDPOINT'> ...}
response_queue: Queue
Queue over which responses to commands are returned
address : str
Public address at which the forwarder will be accessible from the endpoints
redis_address : str
full address to connect to redis. Required
endpoint_ports : (int, int, int)
A triplet of ports: (tasks_port, results_port, commands_port)
Default: (55001, 55002, 55003)
redis_port : int
redis port. Default: 6379
logging_level: int
Logging level. Default: logging.INFO
heartbeat_period: int
heartbeat interval in seconds. Default 2s
result_ttl: int
Task TTL in REDIS after result is available. Default=1hour
keys_dir: str
Directory in which curve keys will be stored, Default: '.curve'
"""
if keys_dir is None:
keys_dir = os.path.abspath(".curve")
super().__init__()
self.command_queue = command_queue
self.response_queue = response_queue
self.address = address
self.redis_url = f"{redis_address}:{redis_port}"
self.rabbitmq_conn_params = rabbitmq_conn_params
self.tasks_port, self.results_port, self.commands_port = endpoint_ports
self.connected_endpoints: t.Dict[str, t.Dict[str, t.Any]] = {}
self.kill_event = Event()
self.heartbeat_period = heartbeat_period
self._last_heartbeat = time.time()
self.keys_dir = keys_dir
self.result_ttl = result_ttl
# TODO: drop support for imperatively configuring the redis host information
# for the forwarder. Instead, FUNCX_COMMON_REDIS_URL should be used
redis_client = default_redis_connection_factory(
f"redis://{redis_address}:{redis_port}")
self.redis_pubsub = FuncxRedisPubSub(redis_client=redis_client)
self.endpoint_db = EndpointDB(redis_client=redis_client)
self.task_storage = get_default_task_storage()
logger.info(f"Initializing forwarder v{funcx_forwarder.__version__}")
logger.info(f"Forwarder running on public address: {self.address}")
logger.info(f"REDIS url: {self.redis_url}")
logger.info(f"Log level set to {loglevels[logging_level]}")
if not os.path.exists(self.keys_dir) or not os.listdir(self.keys_dir):
logger.info(f"Keys dir empty: {self.keys_dir}, creating keys")
os.makedirs(self.keys_dir, exist_ok=True)
forwarder_keyfile, _ = zmq.auth.create_certificates(
self.keys_dir, "server")
else:
logger.info(
f"Keys in {self.keys_dir}: {os.listdir(self.keys_dir)}")
forwarder_keyfile = os.path.join(self.keys_dir, "server.key")
try:
with open(forwarder_keyfile) as f:
self.forwarder_pubkey = f.read()
except Exception:
logger.exception(
f"[CRITICAL] Failed to read server keyfile from {forwarder_keyfile}"
)
raise
@property
def redis_client(self) -> redis.Redis:
# proxy attribute pointing to the underlying pubsub's client
# this is used for now to get a client quickly and easily
#
# TODO: consider changes in funcx-common so that there's a more obvious way
# to share a connection between the Forwarder and its attached pubsub
return self.redis_pubsub.redis_client
def command_processor(self, kill_event):
"""command_processor listens on the self.command_queue
for commands and responds with results on the self.response_queue
COMMAND messages are dicts of the form:
{'command' : ['TERMINATE', 'REGISTER_ENDPOINT' ... 'ENDPOINT_LOAD_CONFIG'],
'id': <ID:int>,
'options' : ...
}
Responses are of the form:
"""
try:
while not kill_event.is_set():
command = self.command_queue.get()
logger.debug(f"[COMMAND] Received command {command}")
if command["command"] == "LIVENESS":
response = {"response": True, "id": command.get("id")}
elif command["command"] == "TERMINATE":
logger.info("[COMMAND] Received TERMINATE command")
response = {"response": True, "id": command.get("id")}
kill_event.set()
elif command["command"] == "REGISTER_ENDPOINT":
logger.info("[COMMAND] Received REGISTER_ENDPOINT command")
result = self.register_endpoint(
command["endpoint_id"],
command["endpoint_address"],
command["client_public_key"],
)
response = {
"response": result,
"id": command.get("id"),
"endpoint_id": command["endpoint_id"],
"forwarder_pubkey": self.forwarder_pubkey,
"public_ip": self.address,
"tasks_port": self.tasks_port,
"results_port": self.results_port,
"commands_port": self.commands_port,
}
else:
response = {
"response": False,
"id": command.get("id"),
"reason": "Unknown command",
}
self.response_queue.put(response)
except Exception:
logger.exception("Caught exception while processing command")
sys.exit(-1)
def register_endpoint(self, endpoint_id, endpoint_address, key):
"""Add new client keys to the zmq authenticator
Registering an existing endpoint_id is allowed
"""
logger.info(
"endpoint_registered",
extra={
"log_type": "endpoint_registered",
"endpoint_id": endpoint_id
},
)
self.update_endpoint_metadata(endpoint_id, endpoint_address)
self.tasks_q.add_client_key(endpoint_id, key)
self.results_q.add_client_key(endpoint_id, key)
self.commands_q.add_client_key(endpoint_id, key)
return True
def update_endpoint_metadata(self, endpoint_id, endpoint_address):
"""Geo locate the endpoint and push as metadata into redis"""
try:
resp = requests.get(f"http://ipinfo.io/{endpoint_address}/json")
self.endpoint_db.set_endpoint_metadata(endpoint_id, resp.json())
except Exception:
logger.error(f"Failed to geo locate {endpoint_address}")
else:
logger.info(f"Endpoint with {endpoint_address} is at {resp}")
def initialize_endpoint_queues(self):
"""Initialize the three queues over which the forwarder communicates with endpoints
TaskQueue in mode='server' binds to all interfaces by default
"""
self.tasks_q = TaskQueue(
"127.0.0.1",
port=self.tasks_port,
RCVTIMEO=1,
keys_dir=self.keys_dir,
mode="server",
)
self.results_q = TaskQueue("127.0.0.1",
port=self.results_port,
keys_dir=self.keys_dir,
mode="server")
self.commands_q = TaskQueue("127.0.0.1",
port=self.commands_port,
keys_dir=self.keys_dir,
mode="server")
return
def disconnect_endpoint(self, endpoint_id):
"""Unsubscribes from Redis pubsub and "removes" endpoint from the tasks channel.
This method does nothing if the endpoint is already disconnected.
Triggered by zmq messages not getting delivered (heartbeats, tasks, result acks)
TODO: This needs some extensive testing. It is unclear how well detecting
failures will work on WAN networks with latencies.
"""
disconnected_endpoint = self.connected_endpoints.pop(endpoint_id, None)
# if the endpoint is already disconnected, simply return
if not disconnected_endpoint:
return
logger.info(
"endpoint_disconnected",
extra={
"log_type": "endpoint_disconnected",
"endpoint_id": endpoint_id
},
)
self.redis_pubsub.unsubscribe(endpoint_id)
def add_endpoint_keys(self, ep_id, ep_key):
"""To remove. this is not used."""
self.tasks_q.add_client_key(ep_key)
self.results_q.add_client_key(ep_key)
self.commands_q.add_client_key(ep_key)
def add_subscriber(self, ep_id):
self.redis_pubsub.subscribe(ep_id)
def heartbeat(self):
"""ZMQ contexts are not thread-safe.
heartbeats should happen on the same thread."""
if self._last_heartbeat + self.heartbeat_period > time.time():
return
logger.info("Heartbeat")
dest_endpoint_list = list(self.connected_endpoints.keys())
for dest_endpoint in dest_endpoint_list:
logger.debug(
f"Sending heartbeat to {dest_endpoint}",
extra={
"log_type": "endpoint_heartbeat_sent",
"endpoint_id": dest_endpoint,
},
)
msg = Heartbeat(endpoint_id=dest_endpoint)
try:
self.tasks_q.put(dest_endpoint.encode("utf-8"), msg.pack())
self.connected_endpoints[dest_endpoint][
"missed_heartbeats"] = 0
except (zmq.error.ZMQError, zmq.Again):
logger.exception(
f"Endpoint:{dest_endpoint} is unreachable over heartbeats")
self.disconnect_endpoint(dest_endpoint)
self._last_heartbeat = time.time()
def handle_endpoint_connection(self):
"""Receive endpoint connection messages. Only connection messages
are sent from the interchange -> forwarder on the task_q
"""
try:
b_ep_id, b_reg_message = self.tasks_q.get(
timeout=0) # timeout in ms # Update to 0ms
# At this point ep_id is authenticated by means having the client keys.
ep_id = b_ep_id.decode("utf-8")
reg_message = pickle.loads(b_reg_message)
if ep_id in self.connected_endpoints:
# this is normal, it just means that the endpoint never reached
# Disconnected state before connecting again
logger.info(
f"[MAIN] Endpoint:{ep_id} attempted connect when it already is in "
"connected list",
extra={
"log_type": "endpoint_reconnected",
"endpoint_id": ep_id
},
)
else:
logger.info(
"endpoint_connected",
extra={
"log_type": "endpoint_connected",
"endpoint_id": ep_id,
"registration_message": reg_message,
},
)
# Now subscribe to messages for ep_id
# if this endpoint is already in self.connected_endpoints, it is already
# subscribed
self.add_subscriber(ep_id)
self.connected_endpoints[ep_id] = {
"registration_message": reg_message,
"missed_heartbeats": 0,
}
except zmq.Again:
pass
except Exception:
logger.exception("Caught exception while waiting for registration")
def log_task_transition(self, task, transition_name):
extra_logging = {
"user_id": task.user_id,
"task_id": task.task_id,
"task_group_id": task.task_group_id,
"function_id": task.function_id,
"endpoint_id": task.endpoint,
"container_id": task.container,
"log_type": "task_transition",
}
logger.info(transition_name, extra=extra_logging)
def forward_task_to_endpoint(self):
"""Migrates one task from redis to the appropriate endpoint
Returns:
int: Count of tasks migrated (0,1)
"""
# Now wait for any messages on REDIS that needs forwarding.
task = None
try:
dest_endpoint, task_id = self.redis_pubsub.get(timeout=0)
task = RedisTask(self.redis_client, task_id)
logger.debug(
f"Got message from REDIS: {dest_endpoint}:{task}",
extra={
"log_type": "forwarder_redis_task_get",
"endpoint_id": dest_endpoint,
},
)
except queue.Empty:
return 0
except Exception:
logger.exception("Caught exception waiting for message from REDIS")
return 0
task_payload = self.task_storage.get_payload(task)
if dest_endpoint not in self.connected_endpoints:
# At this point we should be unsubscribed and receiving only messages
# from the TCP buffers.
logger.warning(
"Putting back REDIS message for unconnected endpoint: %s:%s",
dest_endpoint,
task,
extra={
"log_type": "forwarder_redis_task_put",
"endpoint_id": dest_endpoint,
},
)
self.redis_pubsub.put(dest_endpoint, task)
self.redis_pubsub.unsubscribe(dest_endpoint)
else:
try:
task_id = task.task_id
logger.info(
f"Sending task:{task_id} to endpoint:{dest_endpoint}")
zmq_task = Task(task_id, task.container, task_payload)
except TypeError:
# A TypeError is raised when the Task object can't be recomposed from
# redis due to missing values during high-workload events.
logger.exception(f"Unable to access task {task_id} from redis")
logger.debug(
f"Task:{task_id} is now LOST",
extra={
"log_type": "task_lost",
"endpoint_id": dest_endpoint,
"task_id": task_id,
},
)
return 0
try:
self.tasks_q.put(dest_endpoint.encode("utf-8"),
zmq_task.pack())
except (zmq.error.ZMQError, zmq.Again):
logger.exception(f"Endpoint:{dest_endpoint} is unreachable")
# put task back in redis since it was not sent to endpoint
self.redis_pubsub.put(dest_endpoint, task)
self.disconnect_endpoint(dest_endpoint)
except Exception:
logger.exception(
f"Caught error while sending {task_id} to {dest_endpoint}")
# put task back in redis since it was not sent to endpoint
self.redis_pubsub.put(dest_endpoint, task)
else:
self.log_task_transition(task, "dispatched_to_endpoint")
return 1
def handle_results(self):
"""Receive incoming results on results_q and update Redis with results
NOTE: Results can arrive on this queue when the endpoint is in any of the 3
states (Registered, Connected, Disconnected), and we will accept the results.
This is because we do not tie the connection status of this queue to the state
of the endpoint, as doing so could mean rejecting perfectly good results on a
working ZMQ connection.
Registered =>
Getting results in this state means this zmq pipe has opened and results are
sent over before the connection message has been sent by the endpoint.
Connected =>
Getting results in this state is normal, as a connection message has been
sent and zmq pipes are working.
Disconnected =>
Getting results in this state means the endpoint registered and was
connected, but a zmq send failed over a different pipe, sending the endpoint
do Disconnected state. The results pipe could still be working, or it
could've started working again before the connection message is sent again
"""
try:
# timeout in ms, when 0 it's nonblocking
b_ep_id, b_message = self.results_q.get(block=False, timeout=0)
endpoint_id = b_ep_id.decode("utf-8")
if b_message == b"HEARTBEAT":
logger.debug(
f"Received heartbeat from {endpoint_id} over results channel",
extra={
"log_type": "endpoint_heartbeat_received",
"endpoint_id": endpoint_id,
},
)
return
try:
message = pickle.loads(b_message)
except Exception:
logger.exception(
f"Failed to unpickle message from results_q, message:{b_message}"
)
if isinstance(message, EPStatusReport):
logger.debug(
"endpoint_status_message",
extra={
"log_type": "endpoint_status_message",
"endpoint_id": endpoint_id,
"endpoint_status_message": message.__dict__,
},
)
# Update endpoint status
try:
self.endpoint_db.put(endpoint_id, message.ep_status)
except Exception:
logger.error(
"Caught error while trying to push endpoint status data "
"into redis")
# Update task status from endpoint
task_status_delta = message.task_statuses
for task_id, status_code in task_status_delta.items():
status = status_code_convert(status_code)
logger.debug(
f"Updating Task({task_id}) to status={status}")
task = RedisTask(self.redis_client, task_id)
task.status = status
return
if "registration" in message:
logger.debug(
f"Registration message from {message['registration']}")
return
# only messages with a result or an exception are processed past this point
result_or_exception = "result" in message or "exception" in message
if not result_or_exception:
logger.warning(
"A task result message was received without a result or an "
"exception",
extra={
"endpoint_id": endpoint_id,
"endpoint_status_message": message.__dict__,
},
)
return
task_id = message["task_id"]
if not RedisTask.exists(self.redis_client, task_id):
logger.warning(
f"Got result for task that does not exist in redis: {task_id}"
)
# if the task does not exist in redis, it may mean it was retrieved by
# the user and deleted from redis before it could be acked so the
# endpoint sent another result message. This means we should ack this
# task_id to prevent the endpoint from continuing to send it
self.handle_results_ack(endpoint_id, task_id)
return
task = RedisTask(self.redis_client, task_id)
logger.debug(f"Task info : {task}")
# handle if we get duplicate task ids (if one of the critical sections
# below did not succeed, the task could never reach an internal state
# of COMPLETE, meaning we will retry that section)
try:
if task.internal_status == InternalTaskState.COMPLETE:
logger.debug(
f"Duplicate result received for task: {task_id}")
# resend results ack in case the previous ack was not received for
# this result
self.handle_results_ack(endpoint_id, task_id)
return
except ValueError:
# A ValueError is raised if the task was wiped from redis by a
# client-fetch
# We should ack the endpoint so that it can wipe it's local cache
self.handle_results_ack(endpoint_id, task_id)
logger.warning(f"ACK requested for task:{task_id} which was "
"already fetched by client.")
return
# this critical section is where the final task redis data is set,
# and the task result will not be acked if this fails
if "result" in message:
task.status = TaskState.SUCCESS
self.task_storage.store_result(task, message["result"])
task.completion_time = time.time()
elif "exception" in message:
task.status = TaskState.FAILED
task.exception = message["exception"]
task.completion_time = time.time()
task.set_expire(self.result_ttl)
# this critical section is where the task ID is sent over RabbitMQ,
# and the task result will not be acked if this fails
task_group_id = task.task_group_id
if task_group_id:
connection = pika.BlockingConnection(self.rabbitmq_conn_params)
channel = connection.channel()
channel.exchange_declare(exchange="tasks",
exchange_type="direct")
channel.queue_declare(queue=task_group_id)
channel.queue_bind(task_group_id, "tasks")
# important: the FuncX client must be capable of receiving the same
# task_id multiple times, in case this bit succeeds but code below this
# fails and this must be retried
channel.basic_publish(exchange="tasks",
routing_key=task_group_id,
body=task_id)
logger.debug(
f"Publishing to RabbitMQ routing key {task_group_id} : {task_id}"
)
connection.close()
self.log_task_transition(task, "result_enqueued")
# internally, the task is only considered complete when both critical
# sections above have succeeded (redis data is sent and the task_id is
# sent over RabbitMQ)
task.internal_status = InternalTaskState.COMPLETE
self.handle_results_ack(endpoint_id, task_id)
except zmq.Again:
pass
except Exception:
logger.exception("Caught exception from results queue")
def handle_results_ack(self, endpoint_id, task_id):
if endpoint_id not in self.connected_endpoints:
logger.warning(
"Attempting to send results Ack to disconnected endpoint: %s",
endpoint_id,
extra={
"log_type": "disconnected_ack_attempt",
"endpoint_id": endpoint_id,
"task_id": task_id,
},
)
# TODO: remove once endpoint version 0.3.* is deprecated
reg_message = self.connected_endpoints[endpoint_id][
"registration_message"]
# if the key funcx_endpoint_version is in the registration message, it means
# the endpoint version is >=0.3.3, because 0.3.3 is the first endpoint version
# where we started sending the funcx_endpoint_version key to the forwarder
if "funcx_endpoint_version" not in reg_message:
logger.debug(
f"Ack not sent to endpoint {endpoint_id} for backwards compatability "
"because it has version <0.3.3")
return
msg = ResultsAck(task_id=task_id)
logger.debug(
f"Sending Result Ack to endpoint {endpoint_id} for task {task_id}: {msg}",
extra={
"log_type": "results_ack",
"endpoint_id": endpoint_id,
"task_id": task_id,
},
)
try:
# send an ack
self.tasks_q.put(endpoint_id.encode("utf-8"), msg.pack())
except (zmq.error.ZMQError, zmq.Again):
logger.exception(f"Endpoint:{endpoint_id} results ack send failed")
self.disconnect_endpoint(endpoint_id)
def run(self):
"""Process entry point"""
try:
logger.info("[MAIN] Loop starting")
logger.info("[MAIN] Connecting to redis")
logger.info(
f"[MAIN] Forwarder listening for tasks on: {self.tasks_port}")
logger.info(
f"[MAIN] Forwarder listening for results on: {self.results_port}"
)
logger.info(
f"[MAIN] Forwarder issuing commands on: {self.commands_port}")
self.initialize_endpoint_queues()
self._command_processor_thread = threading.Thread(
target=self.command_processor,
args=(self.kill_event, ),
name="forwarder-command-processor",
)
self._command_processor_thread.start()
while True:
if self.kill_event.is_set():
logger.critical(
"Kill event set. Starting termination sequence")
# 1. [TODO] Unsubscribe from all
# 2. [TODO] Flush all tasks received back to their queues for
# reprocessing.
# 3. [TODO] Figure out how we can trigger a scaling event to replace
# lost forwarder?
# Send heartbeats to every connected manager
self.heartbeat()
self.handle_endpoint_connection()
# [TODO] This step could be in a timed loop. Ideally after we have a
# perf study
self.forward_task_to_endpoint()
self.handle_results()
except Exception:
logger.exception("Caught exception while running forwarder")
sys.exit(-1)
