def response(context, flow):
"""========================================================================
"Called when a server response has been received"... łapię wyłącznie
odpowiedzi, bo interesują mnie zestawy (request/response). Przechwycony
response wraz z requestem wchodzą w skład transakcji, reprezentowanej przez
mitmproxy.models.HTTPFlow()
"HTTPFlow is collection of objects representing a single HTTP transaction".
Więcej info na WWW: http://docs.mitmproxy.org/en/stable/dev/models.html
==========================================================================="""
if flow.request.host.endswith('.thesettlersonline.pl'):
if "application/x-amf" in flow.response.headers.get("Content-Type", "_"):
with decoded(flow.response):
res = flow.response.content
req = flow.request.content
if search( 'defaultGame.Communication.VO.TradeWindow.dTradeWindowResultVO', res )\
and search( 'userAcceptedTradeIDs', res ) and search( 'tradeOffers', res )\
and search( 'GetAvailableOffers', req ):
log.debug("got trade REQ/RESP pair, feeding TDD thread...")
try:
t= Thread(target=ttd._incoming_traffic_handler, args=(context, flow,))
t.setDaemon(True)
t.start()
except (KeyboardInterrupt, SystemExit):
log.info('caught either KeyboardInterrupt or SystemExit, quitting threads')
t.__stop()
import thread
thread.interrupt_main()
def parse_itemDetailView(self, response):
if 'techerror' in response.url:
print 'somehow found tech error at url ', response.request.url
# logfun = logging.getLogger("logfun")
# time.sleep(0.10)
try:
hxs = HtmlXPathSelector(response)
page_title = hxs.select('//title/text()').extract()[0]
med_image = hxs.select('//input[@id="imageURL"]/@value')[0].extract() # 200x200 image
# Should probably do some form of exception, but lazy
large_image = ''
large_image_path = hxs.select('//input[@id="primaryImageURL"]/@value')
if len(large_image_path) > 0:
large_image = large_image_path[0].extract()
# large_image = hxs.select('//input[@id="primaryImageURL"]/@value')[0].extract() # 600x600 image
# Only take information in paragraphs, but preserve HTML & formatting: Combine all paragraphs for this entry
details_list = hxs.select('//div[@id="productDetails-details"]/p').extract()
details = ''.join(details_list)
cut_start = details.find('<a class="')
cut_end = details.rfind('</p>')
# Remove the disclaimer link at bottom
if cut_start > 0 and cut_end > 0:
details = details[0:cut_start] + details[cut_end:]
nutrition_path = hxs.select('//div[@id="productDetails-nutrition"]/table').extract()
nutrition = ''
if len(nutrition_path) > 0:
nutrition = nutrition_path[0]
ingredients = ''
if len(hxs.select('//div[@id="ingredients"]').extract()) > 0:
ingredients = hxs.select('//div[@id="ingredients"]')[0].extract()
#ingredients = hxs.select('//div[@id="ingredients"]')[0].extract()
detailed_item = DetailedShopItem(name = response.meta['name'], size =
response.meta['size'], unit_price = response.meta['unit_price'],
productId = response.meta['productId'], price =
response.meta['price'], cnid = response.meta['cnid'], small_image =
response.meta['thumb'], med_image = med_image, large_image =
large_image, details = details, nutrition = nutrition, ingredients = ingredients)
self.all_items_detail.append(detailed_item)
# except:
# e = sys.exc_info()[0]
# print e.print_stack()
# sys.exit("error has occurred")
except Exception, ex:
print 'gg exception'
print 'came from', response.request.url
raise
#raise KeyboardInterrupt
# logfun.exception("Bad gg")
# logfun.debug("figure this out")
thread.interrupt_main()
def worker_thread(work_queue,shutdown_flag):
while not shutdown_flag.is_set():
fileno, address, head, body = work_queue.get_request()
if fileno == 0:
thread.interrupt_main()
return
parser = Parser(head,body)
request = parser.parse()
if request == None:
response = Http400Response()
else:
request.remote_addr, request.remote_port = address
request._work_queue = work_queue
log("%s - %s" %( request.remote_addr, request.path ))
handler = config.HANDLER_CLASS(request)
try:
response = handler.response()
except:
exc = traceback.format_exc()
log(exc)
if config.DEBUG:
response = Http500Response(exc)
else:
response = Http500Response()
work_queue.submit_response(fileno,response)
def _uoczer(self):
""" "ten co czai" :) :: perform once-upon-a-time checks and cleanups ::
- sprawdza czas ostatnich odświerzeń rynków każdego z realmów
- dla przestarzałych emituje duplikaty, jeśli są dostępne
- planuje kolejne uruchomienie siebie przy pomocy threading.Timer
"""
self.uoczerIterCounter+=1
try: # performing timed checks and cleanups
if self.now()-self.c.lastupdatedTS > 3000 and self.uoczerIterCounter % 1000 == 0:
log.info('PINGING DATABASE')
self.db.ping()
if self._trade_refresh_needed() and self.c.treqs:
recent_key = self.c.treqs.keys()[0]
recent_request= self.c.treqs[recent_key]
raw_trade_response= recent_request.execute()
if raw_trade_response:
parsed_response_body= self._amf_raw_response_parser(raw_trade_response)
self._trade_update_market(parsed_response_body)
log.info('trade updated with duplicated treq (uID: %d)'%recent_key)
else:
log.info('trade requery failed, deleting treq (uID %d)'%recent_key)
del self.c.treqs[recent_key]
except (KeyboardInterrupt, SystemExit): # stop on interrupts
log.info('caught either KeyboardInterrupt or SystemExit, quitting thread')
import thread
thread.interrupt_main()
exit()
finally:
log.debug('scheduling run in %ds'%self.c.watchT)
self.uoczertimer = Timer(self.c.watchT, self._uoczer) # reschedule run
self.uoczertimer.setDaemon(True) # with abort at shutdown
self.uoczertimer.start()
def process_stream(self):
"""Process lines using parseline() forever"""
# Catch-all so we know why the process died.
while True:
# Read the next line
try:
line = self.readline()
print line
except KeyboardInterrupt:
thread.interrupt_main()
except:
logging.error("%s failed with %s"
% (self.instrument, traceback.format_exc()))
continue
# Interpret the next line
try:
self.lock.acquire()
self.parseline(line)
except KeyboardInterrupt:
thread.interrupt_main()
except:
logging.error("%s failed with %s\nLine: %s"
% (self.instrument,
traceback.format_exc(),
self.lastline))
finally:
self.lock.release()
def do_GET(self):
"""Serve a GET request."""
parsed_url = urlparse.urlparse(self.path)
params = urlparse.parse_qs(parsed_url.query)
path = parsed_url.path
if path.startswith('/stop'):
# stop children before we return a response
timeout = int((params.get("timeout", []) or ["30"])[0])
self.funkyserver.pre_stop(timeout=timeout)
t = self.get_response_text()
if self.send_head(t):
self.wfile.write(t)
#I preempt the finish operation here so that processing of this request is all done before we crash or whatever:
self.finish()
self.http_server.shutdown_request(self.connection)
if path.startswith('/crash'):
crash()
if path.startswith('/stop'):
self.funkyserver.stop()
if path.startswith('/interrupt_main'):
thread.interrupt_main()
if path.startswith('/exit'):
os._exit(1)
if path.startswith('/hold_gil'):
t = int((params.get("t", []) or ["100"])[0])
hold_gil(t)
def __init__(self, host, port, username, password):
self.origin = Location()
self.origin.set(130, 71, 83)
self.range = Location()
self.range.set(10, 10, 10)
self.X = 0
self.Y = 0
self.Z = 0
self.bot = Bot()
self.dispatch = PacketDispatch(self)
self.buff = Buffer()
self.packetSend = []
self.sendPacket = PacketSend(self)
self.printable = True
self.receiveSize = 1024
self.compressionThreshold = -1
self.username = username
self.HOST = host
self.PORT = port
self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
self.s.settimeout(None)
try:
self.s.connect((self.HOST, self.PORT))
except socket.error:
print("Connection refused")
thread.interrupt_main()
exit()
def _handle_test_status_file(self, test_name, test_phase, success):
###########################################################################
#
# This complexity is due to sharing of TestStatus responsibilities
#
try:
if (test_phase != RUN_PHASE and
(not success or test_phase == BUILD_PHASE or test_phase == self._phases[-1])):
self._update_test_status_file(test_name)
# If we failed VERY early on in the run phase, it's possible that
# the CIME scripts never got a chance to set the state.
elif (test_phase == RUN_PHASE and not success):
test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME)
statuses = wait_for_tests.parse_test_status_file(test_status_file)[0]
if ( RUN_PHASE not in statuses or
statuses[RUN_PHASE] in [TEST_PASS_STATUS, TEST_PENDING_STATUS] ):
self._update_test_status_file(test_name)
except Exception as e:
# TODO: What to do here? This failure is very severe because the
# only way for test results to be communicated is by the TestStatus
# file.
warning("VERY BAD! Could not handle TestStatus file '%s': '%s'" % (test_status_file, str(e)))
thread.interrupt_main()
def handle_error(self, request, client_address):
"""Override RPCServer method for IDLE
Interrupt the MainThread and exit server if link is dropped.
"""
global quitting
try:
raise
except SystemExit:
raise
except EOFError:
global exit_now
exit_now = True
thread.interrupt_main()
except:
erf = sys.__stderr__
print>>erf, '\n' + '-'*40
print>>erf, 'Unhandled server exception!'
print>>erf, 'Thread: %s' % threading.currentThread().getName()
print>>erf, 'Client Address: ', client_address
print>>erf, 'Request: ', repr(request)
traceback.print_exc(file=erf)
print>>erf, '\n*** Unrecoverable, server exiting!'
print>>erf, '-'*40
quitting = True
thread.interrupt_main()
def check_active_loggers(self):
"""Check if all loggers are deactivated due to I/O errors."""
for logger in self.loggers:
if logger.is_active:
break
else:
thread.interrupt_main()
def f():
for x in range(5):
if waiting:
thread.interrupt_main()
return
print 'tock...', x # <-force the GIL to be released, as
time.sleep(0.1) # time.sleep doesn't do non-translated
def HTTPreq(self,url,data=None,headers=None):
opener=self.opener
kargs={}
kargs['url']=url
if data: kargs['data']=data #Will do a GET if no data else POST
if headers: kargs['headers']=headers
else: kargs['headers']={'Content-Type':'application/octet-stream'}
#Make Request
f=opener.open(urllib2.Request(**kargs))
#If response is gzip encoded
if ('Content-Encoding' in f.info().keys() and f.info()['Content-Encoding']=='gzip') or \
('content-encoding' in f.info().keys() and f.info()['content-encoding']=='gzip'):
url_f = StringIO.StringIO(f.read())
data = gzip.GzipFile(fileobj=url_f).read()
else: #response not encoded
data = f.read()
if f.getcode() != 200:
print "[-] Received status code " + str(f.getcode())
print data
thread.interrupt_main()
return data #Return response
def _test_main_thread():
cherrypy.server.wait()
cherrypy._cputil._cpInitDefaultFilters()
try:
webtest.main()
finally:
thread.interrupt_main()
def _serve_forever(self):
LOG.debug('Starting plotter')
try:
cherrypy.quickstart(self, '/', {'/': {}})
except:
LOG.error(helper.exc_info())
thread.interrupt_main()
def callback():
global thread, s, peersData, my_ip, host, port
if tkMessageBox.askokcancel("Quit", "Do you really wish to quit?"):
s.sendto("exit", (host, port))
s.close()
root.destroy()
thread.interrupt_main()
def run(self):
try:
super(DDPClient, self).run()
except:
traceback.print_exc()
finally:
thread.interrupt_main()
def _process_queue(queue):
time.sleep(2)
while True:
# Block until we have something to plot
#logging.debug("get next plot")
ready = [queue.get()]
# Get all queued requests, deleting the duplicates
try:
while True:
# TODO: don't know how to make this interruptible
#logging.debug("empty queue")
next = queue.get_nowait()
queue.task_done()
if next not in ready: ready.append(next)
except KeyboardInterrupt:
thread.interrupt_main()
except Queue.Empty:
pass
# Make all the plots
#logging.debug("plotting %d"%len(ready))
for stream in ready:
logging.debug("plotting "+stream.instrument)
stream.plot()
# Be nice to the server --- batch plot requests every 30 seconds
#logging.debug("sleep %s"%str(PLOT_INTERVAL))
time.sleep(PLOT_INTERVAL)
def worker():
try:
workerRun()
except Exception, e:
print sys.exc_info()
M6Item.status = 'STOPPED'
thread.interrupt_main()
def stop_code(self, quiet=False):
"""
Attempt to stop the running code by raising a keyboard interrupt in
the main thread.
If the optional quiet flag is True the engine will stop the code but not
print an error message or prompt again.
"""
if self.busy is False:
return
#set the stop flag to catch the error correctly
if quiet:
self._stop_quiet = True
else:
self._stop = True
#make sure we are not stuck in readline(s)
if self._isreading:
self._readevent.set()
#make sure the debugger is not paused.
if self.debug is True:
self.debugger.stop_code()
return
#try a keyboard interrupt - this will not work for the internal engine
# as the error is raised here instead of the running code, hence put in
#try clause.
try:
thread.interrupt_main()
except:
pass
def run(self):
"""Start event listener"""
try_interval = 1
while self.stop_event.is_set() is False:
try:
try_interval *= 2
with self._celery_app.connection() as conn:
handlers = {
"task-succeeded": (self._engine._task_success_handler),
"task-failed": (self._engine._task_failure_handler),
}
recv = events.EventReceiver(conn, handlers=handlers, app=self._celery_app)
while self.stop_event.is_set() is False:
recv.capture(limit=None, timeout=None)
self._running = True
try_interval = 1
except (KeyboardInterrupt, SystemExit):
thread.interrupt_main()
except Exception as e:
self._running = False
LOG.error("Failed to capture events: '%s', " "trying again in %s seconds." % (e, try_interval))
LOG.debug(e, exc_info=True)
time.sleep(try_interval)
def decompte(self, sec=90., dt=.1): endTime = time.time() + sec while True: novTime = time.time() if novTime >= endTime: break if self.tirette.armee(): self.printLog( "sortie du programme sur remise de la tirette apres %.2fs"%(sec-(endTime-novTime)) ) import sys sys.exit(1) #raise KeyboardInterrupt(message) time.sleep(dt) self.disableActuators() self.disableActuators() self.disablePower() self.disablePower() time.sleep(0.5) self.disableActuators() self.disableActuators() self.disablePower() self.disablePower() self.printLog( "fin du temps imparti: %.2fs"%sec ) thread.interrupt_main()
def listen(new_socket, filename, extension, part, size):
#listen on the random port
new_socket.listen(1)
#accept incoming connections
connection, address = new_socket.accept()
full_filename = filename + '.' + extension
#get username
username = os.getlogin()
#fully qualify path
file_path = '/home/' + username + '/p2p/files/' + full_filename
#open the file for reading
FILE = open(file_path, 'rb')
#read the desired parts
FILE.seek(part * 512000)
#set up data buffer for sending data
send_data = FILE.read(size)
#close up file
FILE.close()
#send out your data
connection.send(send_data)
#close up connection
connection.close()
#close up
new_socket.close()
#stop thread
thread.interrupt_main()
def update_config(self):
if self._restart:
print >>sys.stderr, "[hosted.py] restarting service (restart_on_update set)"
import thread, time
thread.interrupt_main()
time.sleep(100)
return
def parse_recursive(options, config, target):
# print 'parsing', config
for option in options:
if not 'name' in option:
continue
if option['type'] == 'list':
items = []
for item in config[option['name']]:
parsed = {}
parse_recursive(option['items'], item, parsed)
items.append(parsed)
target[option['name']] = items
continue
target[option['name']] = types[option['type']](config[option['name']])
parsed = {}
parse_recursive(self._options, self._config, parsed)
print >>sys.stderr, "[hosted.py] updated config"
self._parsed = parsed
def Pinging_Thread(self): print "[+] Starting Ping thread" #self.ptc=threading.Condition() wait=True p=0.1 while 1: #loop forever if wait and (self.ping_delay > 0): self.ptc.acquire() self.ptc.wait(self.ping_delay+self.penalty) #send ping to server interval + penalty self.ptc.release() self.mutex_http_req.acquire() #Ensure that the other thread is not making a request at this time try: resp_data=self.http.HTTPreq(self.url,"") #Read response if self.verbose: self.http.v_print(pings_n=1) if self.penalty<60: self.penalty+=p #Don't wait more than a minute if resp_data: #If response had data write them to socket self.penalty=0 if self.verbose: self.http.v_print(received_d_pt=len(resp_data)) self.TunnaSocket.send(resp_data) #write to socket resp_data="" #clear data wait=False #If data received don't wait else: wait=True except: self.TunnaSocket.close() thread.exit() finally: self.mutex_http_req.release() print "[-] Pinging Thread Exited" #Unrecoverable thread.interrupt_main() #Interupt main thread -> exits
def receive_service(self):
# Receive data from server.
while True:
try:
# Recieve Message.
line = self.network_client_socket.recv(2048)
if (line != '\n'):
if((len(line.split(":")) >= 2) and
(line.split(":")[0].strip().startswith("LINE"))):
self.gui_send_service(line)
elif((len(line.split(":")) == 2) and
(line.split(":")[0].strip() == "JOINED")):
self.create_group_gui(line.split(":")[1].strip())
else:
print("\n %s" % line)
if not line:
print("Server closed connection, thread exiting.")
thread.interrupt_main()
self.quit()
break
except:
print("Recieve Error: Thread exiting.")
thread.interrupt_main()
self.quit()
break
def run(self):
"""
Do not call this directly; call ``start()`` instead.
"""
while not self.halt:
# print 'Heartbeat check...'
self.lock_file.seek(0)
self.lock_file.write(str(time.time()))
self.lock_file.flush()
# Check job status and save heartbeat timestamp.
self.lock.acquire()
Job.objects.update()
job = Job.objects.get(id=self.job_id)
job.last_heartbeat = datetime.now()
force_stop = job.force_stop
job.force_stop = False
job.force_run = False
job.save()
self.lock.release()
# If we noticed we're being forced to stop, then interrupt main.
if force_stop:
self.halt = True
thread.interrupt_main()
return
time.sleep(10)
set_current_heartbeat(None)
def wait(secs):
for n in xrange(timeout_secs):
time.sleep(1)
if signal_finished:
break
else:
thread.interrupt_main()
def receiverThread(cjdns):
timeOfLastSend = time.time();
timeOfLastRecv = time.time();
try:
while True:
if (timeOfLastSend + KEEPALIVE_INTERVAL_SECONDS < time.time()):
if (timeOfLastRecv + 10 < time.time()):
raise Exception("ping timeout");
cjdns.socket.send('d1:q18:Admin_asyncEnabled4:txid8:keepalive');
timeOfLastSend = time.time();
try:
data = cjdns.socket.recv(BUFFER_SIZE);
except (socket.timeout): continue;
try:
benc = bdecode(data);
except (KeyError, ValueError):
print("error decoding [" + data + "]");
continue;
if benc['txid'] == 'keepaliv':
if benc['asyncEnabled'] == 0:
raise Exception("lost session");
timeOfLastRecv = time.time();
else:
#print "putting to queue " + str(benc);
cjdns.queue.put(benc);
except KeyboardInterrupt:
print("interrupted");
import thread
thread.interrupt_main();
def run(self):
""" Run the poll loop. This method never returns.
"""
from _subprocess import WAIT_OBJECT_0, INFINITE
# Build the list of handle to listen on.
handles = []
if self.interrupt_handle:
handles.append(self.interrupt_handle)
if self.parent_handle:
handles.append(self.parent_handle)
# Listen forever.
while True:
result = ctypes.windll.kernel32.WaitForMultipleObjects(
len(handles), # nCount
(ctypes.c_int * len(handles))(*handles), # lpHandles
False, # bWaitAll
INFINITE) # dwMilliseconds
if WAIT_OBJECT_0 <= result < len(handles):
handle = handles[result - WAIT_OBJECT_0]
if handle == self.interrupt_handle:
raw_print('Interrupted by parent poller!')
interrupt_main()
elif handle == self.parent_handle:
raw_print('Killed by parent poller!')
os._exit(1)
def cleanup(self, state):
try:
state.app.session.stop()
finally:
super(MainProcess, self).cleanup(state)
# signal the main thread that MainProcess exited
thread.interrupt_main()
def decode_interrupthook(self):
"interrupt awakened thread"
global quitting
quitting = True
thread.interrupt_main()
def EOFhook(self):
"Override SocketIO method - terminate wait on callback and exit thread"
global quitting
quitting = True
thread.interrupt_main()
def run(self):
"""Starts the process when Process.start() is called.
"""
global JOB_CHECK_INTERVAL
# make sure to catch sys.exit (which raises SystemExit)
try:
# Get directory where this module lives
mod_dir = os.path.dirname(os.path.realpath(__file__))
# Try to connect to the database
try:
db = self._session.get_db()
pilot_col = db["%s.p" % self._session.uid]
logger.debug(
"Connected to MongoDB. Serving requests for PilotManager %s."
% self.pilot_manager_id)
except Exception as e:
logger.exception("Connection error: %s" % e)
return
last_job_check = time.time()
while not self._terminate.is_set():
# Periodically, we pull up all ComputePilots that are pending
# execution or were last seen executing and check if the corresponding
# SAGA job is still pending in the queue. If that is not the case,
# we assume that the job has failed for some reasons and update
# the state of the ComputePilot accordingly.
if last_job_check + JOB_CHECK_INTERVAL < time.time():
last_job_check = time.time()
self.check_pilot_states(pilot_col)
if self._disabled.is_set():
# don't process any new pilot start requests.
# NOTE: this is not clean, in principle there could be other
# launchers alive which want to still start those
# pending pilots. In practice we only ever use one
# pmgr though, and its during its shutdown that we get
# here...
ts = timestamp()
compute_pilot = pilot_col.find_and_modify(
query={
"pilotmanager": self.pilot_manager_id,
"state": PENDING_LAUNCH
},
update={
"$set": {
"state": CANCELED
},
"$push": {
"statehistory": {
"state": CANCELED,
"timestamp": ts
}
}
})
# run state checks more frequently.
JOB_CHECK_INTERVAL = 3
time.sleep(1)
continue
# See if we can find a ComputePilot that is waiting to be launched.
# If we find one, we use SAGA to create a job service, a job
# description and a job that is then send to the local or remote
# queueing system. If this succedes, we set the ComputePilot's
# state to pending, otherwise to failed.
compute_pilot = None
ts = timestamp()
compute_pilot = pilot_col.find_and_modify(
query={
"pilotmanager": self.pilot_manager_id,
"state": PENDING_LAUNCH
},
update={
"$set": {
"state": LAUNCHING
},
"$push": {
"statehistory": {
"state": LAUNCHING,
"timestamp": ts
}
}
})
if not compute_pilot:
time.sleep(IDLE_TIMER)
else:
try:
# ------------------------------------------------------
#
# LAUNCH THE PILOT AGENT VIA SAGA
#
logentries = []
pilot_id = str(compute_pilot["_id"])
logger.info("Launching ComputePilot %s" % pilot_id)
# ------------------------------------------------------
# Database connection parameters
session_id = self._session.uid
database_url = self._session.dburl
# ------------------------------------------------------
# pilot description and resource configuration
number_cores = compute_pilot['description']['cores']
runtime = compute_pilot['description']['runtime']
queue = compute_pilot['description']['queue']
project = compute_pilot['description']['project']
cleanup = compute_pilot['description']['cleanup']
resource_key = compute_pilot['description']['resource']
schema = compute_pilot['description']['access_schema']
memory = compute_pilot['description']['memory']
pilot_sandbox = compute_pilot['sandbox']
global_sandbox = compute_pilot['global_sandbox']
# we expand and exchange keys in the resource config,
# depending on the selected schema so better use a deep
# copy..
resource_cfg = self._session.get_resource_config(
resource_key, schema)
# import pprint
# pprint.pprint (resource_cfg)
# ------------------------------------------------------
# get parameters from cfg, set defaults where needed
agent_launch_method = resource_cfg.get(
'agent_launch_method')
agent_dburl = resource_cfg.get(
'agent_mongodb_endpoint', database_url)
agent_spawner = resource_cfg.get(
'agent_spawner', DEFAULT_AGENT_SPAWNER)
agent_type = resource_cfg.get('agent_type',
DEFAULT_AGENT_TYPE)
rc_agent_config = resource_cfg.get(
'agent_config', DEFAULT_AGENT_CONFIG)
agent_scheduler = resource_cfg.get('agent_scheduler')
tunnel_bind_device = resource_cfg.get(
'tunnel_bind_device')
default_queue = resource_cfg.get('default_queue')
forward_tunnel_endpoint = resource_cfg.get(
'forward_tunnel_endpoint')
js_endpoint = resource_cfg.get('job_manager_endpoint')
lrms = resource_cfg.get('lrms')
mpi_launch_method = resource_cfg.get(
'mpi_launch_method')
pre_bootstrap_1 = resource_cfg.get('pre_bootstrap_1')
pre_bootstrap_2 = resource_cfg.get('pre_bootstrap_2')
python_interpreter = resource_cfg.get(
'python_interpreter')
spmd_variation = resource_cfg.get('spmd_variation')
task_launch_method = resource_cfg.get(
'task_launch_method')
rp_version = resource_cfg.get('rp_version',
DEFAULT_RP_VERSION)
virtenv_mode = resource_cfg.get(
'virtenv_mode', DEFAULT_VIRTENV_MODE)
virtenv = resource_cfg.get('virtenv', DEFAULT_VIRTENV)
stage_cacerts = resource_cfg.get(
'stage_cacerts', 'False')
cores_per_node = resource_cfg.get('cores_per_node')
# Agent configuration that is not part of the public API.
# The agent config can either be a config dict, or
# a string pointing to a configuration name. If neither
# is given, check if 'RADICAL_PILOT_AGENT_CONFIG' is
# set. The last fallback is 'agent_default'
agent_config = compute_pilot['description'].get(
'_config')
if not agent_config:
agent_config = os.environ.get(
'RADICAL_PILOT_AGENT_CONFIG')
if not agent_config:
agent_config = rc_agent_config
if isinstance(agent_config, dict):
# nothing to do
agent_cfg_dict = agent_config
pass
elif isinstance(agent_config, basestring):
try:
if os.path.exists(agent_config):
# try to open as file name
logger.info("Read agent config file: %s" %
agent_config)
agent_cfg_dict = ru.read_json(agent_config)
else:
# otherwise interpret as a config name
# FIXME: load in session just like resource
# configs, including user level overloads
module_path = os.path.dirname(
os.path.abspath(__file__))
config_path = "%s/../configs/" % module_path
agent_cfg_file = os.path.join(
config_path,
"agent_%s.json" % agent_config)
logger.info("Read agent config file: %s" %
agent_cfg_file)
agent_cfg_dict = ru.read_json(
agent_cfg_file)
except Exception as e:
logger.exception(
"Error reading agent config file: %s" % e)
raise
else:
# we can't handle this type
raise TypeError(
'agent config must be string (filename) or dict'
)
# TODO: use booleans all the way?
if stage_cacerts.lower() == 'true':
stage_cacerts = True
else:
stage_cacerts = False
# expand variables in virtenv string
virtenv = virtenv % {
'pilot_sandbox': saga.Url(pilot_sandbox).path,
'global_sandbox': saga.Url(global_sandbox).path
}
# Check for deprecated global_virtenv
global_virtenv = resource_cfg.get('global_virtenv')
if global_virtenv:
logger.warn(
"'global_virtenv' keyword is deprecated -- use 'virtenv' and 'virtenv_mode'"
)
virtenv = global_virtenv
virtenv_mode = 'use'
# Create a host:port string for use by the bootstrap_1.
db_url = saga.Url(agent_dburl)
if db_url.port:
db_hostport = "%s:%d" % (db_url.host, db_url.port)
else:
db_hostport = "%s:%d" % (db_url.host, 27017
) # mongodb default
# Open the remote sandbox
sandbox_tgt = saga.filesystem.Directory(
pilot_sandbox,
session=self._session,
flags=saga.filesystem.CREATE_PARENTS)
BOOTSTRAPPER_SCRIPT = "bootstrap_1.sh"
LOCAL_SCHEME = 'file'
# ------------------------------------------------------
# Copy the bootstrap shell script. This also creates
# the sandbox. We use always "default_bootstrapper.sh"
# TODO: Is this still configurable and/or in the resource configs?
bootstrapper = "default_bootstrapper.sh"
bootstrapper_path = os.path.abspath("%s/../bootstrapper/%s" \
% (mod_dir, bootstrapper))
msg = "Using bootstrapper %s" % bootstrapper_path
logentries.append(Logentry(msg, logger=logger.info))
bs_script_url = saga.Url(
"%s://localhost%s" %
(LOCAL_SCHEME, bootstrapper_path))
msg = "Copying bootstrapper '%s' to agent sandbox (%s)." \
% (bs_script_url, sandbox_tgt)
logentries.append(Logentry(msg, logger=logger.debug))
sandbox_tgt.copy(bs_script_url, BOOTSTRAPPER_SCRIPT)
# ------------------------------------------------------
# the version of the agent is derived from
# rp_version, which has the following format
# and interpretation:
#
# case rp_version:
# @<token>:
# @tag/@branch/@commit: # no sdist staging
# git clone $github_base radical.pilot.src
# (cd radical.pilot.src && git checkout token)
# pip install -t $VIRTENV/rp_install/ radical.pilot.src
# rm -rf radical.pilot.src
# export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH
#
# release: # no sdist staging
# pip install -t $VIRTENV/rp_install radical.pilot
# export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH
#
# local: # needs sdist staging
# tar zxf $sdist.tgz
# pip install -t $VIRTENV/rp_install $sdist/
# export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH
#
# debug: # needs sdist staging
# tar zxf $sdist.tgz
# pip install -t $SANDBOX/rp_install $sdist/
# export PYTHONPATH=$SANDBOX/rp_install:$PYTHONPATH
#
# installed: # no sdist staging
# true
# esac
#
# virtenv_mode
# private : error if ve exists, otherwise create, then use
# update : update if ve exists, otherwise create, then use
# create : use if ve exists, otherwise create, then use
# use : use if ve exists, otherwise error, then exit
# recreate: delete if ve exists, otherwise create, then use
#
# examples :
# [email protected]
# virtenv@devel
# virtenv@release
# virtenv@installed
# stage@local
# stage@/tmp/my_agent.py
#
# Note that some combinations may be invalid,
# specifically in the context of virtenv_mode. If, for
# example, virtenv_mode is 'use', then the 'virtenv:tag'
# will not make sense, as the virtenv is not updated.
# In those cases, the virtenv_mode is honored, and
# a warning is printed.
#
# Also, the 'stage' mode can only be combined with the
# 'local' source, or with a path to the agent (relative
# to mod_dir, or absolute).
#
# A rp_version which does not adhere to the
# above syntax is ignored, and the fallback stage@local
# is used.
if not rp_version.startswith('@') and \
not rp_version in ['installed', 'local', 'debug']:
raise ValueError("invalid rp_version '%s'" %
rp_version)
stage_sdist = True
if rp_version in ['installed', 'release']:
stage_sdist = False
if rp_version.startswith('@'):
stage_sdist = False
rp_version = rp_version[1:] # strip '@'
# ------------------------------------------------------
# Copy the rp sdist if needed. We actually also stage
# the sdists for radical.utils and radical.saga, so that
# we have the complete stack to install...
if stage_sdist:
for sdist_path in [
ru.sdist_path, saga.sdist_path,
rp_sdist_path
]:
sdist_url = saga.Url(
"%s://localhost%s" %
(LOCAL_SCHEME, sdist_path))
msg = "Copying sdist '%s' to sandbox (%s)." % (
sdist_url, pilot_sandbox)
logentries.append(
Logentry(msg, logger=logger.debug))
sandbox_tgt.copy(
sdist_url,
os.path.basename(str(sdist_url)))
# ------------------------------------------------------
# Some machines cannot run pip due to outdated CA certs.
# For those, we also stage an updated certificate bundle
if stage_cacerts:
cc_path = os.path.abspath("%s/../bootstrapper/%s" \
% (mod_dir, 'cacert.pem.gz'))
cc_url = saga.Url("%s://localhost/%s" %
(LOCAL_SCHEME, cc_path))
msg = "Copying CA certificate bundle '%s' to sandbox (%s)." % (
cc_url, pilot_sandbox)
logentries.append(
Logentry(msg, logger=logger.debug))
sandbox_tgt.copy(cc_url,
os.path.basename(str(cc_url)))
# ------------------------------------------------------
# sanity checks
if not agent_spawner:
raise RuntimeError("missing agent spawner")
if not agent_scheduler:
raise RuntimeError("missing agent scheduler")
if not lrms: raise RuntimeError("missing LRMS")
if not agent_launch_method:
raise RuntimeError("missing agentlaunch method")
if not task_launch_method:
raise RuntimeError("missing task launch method")
if not mpi_launch_method:
raise RuntimeError("missing mpi launch method")
# massage some values
if not queue:
queue = default_queue
if cleanup and isinstance(cleanup, bool):
cleanup = 'luve' # l : log files
# u : unit work dirs
# v : virtualenv
# e : everything (== pilot sandbox)
#
# we never cleanup virtenvs which are not private
if virtenv_mode is not 'private':
cleanup = cleanup.replace('v', '')
sdists = ':'.join(
[ru.sdist_name, saga.sdist_name, rp_sdist_name])
# if cores_per_node is set (!= None), then we need to
# allocation full nodes, and thus round up
if cores_per_node:
cores_per_node = int(cores_per_node)
number_cores = int(cores_per_node * math.ceil(
float(number_cores) / cores_per_node))
# set mandatory args
bootstrap_args = ""
bootstrap_args += " -d '%s'" % sdists
bootstrap_args += " -m '%s'" % virtenv_mode
bootstrap_args += " -p '%s'" % pilot_id
bootstrap_args += " -r '%s'" % rp_version
bootstrap_args += " -s '%s'" % session_id
bootstrap_args += " -v '%s'" % virtenv
# set optional args
if agent_type:
bootstrap_args += " -a '%s'" % agent_type
if lrms == "CCM": bootstrap_args += " -c"
if pre_bootstrap_1:
bootstrap_args += " -e '%s'" % "' -e '".join(
pre_bootstrap_1)
if pre_bootstrap_2:
bootstrap_args += " -w '%s'" % "' -w '".join(
pre_bootstrap_2)
if forward_tunnel_endpoint:
bootstrap_args += " -f '%s'" % forward_tunnel_endpoint
if forward_tunnel_endpoint:
bootstrap_args += " -h '%s'" % db_hostport
if python_interpreter:
bootstrap_args += " -i '%s'" % python_interpreter
if tunnel_bind_device:
bootstrap_args += " -t '%s'" % tunnel_bind_device
if cleanup: bootstrap_args += " -x '%s'" % cleanup
# set some agent configuration
agent_cfg_dict['cores'] = number_cores
agent_cfg_dict['debug'] = os.environ.get(
'RADICAL_PILOT_AGENT_VERBOSE',
logger.getEffectiveLevel())
agent_cfg_dict['mongodb_url'] = str(agent_dburl)
agent_cfg_dict['lrms'] = lrms
agent_cfg_dict['spawner'] = agent_spawner
agent_cfg_dict['scheduler'] = agent_scheduler
agent_cfg_dict['runtime'] = runtime
agent_cfg_dict['pilot_id'] = pilot_id
agent_cfg_dict['session_id'] = session_id
agent_cfg_dict[
'agent_launch_method'] = agent_launch_method
agent_cfg_dict[
'task_launch_method'] = task_launch_method
agent_cfg_dict['mpi_launch_method'] = mpi_launch_method
if cores_per_node:
agent_cfg_dict['cores_per_node'] = cores_per_node
# ------------------------------------------------------
# Write agent config dict to a json file in pilot sandbox.
cfg_tmp_handle, cf_tmp_file = tempfile.mkstemp(
suffix='.json', prefix='rp_agent_cfg_')
# Convert dict to json file
msg = "Writing agent configuration to file '%s'." % cf_tmp_file
logentries.append(Logentry(msg, logger=logger.debug))
ru.write_json(agent_cfg_dict, cf_tmp_file)
cf_url = saga.Url("%s://localhost%s" %
(LOCAL_SCHEME, cf_tmp_file))
msg = "Copying agent configuration file '%s' to sandbox (%s)." % (
cf_url, pilot_sandbox)
logentries.append(Logentry(msg, logger=logger.debug))
sandbox_tgt.copy(cf_url, 'agent_0.cfg')
# close and remove temp file
os.close(cfg_tmp_handle)
os.unlink(cf_tmp_file)
# ------------------------------------------------------
# Done with all transfers to pilot sandbox, close handle
sandbox_tgt.close()
# ------------------------------------------------------
# now that the scripts are in place and configured,
# we can launch the agent
js_url = saga.Url(js_endpoint)
logger.debug("saga.job.Service ('%s')" % js_url)
if js_url in self._shared_worker_data['job_services']:
js = self._shared_worker_data['job_services'][
js_url]
else:
js = saga.job.Service(js_url,
session=self._session)
self._shared_worker_data['job_services'][
js_url] = js
# ------------------------------------------------------
# Create SAGA Job description and submit the pilot job
jd = saga.job.Description()
jd.executable = "/bin/bash"
jd.arguments = [
"-l %s" % BOOTSTRAPPER_SCRIPT, bootstrap_args
]
jd.working_directory = saga.Url(pilot_sandbox).path
jd.project = project
jd.output = "bootstrap_1.out"
jd.error = "bootstrap_1.err"
jd.total_cpu_count = number_cores
jd.processes_per_host = cores_per_node
jd.wall_time_limit = runtime
jd.total_physical_memory = memory
jd.queue = queue
jd.environment = dict()
# Set the SPMD variation only if required
if spmd_variation:
jd.spmd_variation = spmd_variation
if 'RADICAL_PILOT_PROFILE' in os.environ:
jd.environment['RADICAL_PILOT_PROFILE'] = 'TRUE'
logger.debug("Bootstrap command line: %s %s" %
(jd.executable, jd.arguments))
msg = "Submitting SAGA job with description: %s" % str(
jd.as_dict())
logentries.append(Logentry(msg, logger=logger.debug))
pilotjob = js.create_job(jd)
pilotjob.run()
# do a quick error check
if pilotjob.state == saga.FAILED:
raise RuntimeError("SAGA Job state is FAILED.")
saga_job_id = pilotjob.id
self._shared_worker_data['job_ids'][pilot_id] = [
saga_job_id, js_url
]
msg = "SAGA job submitted with job id %s" % str(
saga_job_id)
logentries.append(Logentry(msg, logger=logger.debug))
#
# ------------------------------------------------------
log_dicts = list()
for le in logentries:
log_dicts.append(le.as_dict())
# Update the Pilot's state to 'PENDING_ACTIVE' if SAGA job submission was successful.
ts = timestamp()
ret = pilot_col.update(
{
"_id": pilot_id,
"state": LAUNCHING
}, {
"$set": {
"state": PENDING_ACTIVE,
"saga_job_id": saga_job_id,
"agent_config": agent_cfg_dict
},
"$push": {
"statehistory": {
"state": PENDING_ACTIVE,
"timestamp": ts
}
},
"$pushAll": {
"log": log_dicts
}
})
if ret['n'] == 0:
# could not update, probably because the agent is
# running already. Just update state history and
# jobid then
# FIXME: make sure of the agent state!
ret = pilot_col.update({"_id": pilot_id}, {
"$set": {
"saga_job_id": saga_job_id
},
"$push": {
"statehistory": {
"state": PENDING_ACTIVE,
"timestamp": ts
}
},
"$pushAll": {
"log": log_dicts
}
})
except Exception as e:
# Update the Pilot's state 'FAILED'.
out, err, log = self._get_pilot_logs(
pilot_col, pilot_id)
ts = timestamp()
# FIXME: we seem to be unable to bson/json handle saga
# log messages containing an '#'. This shows up here.
# Until we find a clean workaround, make log shorter and
# rely on saga logging to reveal the problem.
msg = "Pilot launching failed! (%s)" % e
logentries.append(Logentry(msg))
log_dicts = list()
log_messages = list()
for le in logentries:
log_dicts.append(le.as_dict())
log_messages.append(le.message)
pilot_col.update(
{
"_id": pilot_id,
"state": {
"$ne": FAILED
}
}, {
"$set": {
"state": FAILED,
"stdout": out,
"stderr": err,
"logfile": log
},
"$push": {
"statehistory": {
"state": FAILED,
"timestamp": ts
}
},
"$pushAll": {
"log": log_dicts
}
})
logger.exception('\n'.join(log_messages))
except SystemExit as e:
logger.exception(
"pilot launcher thread caught system exit -- forcing application shutdown"
)
import thread
thread.interrupt_main()
PTP_GUID = "00:00:00:00:00:00:00:00:ff:ff:08:00:27:f5:16:4f"
DEBUG = False
#check for commandline args
for i, arg in enumerate(sys.argv):
#Let commandline arg override config
if arg == '-d':
DEBUG = True
#Set logging file
elif arg == '-l':
if len(sys.argv) > i + 1:
sys.stdout = open(sys.argv[i + 1], 'w')
sys.stderr = open(sys.argv[i + 1], 'w')
else:
print "Please provide path the log file after -l"
thread.interrupt_main() #exiting program
#Setup Logging Output
logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s")
L = logging.getLogger()
if DEBUG is True:
L.setLevel(logging.DEBUG)
else:
L.setLevel(logging.INFO)
consoleHandler = logging.StreamHandler(sys.stdout)
consoleHandler.setFormatter(logFormatter)
L.addHandler(consoleHandler)
L.info("Server starting")
def traction_Orders_Callback(param):
pass
### START XBEE CODE ###
if rf_wifi_selector==1:
try:
xBeeSerial = serial.Serial(DIRECCION_XBEE, baudrate=BAUD_RATE)
print("\t \t -- Conexion con XBee S8 inicializada --")
except:
print("\033[1;31mNo se pudo inicializar la conexion con la XBee S8 \033[0;0m")
thread.interrupt_main()
if rf_wifi_selector==2:
s = socket.socket()
s.settimeout(1)
try:
s.connect((HOST, PORT))
s.send(str.encode("C+RF"))
except:
print("\033[1;31mError en conexion Wi-Fi \033[0;0m")
_thread.interrupt_main()
s.close()
if order_connect== 0:
print("\033[1;31mNo estan activas todas las conexiones, encontrara funcionalidad limitada \033[0;0m")
print("\t \t -- Modo de control RF --")
### Mensaje de inicializacion ###
def master_connection_init(conn): while 1: i=recvo(conn) if i==2: break thread.interrupt_main()
def handler():
debug("Shutting down HTTP server via timeout (%d) seconds." %
(ARGS['timeout']))
thread.interrupt_main()
def command(com, args, tw_user=None):
global e
if (e.is_set()): # System Tray Icon exit must trigger this
exit(0)
if not com or not isinstance(com, str):
return False
original_com = com
global inactive
global user_full_name
global user_prefix
global config_file
userin.twitter_user = tw_user
com = com.upper()
com = re.sub(r'([^\s\w]|_)+', '', com).strip()
print("You: " + com)
if inactive and com not in ("DRAGONFIRE", "DRAGON FIRE", "WAKE UP",
"HEY"):
return True
if USER_ANSWERING['status']:
if com.startswith("FIRST") or com.startswith(
"THE FIRST") or com.startswith("SECOND") or com.startswith(
"THE SECOND") or com.startswith(
"THIRD") or com.startswith("THE THIRD"):
USER_ANSWERING['status'] = False
selection = None
if com.startswith("FIRST") or com.startswith("THE FIRST"):
selection = 0
elif com.startswith("SECOND") or com.startswith("THE SECOND"):
selection = 1
elif com.startswith("THIRD") or com.startswith("THE THIRD"):
selection = 2
if USER_ANSWERING['for'] == 'wikipedia':
with nostderr():
search_query = USER_ANSWERING['options'][selection]
try:
wikiresult = wikipedia.search(search_query)
if len(wikiresult) == 0:
userin.say(
"Sorry, " + user_prefix +
". But I couldn't find anything about " +
search_query + " in Wikipedia.")
return True
wikipage = wikipedia.page(wikiresult[0])
wikicontent = "".join([
i if ord(i) < 128 else ' '
for i in wikipage.content
])
wikicontent = re.sub(r'\([^)]*\)', '', wikicontent)
userin.define_and_execute(
["sensible-browser", wikipage.url],
search_query)
userin.say(wikicontent)
return True
except requests.exceptions.ConnectionError:
userin.define_and_execute(
[" "], "Wikipedia connection error.")
userin.say(
"Sorry, " +
user_prefix +
". But I'm unable to connect to Wikipedia servers.")
return True
except Exception:
return True
if com in ("DRAGONFIRE", "DRAGON FIRE", "WAKE UP", "HEY"):
tts_kill()
inactive = False
userin.define([" "], " ")
userin.say(choice([
"Yes, " + user_prefix + ".",
"Yes. I'm waiting.",
"What is your order?",
"Ready for the orders!",
user_prefix + ", tell me your wish."
]))
elif "GO TO SLEEP" == com:
tts_kill()
inactive = True
userin.define_and_execute(
["echo"],
"Dragonfire deactivated. To reactivate say 'Dragonfire!' or 'Wake Up!'")
userin.say("I'm going to sleep")
elif com in ("ENOUGH", "SHUT UP"):
print("Dragonfire quiets.")
tts_kill()
elif VirtualAssistant.exact_match(com):
return True # the request has been handled
elif VirtualAssistant.in_match(com):
return True # the request has been handled
elif ("SEARCH" in com
or "FIND" in com) and VirtualAssistant.search_command(com):
pass # the request has been handled
elif com in ("MY TITLE IS LADY", "I'M A LADY", "I'M A WOMAN",
"I'M A GIRL"):
tts_kill()
config_file.update({
'gender': 'female'
},
Query().datatype == 'gender')
user_prefix = "My Lady"
userin.define([" "], " ")
userin.say("Pardon, " + user_prefix + ".")
elif com in ("MY TITLE IS SIR", "I'M A MAN", "I'M A BOY"):
tts_kill()
config_file.update({
'gender': 'male'
},
Query().datatype == 'gender')
user_prefix = "Sir"
userin.define([" "], " ")
userin.say("Pardon, " + user_prefix + ".")
elif com.startswith("CALL ME "):
tts_kill()
callme_config = config_file.search(Query().datatype == 'callme')
if callme_config:
config_file.update({
'title': original_com[8:].lower()
},
Query().datatype == 'callme')
else:
config_file.insert({
'datatype': 'callme',
'title': original_com[8:].lower()
})
user_prefix = original_com[8:].lower().encode("utf8")
userin.define([" "], " ")
userin.say("Pardon, " + user_prefix + ".")
# only for The United States today but prepared for all countries. Also
# only for celsius degrees today. --> by Radan Liska :-)
elif "WHAT" in com and "TEMPERATURE" in com:
tts_kill()
capture = re.search(
"(?:WHAT IS|WHAT'S) THE TEMPERATURE (?:IN|ON|AT|OF)? (?P<city>.*)", com)
if capture:
city = capture.group('city')
owm = pyowm.OWM("16d66c84e82424f0f8e62c3e3b27b574")
reg = owm.city_id_registry()
weather = owm.weather_at_id(
reg.ids_for(city)[0][0]).get_weather()
fmt = "The temperature in {} is {} degrees celsius"
msg = fmt.format(city,
weather.get_temperature('celsius')['temp'])
userin.define_and_execute([" "], msg)
userin.say(msg)
elif "FILE MANAGER" in com or "OPEN FILES" == com:
tts_kill()
userin.define_and_execute(["dolphin"], "File Manager") # KDE neon
userin.define_and_execute(["pantheon-files"],
"File Manager") # elementary OS
userin.define_and_execute(["nautilus", "--browser"],
"File Manager") # Ubuntu
userin.say("File Manager")
elif "OPEN CAMERA" == com:
tts_kill()
userin.define_and_execute(["kamoso"], "Camera") # KDE neon
userin.define_and_execute(["snap-photobooth"],
"Camera") # elementary OS
userin.define_and_execute(["cheese"], "Camera") # Ubuntu
userin.say("Camera")
elif "OPEN CALENDAR" == com:
tts_kill()
userin.define_and_execute(["korganizer"], "Calendar") # KDE neon
userin.define_and_execute(["maya-calendar"],
"Calendar") # elementary OS
userin.define_and_execute(["orage"], "Calendar") # Ubuntu
userin.say("Calendar")
elif "OPEN CALCULATOR" == com:
tts_kill()
userin.define_and_execute(["kcalc"], "Calculator") # KDE neon
userin.define_and_execute(["pantheon-calculator"],
"Calculator") # elementary OS
userin.define_and_execute(["gnome-calculator"],
"Calculator") # Ubuntu
userin.say("Calculator")
elif "SOFTWARE CENTER" in com:
tts_kill()
userin.define_and_execute(["plasma-discover"],
"Software Center") # KDE neon
userin.define_and_execute(
["software-center"],
"Software Center") # elementary OS & Ubuntu
userin.say("Software Center")
elif com.startswith("KEYBOARD "):
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
for character in original_com[9:]:
k.tap_key(character)
k.tap_key(" ")
elif com == "ENTER":
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.tap_key(k.enter_key)
elif com == "NEW TAB":
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.press_keys([k.control_l_key, 't'])
elif com == "SWITCH TAB":
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.press_keys([k.control_l_key, k.tab_key])
elif com in ("CLOSE", "ESCAPE"):
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.press_keys([k.control_l_key, 'w'])
k.tap_key(k.escape_key)
elif com == "GO BACK":
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.press_keys([k.alt_l_key, k.left_key])
elif com == "GO FORWARD":
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.press_keys([k.alt_l_key, k.right_key])
elif com == "SCROLL LEFT":
tts_kill()
with nostdout():
with nostderr():
m = PyMouse()
m.scroll(0, -5)
elif com == "SCROLL RIGHT":
tts_kill()
with nostdout():
with nostderr():
m = PyMouse()
m.scroll(0, 5)
elif com == "SCROLL UP":
tts_kill()
with nostdout():
with nostderr():
m = PyMouse()
m.scroll(5, 0)
elif com == "SCROLL DOWN":
tts_kill()
with nostdout():
with nostderr():
m = PyMouse()
m.scroll(-5, 0)
elif com in ("PLAY", "PAUSE", "SPACEBAR"):
tts_kill()
with nostdout():
with nostderr():
k = PyKeyboard()
k.tap_key(" ")
elif "SHUTDOWN THE COMPUTER" == com:
tts_kill()
userin.define(["sudo", "poweroff"], "Shutting down")
userin.say("Shutting down")
userin.execute(3)
elif com in ("GOODBYE", "GOOD BYE", "BYE BYE", "SEE YOU LATER", "CATCH YOU LATER"):
tts_kill()
userin.define([" "], " ")
userin.say("Goodbye, " + user_prefix)
# raise KeyboardInterrupt
thread.interrupt_main()
elif "WIKIPEDIA" in com and ("SEARCH" in com or "FIND" in com):
tts_kill()
with nostderr():
capture = re.search(
"(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? WIKIPEDIA", com)
if capture:
search_query = capture.group('query')
try:
wikiresult = wikipedia.search(search_query)
if len(wikiresult) == 0:
userin.say("Sorry, " + user_prefix +
". But I couldn't find anything about "
+ search_query + " in Wikipedia.")
return True
wikipage = wikipedia.page(wikiresult[0])
wikicontent = "".join([
i if ord(i) < 128 else ' '
for i in wikipage.content
])
wikicontent = re.sub(r'\([^)]*\)', '', wikicontent)
userin.define_and_execute(
["sensible-browser", wikipage.url], search_query)
userin.say(wikicontent)
except requests.exceptions.ConnectionError:
userin.define_and_execute(
[" "], "Wikipedia connection error.")
userin.say(
"Sorry, " + user_prefix +
". But I'm unable to connect to Wikipedia servers."
)
except wikipedia.exceptions.DisambiguationError as disambiguation:
USER_ANSWERING['status'] = True
USER_ANSWERING['for'] = 'wikipedia'
USER_ANSWERING['reason'] = 'disambiguation'
USER_ANSWERING['options'] = disambiguation.options[:3]
notify = "Wikipedia disambiguation. Which one of these you meant?:\n - " + \
disambiguation.options[0]
message = user_prefix + ", there is a disambiguation. Which one of these you meant? " + \
disambiguation.options[0]
for option in disambiguation.options[1:3]:
message += ", or " + option
notify += "\n - " + option
notify += '\nSay, for example: "THE FIRST ONE" to choose.'
userin.define_and_execute([" "], notify)
userin.say(message)
except BaseException:
pass
elif "YOUTUBE" in com and ("SEARCH" in com or "FIND" in com):
tts_kill()
with nostdout():
with nostderr():
capture = re.search(
"(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? YOUTUBE", com)
if capture:
search_query = capture.group('query')
info = youtube_dl.YoutubeDL({}).extract_info(
'ytsearch:' + search_query,
download=False,
ie_key='YoutubeSearch')
if len(info['entries']) > 0:
youtube_title = info['entries'][0]['title']
youtube_url = "https://www.youtube.com/watch?v=%s" % (
info['entries'][0]['id'])
userin.define(["sensible-browser", youtube_url],
youtube_title)
youtube_title = "".join([
i if ord(i) < 128 else ' '
for i in youtube_title
])
else:
youtube_title = "No video found, " + user_prefix + "."
userin.define(" ", " ")
userin.execute(0)
userin.say(youtube_title)
time.sleep(5)
k = PyKeyboard()
k.tap_key(k.tab_key)
k.tap_key(k.tab_key)
k.tap_key(k.tab_key)
k.tap_key(k.tab_key)
k.tap_key('f')
elif ("GOOGLE" in com
or "WEB" in com) and "IMAGE" not in com and ("SEARCH" in com
or "FIND" in com):
tts_kill()
with nostdout():
with nostderr():
capture = re.search(
"(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? (?:GOOGLE|WEB)?", com)
if capture:
search_query = capture.group('query')
tab_url = "http://google.com/?#q=" + search_query
userin.define_and_execute(
["sensible-browser", tab_url], search_query)
userin.say(search_query)
elif ("GOOGLE" in com
or "WEB" in com) and "IMAGE" in com and ("SEARCH" in com
or "FIND" in com):
tts_kill()
with nostdout():
with nostderr():
capture = re.search("(?:SEARCH IMAGES OF|FIND IMAGES OF|SEARCH|FIND) "
"(?P<query>.*) (?:IN|ON|AT|USING)? "
"(?:GOOGLE|WEB|GOOGLE IMAGES|WEB IMAGES)?", com)
if capture:
search_query = capture.group('query')
tab_url = "http://google.com/?#q=" + search_query + "&tbm=isch"
userin.define(["sensible-browser", tab_url],
search_query)
userin.execute(0)
userin.say(search_query)
else:
tts_kill()
arithmetic_response = arithmeticParser(com)
if arithmetic_response:
userin.define([" "], " ")
userin.say(arithmetic_response)
else:
learn_response = learn_.respond(original_com)
if learn_response:
userin.define([" "], " ")
userin.say(learn_response)
else:
if not omniscient_.respond(original_com, not args["silent"], userin, user_prefix, args["server"]):
dc_response = dc.respond(original_com, user_prefix)
if dc_response:
userin.define([" "], " ")
userin.say(dc_response)
def interrupt_the_server(self):
if interruptable:
thread.interrupt_main()
def handle_close(self): # Client disconnected
thread.interrupt_main()
def runsubprocess(args,verbose=False,shell=False,polling=False,printstdout=True,preexec_fn=None):
"""takes a subprocess argument list and runs Popen/communicate or Popen/poll() (if polling=True); if verbose=True, processname (string giving command call) is printed to screen (processname is always printed if a process results in error); errors are handled at multiple levels i.e. subthread error handling; fuction can be used fruitfully (returns stdout)"""
#function setup
import subprocess,sys,signal
try:
import thread
except:
import _thread
def subprocess_setup(): #see: https://github.com/vsbuffalo/devnotes/wiki/Python-and-SIGPIPE
signal.signal(signal.SIGPIPE, signal.SIG_DFL)
if preexec_fn=='sigpipefix':
preexec_fn=subprocess_setup
else:
assert preexec_fn==None,'Error: unrecognised preexec_fn argument %s'%preexec_fn
if shell==True:
processname=args[0]
processname=processname[0].split()
processname=(" ".join(a for a in processname))
else:
processname=(" ".join(a for a in args))
#
if verbose==True:
print('{0} {1}'.format(processname, 'processname'))
try:
if polling==True:
p=subprocess.Popen(args, stdout=subprocess.PIPE,shell=shell,preexec_fn=preexec_fn)
while True:
stdout=p.stdout.readline()
if p.poll() is not None:
break
if stdout: #if stdout not empty...
if printstdout==True:
print('{0}'.format(stdout.decode().strip()))
else:
p=subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=shell,preexec_fn=preexec_fn)
stdout, stderr= p.communicate()
if stdout:
if printstdout==True:
print('{0}'.format(stdout.decode()))
if stderr:
try: #want to output to stderr stream
if (sys.version_info > (3, 0)):
print('{0}'.format(stderr.decode()),file=sys.stderr) #Python3
else:
print>>sys.stderr,stderr #Python2
except: #if above code block fails for some reason, print stderr (to stdout)
print('{0}'.format(stderr.decode()))
if p.returncode==0:
if verbose==True:
print('{0} {1}'.format(processname, 'code has run successfully'))
else:
sys.exit() #triggers except below
except:
print('{0} {1}'.format(processname, '#this pipeline step produced error'))
print('unexpected error; exiting')
sys.exit()
if p.returncode!=0:
print('unexpected error; exiting')
try:
thread.interrupt_main()
except:
_thread.interrupt_main()
else:
if stdout:
return stdout.decode().strip()
def __playercheck(
self): #Two underscores, mangles to _GridObject__playercheck
if self.myinterface == None:
return
if self.level == None:
return
if self.game == None:
return
if self.vitality <= 0:
self.die()
return
self.level.redraw()
if self.status != "placed":
return
if len(self.level.child_interfaces) > 1:
self.myinterface.push_message("Your turn.")
self._lock.acquire()
while 1:
try:
e = self.myinterface.get_key_event()
except HTTPException:
#Obviously using RPC so just print
print("HTTP Exception in __playercheck() %r" %
(sys.exc_info(), ))
time.sleep(1)
else:
break
self._lock.release()
inexpensive = 0
if e in ("\x03", "\x04",
"\x1a"): #ETX ^C, EOT ^D, and ^Z (on Windows).
#Does not go through to Python otherwise, meaning that Linux main terminals
#may be rendered otherwise out of order until someone kills the game
#from a different terminal or over SSH (or rlogin).
#This is relevant if running this on an RPi (long ago the intent).
self.myinterface.interrupt()
elif e in ("down", "up", "left", "right", "8", "4", "6", "2", "7", "9",
"1", "3", "h", "j", "k", "l", "y", "u", "b", "n"):
direction = self.conv_to_direction(e)
target = self.direction_to_target(direction)
if self.game.debug_ghost or self.level.handle_move(target, self):
self.place(*target)
elif e == "#":
name = "#" + self.myinterface.ask_question("#")
if name in ("#debug", "#debugon"):
self.game.debug = 1
elif self.game.debug:
inexpensive = 1
if name in ("#debugoff", ):
self.game.debug = 0
elif name in ("#ghost", "#ghoston"):
self.game.debug_ghost = 1
elif name in ("#ghostoff", ):
self.game.debug_ghost = 0
elif name in ("#fovoff", "#fovoffon", "#clairvoyant",
"#allsight", "#seeall"):
self.game.debug_fov_off = 1
elif name in ("#fov", "#fovon", "#fovoffoff",
"#clairvoyantoff", "#allsightoff", "#seealloff"):
self.game.debug_fov_off = 0
elif name.startswith("#passthrough "):
self.game.handle_command(name.split(" ", 1)[1], self)
elif name in ("#bugreport", "#report", "#gurumeditation",
"#guru"):
self.game.dump_report()
elif name in ("#save", "#s"):
self.game.save(
self.myinterface.ask_question("Save file: "))
elif name in ("#testerror", ):
raise RuntimeError("testing error handler")
elif name in ("#abort", "#abrt", "#kill"):
import os
os.abort()
elif name in ("#quit", ):
interrupt_main()
else:
inexpensive = self.level.handle_command(name, self)
else:
inexpensive = self.level.handle_command(name, self)
else:
inexpensive = self.level.handle_command(e, self)
if inexpensive:
# Did not expend the players go - give them the go then.
self.__playercheck()
return
if len(self.level.child_interfaces) > 1:
self.myinterface.push_message("Turn over.")
self.myinterface.dump_messages()
def wait(secs):
for n in xrange(timeout_secs):
time.sleep(1)
if signal_finished: break
else:
thread.interrupt_main()
def run(self):
"""run() is called when the process is started via
PilotManagerController.start().
"""
# make sure to catch sys.exit (which raises SystemExit)
try :
logger.debug("Worker thread (ID: %s[%s]) for PilotManager %s started." %
(self.name, self.ident, self._pm_id))
while not self._stop.is_set():
# # Check if one or more startup requests have finished.
# self.startup_results_lock.acquire()
# new_startup_results = list()
# for transfer_result in self.startup_results:
# if transfer_result.ready():
# result = transfer_result.get()
# self._db.update_pilot_state(
# pilot_uid=result["pilot_uid"],
# state=result["state"],
# sagajobid=result["saga_job_id"],
# pilot_sandbox=result["sandbox"],
# global_sandbox=result["global_sandbox"],
# submitted=result["submitted"],
# logs=result["logs"]
# )
# else:
# new_startup_results.append(transfer_result)
# self.startup_results = new_startup_results
# self.startup_results_lock.release()
# Check and update pilots. This needs to be optimized at
# some point, i.e., state pulling should be conditional
# or triggered by a tailable MongoDB cursor, etc.
pilot_list = self._db.get_pilots(pilot_manager_id=self._pm_id)
action = False
for pilot in pilot_list:
pilot_id = str(pilot["_id"])
new_state = pilot["state"]
if pilot_id in self._shared_data:
old_state = self._shared_data[pilot_id]["data"]["state"]
else:
old_state = None
self._shared_data[pilot_id] = {
'data': pilot,
'callbacks': [],
'facade_object': None
}
self._shared_data[pilot_id]['data'] = pilot
# FIXME: *groan* what a hack... The Canceling state is by
# the nature of it not recorded in the database, but only in
# the local cache. So if we see it as old state, we have to
# avoid state transitions into non-final states in the cache
# at all cost -- so we catch this here specifically
no_cb = False
if old_state == CANCELING :
if new_state not in [DONE, FAILED, CANCELED] :
# restore old state, making the cache explicitly
# different than the DB recorded state
self._shared_data[pilot_id]["data"]["state"] = old_state
# do not tr igger a state cb!
no_cb = True
if new_state != old_state :
action = True
if not no_cb :
# On a state change, we fire zee callbacks.
logger.info("ComputePilot '%s' state changed from '%s' to '%s'." \
% (pilot_id, old_state, new_state))
# The state of the pilot has changed, We call all
# pilot-level callbacks to propagate this. This also
# includes communication to the unit scheduler which
# may, or may not, cancel the pilot's units.
self.call_callbacks(pilot_id, new_state)
# If the state is 'DONE', 'FAILED' or 'CANCELED', we also
# set the state of the compute unit accordingly (but only
# for non-final units)
if new_state in [FAILED, DONE, CANCELED]:
unit_ids = self._db.pilot_list_compute_units(pilot_uid=pilot_id)
self._db.set_compute_unit_state (
unit_ids=unit_ids,
state=CANCELED,
src_states=[ PENDING_INPUT_STAGING,
STAGING_INPUT,
PENDING_EXECUTION,
SCHEDULING,
EXECUTING,
PENDING_OUTPUT_STAGING,
STAGING_OUTPUT
],
log="Pilot '%s' has terminated with state '%s'. CU canceled." % (pilot_id, new_state))
# After the first iteration, we are officially initialized!
if not self._initialized.is_set():
self._initialized.set()
# sleep a little if this cycle was idle
if not action :
time.sleep(IDLE_TIME)
except SystemExit as e :
logger.exception ("pilot manager controller thread caught system exit -- forcing application shutdown")
import thread
thread.interrupt_main ()
finally :
# shut down the autonomous pilot launcher worker(s)
for worker in self._pilot_launcher_worker_pool:
logger.debug("pworker %s stops launcher %s" % (self.name, worker.name))
worker.stop ()
logger.debug("pworker %s stopped launcher %s" % (self.name, worker.name))
def _receiverThread(session):
"""Receiving messages from cjdns admin server"""
timeOfLastSend = time.time()
timeOfLastRecv = time.time()
try:
while True:
if (timeOfLastSend + KEEPALIVE_INTERVAL_SECONDS < time.time()):
if (timeOfLastRecv + 10 < time.time()):
raise Exception("ping timeout")
session.socket.send(
'd1:q18:Admin_asyncEnabled4:txid8:keepalive')
timeOfLastSend = time.time()
# Did we get data from the socket?
got_data = False
while True:
# This can be interrupted and we need to loop it.
try:
data = session.socket.recv(BUFFER_SIZE)
except (socket.timeout):
# Stop retrying, but note we have no data
break
except socket.error as e:
if e.errno != errno.EINTR:
# Forward errors that aren't being interrupted
raise
# Otherwise it was interrupted so we try again.
else:
# Don't try again, we got data
got_data = True
break
if not got_data:
# Try asking again.
continue
try:
benc = bdecode(data)
except (KeyError, ValueError):
print("error decoding [" + data + "]")
continue
if benc['txid'] == 'keepaliv':
if benc['asyncEnabled'] == 0:
raise Exception("lost session")
timeOfLastRecv = time.time()
else:
# print "putting to queue " + str(benc)
session.queue.put(benc)
except KeyboardInterrupt:
print("interrupted")
import thread
thread.interrupt_main()
except Exception as e:
# Forward along any errors, before killing the thread.
session.queue.put(e)
def quit_function(fn_name):
# print to stderr, unbuffered in Python 2.
logging.error("{0} took too long".format(fn_name))
sys.stderr.flush() # Python 3 stderr is likely buffered.
thread.interrupt_main()
def exit(self, value=0):
print 'Exiting...'
# sys.exit(value)
self.loop.quit()
# cleanup_stop_thread()
thread.interrupt_main()
def _telnet_server(self, pstdin_w, pstdout_r, sock, orig_stdout):
queue = PriorityQueue()
inputbuffer = b''
self._telnet_seq = 0
try:
t = threading.Thread(target=self._telnet_server_writer,
args=(queue, sock))
t.daemon = True
t.start()
lock = threading.Lock()
def writeall(data):
start = 0
while start < len(data):
size = os.write(pstdin_w, data[start:])
start += size
def sendcontrol(t, data):
lock.acquire()
try:
self._telnet_seq += 1
seq = self._telnet_seq
finally:
lock.release()
queue.put((t, seq, data))
t2 = threading.Thread(target=self._telnet_server_writer2,
args=(pstdout_r, queue, lock, orig_stdout))
t2.daemon = True
t2.start()
escaping = False
option = None
while True:
newdata = sock.recv(1024)
if newdata == b'':
break
for i in range(0, len(newdata)):
c = newdata[i:i + 1]
if escaping:
if option == b'\xfd' and c == b'\x06':
sendcontrol(1, b'\xff\xfb\x06')
option = None
escaping = False
elif option == b'\xfd' or option == b'\xfe':
sendcontrol(1, b'\xff\xfc' + c)
option = None
escaping = False
elif option == b'\xfb' or option == b'\xfc':
sendcontrol(1, b'\xff\xfe' + c)
option = None
escaping = False
elif c in (b'\xfb', b'\xfc', b'\xfd', b'\xfe'):
option = c
else:
option = None
if c == b'\xf3' or c == b'\xf4':
thread.interrupt_main()
escaping = False
else:
if c == b'\x03':
thread.interrupt_main()
elif c == b'\x08':
inputbuffer = inputbuffer[:-1]
elif c == b'\x00':
inputbuffer += b'\n'
writeall(inputbuffer)
inputbuffer = b''
elif c == b'\r' or c == b'\n':
inputbuffer += c
writeall(inputbuffer)
inputbuffer = b''
elif c == b'\xff':
escaping = True
else:
inputbuffer += c
except OSError:
pass
except IOError:
pass
finally:
try:
os.close(pstdin_w)
except:
pass
queue.put((-1, -1, -1))
def handler():
print("[DEBUG] Shutting down HTTP server via timeout (%d) seconds."
% (ARGS.timeout))
thread.interrupt_main()
def f():
time.sleep(0.5)
thread.interrupt_main()
def input_alert(self,
title='',
message='',
input='',
ok_button_title='ok',
hide_cancel_button=False):
#first, cover up view
#then
from threading import Timer
print(threading.enumerate())
from thread import interrupt_main
self.e.clear()
for s in self.subviews:
self.remove_subview(s)
if self.superview:
dim = ui.View(frame=(0, 0, self.superview.width,
self.superview.height),
bg_color=(0.5, 0.5, 0.5, 0.75))
inputbox = ui.View()
inputbox.width = 300
inputbox.corner_radius = 10
inputbox.bg_color = 'white'
inputbox.border_color = 'black'
inputbox.center = (dim.width / 2, dim.height / 2)
titlelabel = ui.Label()
titlelabel.text = title
titlelabel.alignment = ui.ALIGN_CENTER
titlelabel.font = ('<system-bold>', 14)
titlelabel.size_to_fit()
titlelabel.y = 10
titlelabel.x = 0
messagelabel = ui.Label()
messagelabel.text = message
messagelabel.alignment = ui.ALIGN_CENTER
messagelabel.font = ('<system>', 12)
messagelabel.x = 0
messagelabel.y = titlelabel.height + titlelabel.y
messagelabel.size_to_fit()
inputfield = ui.TextField()
inputfield.text = input
inputfield.font = ('<system>', 12)
inputfield.height = titlelabel.height * 1.5
inputfield.x = 0.1 * inputbox.width
inputfield.y = messagelabel.height + messagelabel.y
inputfield.width = .8 * inputbox.width
inputfield.corner_radius = 0
inputfield.border_width = 1
inputfield.action = self.input_action
buttons = ui.SegmentedControl()
buttons.corner_radius = 0
buttons.height = 30
buttons.segments = ('Cancel', 'Ok')
buttons.width = titlelabel.width = messagelabel.width = inputbox.width
buttons.width += 10
buttons.x -= 5
buttons.y = inputfield.y + inputfield.height + 10
buttons.action = self.button_action
inputbox.height = buttons.y + buttons.height - 1
dim.add_subview(inputbox)
inputbox.add_subview(titlelabel)
inputbox.add_subview(messagelabel)
inputbox.add_subview(inputfield)
inputbox.add_subview(buttons)
self.superview.add_subview(dim)
self.bring_to_front()
dim.bring_to_front()
self.width = dim.width
self.height = dim.height
def check_if_onscreen():
if not self.on_screen:
self.e.set()
else:
Timer(0.25, check_if_onscreen).start()
check_if_onscreen()
self.e.wait()
self.superview.remove_subview(dim)
if not hide_cancel_button and buttons.selected_index == 0:
rtnvalue = None
self.width = 1
self.height = 1
interrupt_main()
self.e.set()
else:
rtnvalue = inputfield.text
self.width = 1
self.height = 1
return rtnvalue
def input_thread():
global bool_send_default_program
while True:
key_pressed = read_key()
print("Key pressed-{}".format(key_pressed))
if key_pressed.lower() == "q":
thread.interrupt_main()
break
else:
res = ''
if key_pressed.lower() == "i":
res = command.init_seq_motor_1(6, 8, 7)
if key_pressed.lower() == "o":
res = command.init_seq_motor_2(10, 11, 12)
if key_pressed.lower() == "a":
res = command.move(100, 'R')
if key_pressed.lower() == "b":
res = command.move(200, 'L')
if key_pressed.lower() == "0":
res = command.moveto(0)
if key_pressed.lower() == "1":
res = command.moveto(90)
if key_pressed.lower() == "2":
res = command.moveto(180)
if key_pressed.lower() == "3":
res = command.moveto(-360)
if key_pressed.lower() == "4":
res = command.moveto(360 * 4)
if key_pressed.lower() == "c":
res = command.define_vel_acc(60, 400, 1)
if key_pressed.lower() == "d":
res = command.disable_pins(0)
if key_pressed.lower() == "e":
res = command.define_vel_acc(60, 400, 15)
if key_pressed.lower() == "f":
res = command.define_vel_acc(60, 400, 30)
if key_pressed.lower() == "g":
res = command.define_vel_acc(50, 120, 60)
if key_pressed.lower() == "9":
res = command.delay(300)
if key_pressed.lower() == "8":
res = command.delay(600)
if key_pressed.lower() == "s":
res = command.show_prog(0)
if key_pressed.lower() == "[":
res = command.program_start(0)
if key_pressed.lower() == "]":
res = command.program_end(0)
if key_pressed.lower() == "l":
res = command.show_prog(1)
if key_pressed.lower() == "{":
res = command.program_start(1)
if key_pressed.lower() == "}":
res = command.program_end(1)
if key_pressed.lower() == "t":
res = command.run_prog(0)
if key_pressed.lower() == "y":
res = command.run_prog(1)
if key_pressed.lower() == "=":
bool_send_default_program = True
if not res == '': ser.write(res)
def do_GET(self):
url = urlparse.urlsplit(self.path)
query = dict(cgi.parse_qsl(url[3]))
index_query = self.path.find('?')
if index_query == -1:
curr_path = self.path
else:
curr_path = self.path[:index_query]
# non-command line HTTP clients, i.e. Internet browsers, may request a favorite icon
if curr_path == '/favicon.ico':
if ThreadedRequestHandler.does_favicon_exist:
self.send_response(200)
self.send_header('Content-Type', 'image/x-icon')
self.end_headers()
self.wfile.write(ThreadedRequestHandler.favicon_data)
self.wfile.close()
else:
self.send_error(404, 'ERROR - ICON NOT FOUND: %s' % self.path)
# stop the server, if only called from localhost
elif curr_path == '/stop':
if self.client_address[0] == LOCAL_HOST:
body = 'stopped\n'
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
self.wfile.close()
if sys.version_info < (2, 5):
self.call_shell('./kill.sh')
else:
thread.interrupt_main()
else:
self.send_error(403, 'FORBIDDEN: not localhost')
# get the directory listing as JSON {name, size}
elif curr_path == '/list':
file_dir = query.get('dir')
file_prefix = query.get('prefix')
file_suffix = query.get('suffix')
check = False
if not file_dir:
self.send_error(
400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path)
elif not does_dir_exist(file_dir):
self.send_error(
400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path)
else:
# De Morgan's Laws
# https://en.wikipedia.org/wiki/De_Morgan%27s_laws
#if not file_prefix and not file_suffix:
if not (file_prefix or file_suffix):
check = False
else:
if not file_prefix:
file_prefix = ''
if not file_suffix:
file_suffix = ''
check = True
file_info = []
curr_dir = EXPORT_DIR_ABS + os.sep + file_dir
for file_name in os.listdir(curr_dir):
add = False
if os.path.isfile(curr_dir + os.sep + file_name):
if check:
add = (file_name.startswith(file_prefix)
and file_name.endswith(file_suffix))
else:
add = True
if add:
file_info.append(
'{"name":"' + file_name + '","size":' +
str(os.path.getsize(curr_dir + os.sep +
file_name)) + '}')
if len(file_info) > 0:
list_json = '['
list_json += ','.join(file_info)
list_json += ']'
else:
list_json = '[]'
list_json += '\n'
self.send_response(200)
self.send_header('Content-Type', 'application/json')
self.end_headers()
self.wfile.write(list_json)
self.wfile.close()
# get the file, optionally as compressed
elif curr_path == '/get':
file_dir = query.get('dir')
file_name = query.get('name')
comp_ext = query.get('compress')
compress = False
if comp_ext:
if comp_ext in ['zip', 'gz']:
compress = True
if not file_dir:
self.send_error(
400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path)
elif not does_dir_exist(file_dir):
self.send_error(
400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path)
elif not file_name:
self.send_error(400,
'ERROR - FILE NAME IS MISSING: %s' % self.path)
elif not does_file_exist(file_dir, file_name):
self.send_error(400,
'ERROR - FILE DOES NOT EXIST: %s' % self.path)
else:
curr_dir = EXPORT_DIR_ABS + os.sep + file_dir
try:
sendReply = False
if file_name.endswith(".html"):
mime_type = 'text/html'
sendReply = True
elif file_name.endswith(".csv"):
mime_type = 'text/csv'
sendReply = True
elif file_name.endswith(".gz"):
mime_type = 'application/gzip'
sendReply = True
elif file_name.endswith(".zip"):
mime_type = 'application/zip'
sendReply = True
else:
mime_type = 'text/plain'
sendReply = True
if sendReply:
f = open(curr_dir + os.sep + file_name, 'rb')
if compress:
ok = True
base_name = os.path.splitext(
os.path.basename(file_name))[0]
file_content = f.read()
file_size = 0
file_comp = ''
mime_type = ''
content = ''
if comp_ext == 'gz':
gz_file = self.gz_content(
file_name, file_content)
gz_size = self.get_size(gz_file)
content = gz_file.getvalue()
gz_file.close()
file_comp = base_name + '.gz'
file_size = gz_size
mime_type = 'application/gzip'
elif comp_ext == 'zip':
zip_file = self.zip_content(
file_name, file_content)
zip_size = self.get_size(zip_file)
content = zip_file.getvalue()
zip_file.close()
file_comp = base_name + '.zip'
file_size = zip_size
mime_type = 'application/zip'
else:
ok = False
if ok:
self.send_response(200)
self.send_header('Content-Type', mime_type)
self.send_header('Content-Length', file_size)
self.send_header(
'Content-Disposition',
'attachment; filename=' + file_comp)
self.end_headers()
self.wfile.write(content)
self.wfile.close()
else:
self.send_error(
400,
'ERROR - UNKNOWN COMPRESSION FORMAT: %s' %
self.path)
else:
self.send_response(200)
self.send_header('Content-Type', mime_type)
self.send_header(
'Content-Length',
str(
os.path.getsize(curr_dir + os.sep +
file_name)))
self.send_header(
'Content-Disposition',
'attachment; filename=' + file_name)
self.end_headers()
while True:
data = f.read(4096)
self.wfile.write(data)
if not data:
break
self.wfile.write('\n')
self.wfile.close()
f.close()
else:
self.send_error(
400, 'ERROR - BAD FILE REQUEST: %s' % self.path)
return
except IOError:
self.send_error(404,
'ERROR - FILE NOT FOUND: %s' % self.path)
# create a tarball which may take ages and then call back to inform the client
elif curr_path == '/tar':
file_dir = query.get('dir')
file_name = query.get('name')
if not file_dir:
self.send_error(
400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path)
elif not does_dir_exist(file_dir):
self.send_error(
400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path)
elif not file_name:
self.send_error(400,
'ERROR - FILE NAME IS MISSING: %s' % self.path)
elif not does_file_exist(file_dir, file_name):
self.send_error(400,
'ERROR - FILE DOES NOT EXIST: %s' % self.path)
else:
cb_host = query.get('host')
cb_port = query.get('port')
cb_path = query.get('path')
if cb_host and cb_port and cb_path:
body = 'working ...\n'
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.send_header('Content-Length', str(len(body)))
self.end_headers()
self.wfile.write(body)
self.wfile.close()
base_name = os.path.splitext(
os.path.basename(file_name))[0]
curr_dir = file_dir
tgz_file = base_name + '.tgz'
file_comp = tgz_file
cb_url = {
'host': cb_host,
'port': cb_port,
'path': cb_path,
'timeout': HTTP_CB_TIMEOUT,
'tar': tgz_file
}
if does_file_exist(file_dir, file_comp):
logger.info('file already exists: %s' % file_dir +
os.sep + file_comp)
ret = 0
else:
args = 'cd ' + curr_dir + '; tar -czf ' + file_comp + ' ' + file_name
ret = self.call_shell(args)
if ret == 0:
cb_url['query'] = 'success=true'
else:
cb_url['query'] = 'success=false'
self.http_get(cb_url)
else:
self.send_error(
400, 'ERROR - CALLBACK PARAMETERS ARE MISSING: %s' %
self.path)
# delete file in a given directory
elif curr_path == '/delete':
file_dir = query.get('dir')
file_name = query.get('name')
file_prefix = query.get('prefix')
file_suffix = query.get('suffix')
if not file_dir:
self.send_error(
400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path)
elif not does_dir_exist(file_dir):
self.send_error(
400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path)
else:
curr_dir = EXPORT_DIR_ABS + os.sep + file_dir
if file_name:
if does_file_exist('', curr_dir + os.sep + file_name):
os.remove(curr_dir + os.sep + file_name)
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
self.wfile.write('deleted: ' + curr_dir + os.sep +
file_name)
self.wfile.close()
else:
self.send_error(
400, 'ERROR - FILE DOES NOT EXIST: %s' % curr_dir +
os.sep + file_name)
else:
if not (file_prefix or file_suffix):
self.send_error(
400, 'ERROR - DELETE PARAMETERS ARE MISSING: %s' %
self.path)
else:
del_file = 0
if not file_prefix:
file_prefix = '*'
if not file_suffix:
file_suffix = '*'
for file_name in os.listdir(curr_dir):
if file_prefix == '*' and not file_suffix == '*':
if file_name.endswith(file_suffix):
del_file += 1
os.remove(curr_dir + os.sep + file_name)
elif file_suffix == '*' and not file_prefix == '*':
if file_name.startswith(file_prefix):
del_file += 1
os.remove(curr_dir + os.sep + file_name)
elif file_suffix == '*' and file_prefix == '*':
del_file += 1
os.remove(curr_dir + os.sep + file_name)
else:
if file_name.startswith(
file_prefix) and file_name.endswith(
file_suffix):
del_file += 1
os.remove(curr_dir + os.sep + file_name)
self.send_response(200)
self.send_header('Content-Type', 'text/plain')
self.end_headers()
if del_file > 0:
self.wfile.write('deleted ' + str(del_file) +
' files\n')
else:
self.wfile.write('no file is deleted\n')
self.wfile.close()
else:
self.send_error(400,
'ERROR - BAD RESOURCE REQUEST: %s' % self.path)
def handle_error(request, client_address):
_type, value, tb = sys.exc_info()
if isinstance(value, KeyboardInterrupt):
interrupt_main()
def action():
time.sleep(2.0)
thread.interrupt_main()
def quit_function(func):
print('{0} TimeoutError'.format(func), file=sys.stderr)
sys.stderr.flush()
thread.interrupt_main()
def kill_proc(p):
p.kill()
# The timer is running in a separate thread, so
# use interrupt_main() to throw a KeyboardInterrupt
# back in the main thread.
thread.interrupt_main()
def interrupter():
time.sleep(0.25)
# Raises KeyboardInterrupt in the main thread
thread.interrupt_main()
def process_output_parser(self, invoked_process):
logger = logging.getLogger("ProcessBot.process_output_parser")
logger.info("start parsing stdout of created process " + self.service)
with self._lock_bus:
if self.bus.state() == 'connected':
self.bus.SendMessage(
busIndipendentMessages.LogMessage(
self._remote_wps_endpoint, "INFO",
"start parsing stdout of created process " +
self.service))
else:
try:
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
# self.bus.xmpp.get_roster()
if self.bus.state() == 'connected':
self.bus.SendMessage(
busIndipendentMessages.LogMessage(
self._remote_wps_endpoint, "INFO",
"start parsing stdout of created process " +
self.service))
else:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint "
+ str(self._remote_wps_endpoint))
except:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint " +
str(self._remote_wps_endpoint))
# Listen to stdout
stdout_parser_compiled = [re.compile(r) for r in self._stdout_parser]
stack_trace = []
while True:
line = invoked_process.stdout.readline()
if line != '':
line = line.strip()
logger.debug("Received line: " + line)
stack_trace.append(line)
match = False
#todo: define a line to get pid of computational job process
for rgx, action in zip(stdout_parser_compiled,
self._stdout_action):
res = rgx.match(line)
if (res):
if (action == "progress"):
with self._lock_bus:
if self.bus.state() != 'connected':
try:
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
# self.bus.xmpp.get_roster()
except:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint "
+ str(self._remote_wps_endpoint))
self.bus.SendMessage(
busIndipendentMessages.ProgressMessage(
self._remote_wps_endpoint,
float(res.group(1).strip())))
match = True
break
elif (action == "log"):
with self._lock_bus:
if self.bus.state() != 'connected':
try:
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
# self.bus.xmpp.get_roster()
except:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint "
+ str(self._remote_wps_endpoint))
self.bus.SendMessage(
busIndipendentMessages.LogMessage(
self._remote_wps_endpoint,
res.group(1).strip(),
res.group(2).strip()))
match = True
break
elif (action == "abort"):
with self._lock_bus:
if self.bus.state() != 'connected':
try:
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
# self.bus.xmpp.get_roster()
except:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint "
+ str(self._remote_wps_endpoint))
self.bus.SendMessage(
busIndipendentMessages.ErrorMessage(
self._remote_wps_endpoint,
res.group(2).strip()))
match = True
break
elif (action == "ignore"):
match = True
break
else:
pass
else:
#end of stream
break
logger.debug("process " + self.service + "stdout is over")
#wait for process exit code
return_code = invoked_process.wait()
logger.info("process exit code is " + str(return_code))
if return_code == 0:
#success
logger.info("process exit code is " + str(return_code) +
": success")
logger.info(
"send job-completed message to WPS with output parameter")
outputs = dict()
try:
for p in self._output_parameters_defs.parameters():
outputs[p.get_name()] = [
p.get_value(),
p.get_description(),
p.get_title(),
p.get_type(),
p.is_publish_as_layer(),
p.get_publish_layer_name(),
p.get_publish_default_style(),
p.get_publish_target_workspace(),
p.get_metadata()
]
except:
logging.exception("Process " + str(self._uniqueExeId) +
" Exception: " +
str(traceback.format_exc(sys.exc_info())))
error_message = "process exit code is " + str(
return_code) + ": failure\n" + "\n".join(
str(e) for e in stack_trace)
self.send_error_message(error_message)
#self.bus.disconnect()
logger.info("after send job-error message to WPS")
thread.interrupt_main()
os._exit(return_code)
logger.info("trying to acquire bus lock...")
with self._lock_bus:
logger.info("bus lock acquired...")
if self.bus.state() != 'connected':
try:
self.bus.xmpp.reconnect()
self.bus.xmpp.send_presence()
# self.bus.xmpp.get_roster()
except:
logger.info(
"[XMPP Disconnected]: Process " +
str(self._uniqueExeId) +
" Could not send info message to GeoServer Endpoint "
+ str(self._remote_wps_endpoint))
counter = 1
while not self._finished:
logger.info("sending 'completed' message tentative #" +
str(counter))
self.bus.SendMessage(
busIndipendentMessages.CompletedMessage(
self._remote_wps_endpoint,
self._remote_wps_baseurl, outputs))
counter = counter + 1
if counter < 100:
sleep(10)
else:
logger.error(
"Could not contact Remote WPS with. Forcibly shutdown the process..."
)
thread.interrupt_main()
os._exit(-1)
logger.info("after send job-completed message to WPS")
else:
error_message = "process exit code is " + str(
return_code) + ": failure\n" + "\n".join(
str(e) for e in stack_trace)
logger.critical("process exit code is " + str(return_code) +
": failure")
#todo: should i wait for finish message here as well? No
self.send_error_message(error_message)
#self.bus.disconnect()
logger.info("after send job-error message to WPS")
thread.interrupt_main()
os._exit(return_code)
def end():
tb.stop()
tb.close()
thread.interrupt_main()
