def response(context, flow): """======================================================================== "Called when a server response has been received"... łapię wyłącznie odpowiedzi, bo interesują mnie zestawy (request/response). Przechwycony response wraz z requestem wchodzą w skład transakcji, reprezentowanej przez mitmproxy.models.HTTPFlow() "HTTPFlow is collection of objects representing a single HTTP transaction". Więcej info na WWW: http://docs.mitmproxy.org/en/stable/dev/models.html ===========================================================================""" if flow.request.host.endswith('.thesettlersonline.pl'): if "application/x-amf" in flow.response.headers.get("Content-Type", "_"): with decoded(flow.response): res = flow.response.content req = flow.request.content if search( 'defaultGame.Communication.VO.TradeWindow.dTradeWindowResultVO', res )\ and search( 'userAcceptedTradeIDs', res ) and search( 'tradeOffers', res )\ and search( 'GetAvailableOffers', req ): log.debug("got trade REQ/RESP pair, feeding TDD thread...") try: t= Thread(target=ttd._incoming_traffic_handler, args=(context, flow,)) t.setDaemon(True) t.start() except (KeyboardInterrupt, SystemExit): log.info('caught either KeyboardInterrupt or SystemExit, quitting threads') t.__stop() import thread thread.interrupt_main()
def parse_itemDetailView(self, response): if 'techerror' in response.url: print 'somehow found tech error at url ', response.request.url # logfun = logging.getLogger("logfun") # time.sleep(0.10) try: hxs = HtmlXPathSelector(response) page_title = hxs.select('//title/text()').extract()[0] med_image = hxs.select('//input[@id="imageURL"]/@value')[0].extract() # 200x200 image # Should probably do some form of exception, but lazy large_image = '' large_image_path = hxs.select('//input[@id="primaryImageURL"]/@value') if len(large_image_path) > 0: large_image = large_image_path[0].extract() # large_image = hxs.select('//input[@id="primaryImageURL"]/@value')[0].extract() # 600x600 image # Only take information in paragraphs, but preserve HTML & formatting: Combine all paragraphs for this entry details_list = hxs.select('//div[@id="productDetails-details"]/p').extract() details = ''.join(details_list) cut_start = details.find('<a class="') cut_end = details.rfind('</p>') # Remove the disclaimer link at bottom if cut_start > 0 and cut_end > 0: details = details[0:cut_start] + details[cut_end:] nutrition_path = hxs.select('//div[@id="productDetails-nutrition"]/table').extract() nutrition = '' if len(nutrition_path) > 0: nutrition = nutrition_path[0] ingredients = '' if len(hxs.select('//div[@id="ingredients"]').extract()) > 0: ingredients = hxs.select('//div[@id="ingredients"]')[0].extract() #ingredients = hxs.select('//div[@id="ingredients"]')[0].extract() detailed_item = DetailedShopItem(name = response.meta['name'], size = response.meta['size'], unit_price = response.meta['unit_price'], productId = response.meta['productId'], price = response.meta['price'], cnid = response.meta['cnid'], small_image = response.meta['thumb'], med_image = med_image, large_image = large_image, details = details, nutrition = nutrition, ingredients = ingredients) self.all_items_detail.append(detailed_item) # except: # e = sys.exc_info()[0] # print e.print_stack() # sys.exit("error has occurred") except Exception, ex: print 'gg exception' print 'came from', response.request.url raise #raise KeyboardInterrupt # logfun.exception("Bad gg") # logfun.debug("figure this out") thread.interrupt_main()
def worker_thread(work_queue,shutdown_flag): while not shutdown_flag.is_set(): fileno, address, head, body = work_queue.get_request() if fileno == 0: thread.interrupt_main() return parser = Parser(head,body) request = parser.parse() if request == None: response = Http400Response() else: request.remote_addr, request.remote_port = address request._work_queue = work_queue log("%s - %s" %( request.remote_addr, request.path )) handler = config.HANDLER_CLASS(request) try: response = handler.response() except: exc = traceback.format_exc() log(exc) if config.DEBUG: response = Http500Response(exc) else: response = Http500Response() work_queue.submit_response(fileno,response)
def _uoczer(self): """ "ten co czai" :) :: perform once-upon-a-time checks and cleanups :: - sprawdza czas ostatnich odświerzeń rynków każdego z realmów - dla przestarzałych emituje duplikaty, jeśli są dostępne - planuje kolejne uruchomienie siebie przy pomocy threading.Timer """ self.uoczerIterCounter+=1 try: # performing timed checks and cleanups if self.now()-self.c.lastupdatedTS > 3000 and self.uoczerIterCounter % 1000 == 0: log.info('PINGING DATABASE') self.db.ping() if self._trade_refresh_needed() and self.c.treqs: recent_key = self.c.treqs.keys()[0] recent_request= self.c.treqs[recent_key] raw_trade_response= recent_request.execute() if raw_trade_response: parsed_response_body= self._amf_raw_response_parser(raw_trade_response) self._trade_update_market(parsed_response_body) log.info('trade updated with duplicated treq (uID: %d)'%recent_key) else: log.info('trade requery failed, deleting treq (uID %d)'%recent_key) del self.c.treqs[recent_key] except (KeyboardInterrupt, SystemExit): # stop on interrupts log.info('caught either KeyboardInterrupt or SystemExit, quitting thread') import thread thread.interrupt_main() exit() finally: log.debug('scheduling run in %ds'%self.c.watchT) self.uoczertimer = Timer(self.c.watchT, self._uoczer) # reschedule run self.uoczertimer.setDaemon(True) # with abort at shutdown self.uoczertimer.start()
def process_stream(self): """Process lines using parseline() forever""" # Catch-all so we know why the process died. while True: # Read the next line try: line = self.readline() print line except KeyboardInterrupt: thread.interrupt_main() except: logging.error("%s failed with %s" % (self.instrument, traceback.format_exc())) continue # Interpret the next line try: self.lock.acquire() self.parseline(line) except KeyboardInterrupt: thread.interrupt_main() except: logging.error("%s failed with %s\nLine: %s" % (self.instrument, traceback.format_exc(), self.lastline)) finally: self.lock.release()
def do_GET(self): """Serve a GET request.""" parsed_url = urlparse.urlparse(self.path) params = urlparse.parse_qs(parsed_url.query) path = parsed_url.path if path.startswith('/stop'): # stop children before we return a response timeout = int((params.get("timeout", []) or ["30"])[0]) self.funkyserver.pre_stop(timeout=timeout) t = self.get_response_text() if self.send_head(t): self.wfile.write(t) #I preempt the finish operation here so that processing of this request is all done before we crash or whatever: self.finish() self.http_server.shutdown_request(self.connection) if path.startswith('/crash'): crash() if path.startswith('/stop'): self.funkyserver.stop() if path.startswith('/interrupt_main'): thread.interrupt_main() if path.startswith('/exit'): os._exit(1) if path.startswith('/hold_gil'): t = int((params.get("t", []) or ["100"])[0]) hold_gil(t)
def __init__(self, host, port, username, password): self.origin = Location() self.origin.set(130, 71, 83) self.range = Location() self.range.set(10, 10, 10) self.X = 0 self.Y = 0 self.Z = 0 self.bot = Bot() self.dispatch = PacketDispatch(self) self.buff = Buffer() self.packetSend = [] self.sendPacket = PacketSend(self) self.printable = True self.receiveSize = 1024 self.compressionThreshold = -1 self.username = username self.HOST = host self.PORT = port self.s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) self.s.settimeout(None) try: self.s.connect((self.HOST, self.PORT)) except socket.error: print("Connection refused") thread.interrupt_main() exit()
def _handle_test_status_file(self, test_name, test_phase, success): ########################################################################### # # This complexity is due to sharing of TestStatus responsibilities # try: if (test_phase != RUN_PHASE and (not success or test_phase == BUILD_PHASE or test_phase == self._phases[-1])): self._update_test_status_file(test_name) # If we failed VERY early on in the run phase, it's possible that # the CIME scripts never got a chance to set the state. elif (test_phase == RUN_PHASE and not success): test_status_file = os.path.join(self._get_test_dir(test_name), TEST_STATUS_FILENAME) statuses = wait_for_tests.parse_test_status_file(test_status_file)[0] if ( RUN_PHASE not in statuses or statuses[RUN_PHASE] in [TEST_PASS_STATUS, TEST_PENDING_STATUS] ): self._update_test_status_file(test_name) except Exception as e: # TODO: What to do here? This failure is very severe because the # only way for test results to be communicated is by the TestStatus # file. warning("VERY BAD! Could not handle TestStatus file '%s': '%s'" % (test_status_file, str(e))) thread.interrupt_main()
def handle_error(self, request, client_address): """Override RPCServer method for IDLE Interrupt the MainThread and exit server if link is dropped. """ global quitting try: raise except SystemExit: raise except EOFError: global exit_now exit_now = True thread.interrupt_main() except: erf = sys.__stderr__ print>>erf, '\n' + '-'*40 print>>erf, 'Unhandled server exception!' print>>erf, 'Thread: %s' % threading.currentThread().getName() print>>erf, 'Client Address: ', client_address print>>erf, 'Request: ', repr(request) traceback.print_exc(file=erf) print>>erf, '\n*** Unrecoverable, server exiting!' print>>erf, '-'*40 quitting = True thread.interrupt_main()
def check_active_loggers(self): """Check if all loggers are deactivated due to I/O errors.""" for logger in self.loggers: if logger.is_active: break else: thread.interrupt_main()
def f(): for x in range(5): if waiting: thread.interrupt_main() return print 'tock...', x # <-force the GIL to be released, as time.sleep(0.1) # time.sleep doesn't do non-translated
def HTTPreq(self,url,data=None,headers=None): opener=self.opener kargs={} kargs['url']=url if data: kargs['data']=data #Will do a GET if no data else POST if headers: kargs['headers']=headers else: kargs['headers']={'Content-Type':'application/octet-stream'} #Make Request f=opener.open(urllib2.Request(**kargs)) #If response is gzip encoded if ('Content-Encoding' in f.info().keys() and f.info()['Content-Encoding']=='gzip') or \ ('content-encoding' in f.info().keys() and f.info()['content-encoding']=='gzip'): url_f = StringIO.StringIO(f.read()) data = gzip.GzipFile(fileobj=url_f).read() else: #response not encoded data = f.read() if f.getcode() != 200: print "[-] Received status code " + str(f.getcode()) print data thread.interrupt_main() return data #Return response
def _test_main_thread(): cherrypy.server.wait() cherrypy._cputil._cpInitDefaultFilters() try: webtest.main() finally: thread.interrupt_main()
def _serve_forever(self): LOG.debug('Starting plotter') try: cherrypy.quickstart(self, '/', {'/': {}}) except: LOG.error(helper.exc_info()) thread.interrupt_main()
def callback(): global thread, s, peersData, my_ip, host, port if tkMessageBox.askokcancel("Quit", "Do you really wish to quit?"): s.sendto("exit", (host, port)) s.close() root.destroy() thread.interrupt_main()
def run(self): try: super(DDPClient, self).run() except: traceback.print_exc() finally: thread.interrupt_main()
def _process_queue(queue): time.sleep(2) while True: # Block until we have something to plot #logging.debug("get next plot") ready = [queue.get()] # Get all queued requests, deleting the duplicates try: while True: # TODO: don't know how to make this interruptible #logging.debug("empty queue") next = queue.get_nowait() queue.task_done() if next not in ready: ready.append(next) except KeyboardInterrupt: thread.interrupt_main() except Queue.Empty: pass # Make all the plots #logging.debug("plotting %d"%len(ready)) for stream in ready: logging.debug("plotting "+stream.instrument) stream.plot() # Be nice to the server --- batch plot requests every 30 seconds #logging.debug("sleep %s"%str(PLOT_INTERVAL)) time.sleep(PLOT_INTERVAL)
def worker(): try: workerRun() except Exception, e: print sys.exc_info() M6Item.status = 'STOPPED' thread.interrupt_main()
def stop_code(self, quiet=False): """ Attempt to stop the running code by raising a keyboard interrupt in the main thread. If the optional quiet flag is True the engine will stop the code but not print an error message or prompt again. """ if self.busy is False: return #set the stop flag to catch the error correctly if quiet: self._stop_quiet = True else: self._stop = True #make sure we are not stuck in readline(s) if self._isreading: self._readevent.set() #make sure the debugger is not paused. if self.debug is True: self.debugger.stop_code() return #try a keyboard interrupt - this will not work for the internal engine # as the error is raised here instead of the running code, hence put in #try clause. try: thread.interrupt_main() except: pass
def run(self): """Start event listener""" try_interval = 1 while self.stop_event.is_set() is False: try: try_interval *= 2 with self._celery_app.connection() as conn: handlers = { "task-succeeded": (self._engine._task_success_handler), "task-failed": (self._engine._task_failure_handler), } recv = events.EventReceiver(conn, handlers=handlers, app=self._celery_app) while self.stop_event.is_set() is False: recv.capture(limit=None, timeout=None) self._running = True try_interval = 1 except (KeyboardInterrupt, SystemExit): thread.interrupt_main() except Exception as e: self._running = False LOG.error("Failed to capture events: '%s', " "trying again in %s seconds." % (e, try_interval)) LOG.debug(e, exc_info=True) time.sleep(try_interval)
def decompte(self, sec=90., dt=.1): endTime = time.time() + sec while True: novTime = time.time() if novTime >= endTime: break if self.tirette.armee(): self.printLog( "sortie du programme sur remise de la tirette apres %.2fs"%(sec-(endTime-novTime)) ) import sys sys.exit(1) #raise KeyboardInterrupt(message) time.sleep(dt) self.disableActuators() self.disableActuators() self.disablePower() self.disablePower() time.sleep(0.5) self.disableActuators() self.disableActuators() self.disablePower() self.disablePower() self.printLog( "fin du temps imparti: %.2fs"%sec ) thread.interrupt_main()
def listen(new_socket, filename, extension, part, size): #listen on the random port new_socket.listen(1) #accept incoming connections connection, address = new_socket.accept() full_filename = filename + '.' + extension #get username username = os.getlogin() #fully qualify path file_path = '/home/' + username + '/p2p/files/' + full_filename #open the file for reading FILE = open(file_path, 'rb') #read the desired parts FILE.seek(part * 512000) #set up data buffer for sending data send_data = FILE.read(size) #close up file FILE.close() #send out your data connection.send(send_data) #close up connection connection.close() #close up new_socket.close() #stop thread thread.interrupt_main()
def update_config(self): if self._restart: print >>sys.stderr, "[hosted.py] restarting service (restart_on_update set)" import thread, time thread.interrupt_main() time.sleep(100) return def parse_recursive(options, config, target): # print 'parsing', config for option in options: if not 'name' in option: continue if option['type'] == 'list': items = [] for item in config[option['name']]: parsed = {} parse_recursive(option['items'], item, parsed) items.append(parsed) target[option['name']] = items continue target[option['name']] = types[option['type']](config[option['name']]) parsed = {} parse_recursive(self._options, self._config, parsed) print >>sys.stderr, "[hosted.py] updated config" self._parsed = parsed
def Pinging_Thread(self): print "[+] Starting Ping thread" #self.ptc=threading.Condition() wait=True p=0.1 while 1: #loop forever if wait and (self.ping_delay > 0): self.ptc.acquire() self.ptc.wait(self.ping_delay+self.penalty) #send ping to server interval + penalty self.ptc.release() self.mutex_http_req.acquire() #Ensure that the other thread is not making a request at this time try: resp_data=self.http.HTTPreq(self.url,"") #Read response if self.verbose: self.http.v_print(pings_n=1) if self.penalty<60: self.penalty+=p #Don't wait more than a minute if resp_data: #If response had data write them to socket self.penalty=0 if self.verbose: self.http.v_print(received_d_pt=len(resp_data)) self.TunnaSocket.send(resp_data) #write to socket resp_data="" #clear data wait=False #If data received don't wait else: wait=True except: self.TunnaSocket.close() thread.exit() finally: self.mutex_http_req.release() print "[-] Pinging Thread Exited" #Unrecoverable thread.interrupt_main() #Interupt main thread -> exits
def receive_service(self): # Receive data from server. while True: try: # Recieve Message. line = self.network_client_socket.recv(2048) if (line != '\n'): if((len(line.split(":")) >= 2) and (line.split(":")[0].strip().startswith("LINE"))): self.gui_send_service(line) elif((len(line.split(":")) == 2) and (line.split(":")[0].strip() == "JOINED")): self.create_group_gui(line.split(":")[1].strip()) else: print("\n %s" % line) if not line: print("Server closed connection, thread exiting.") thread.interrupt_main() self.quit() break except: print("Recieve Error: Thread exiting.") thread.interrupt_main() self.quit() break
def run(self): """ Do not call this directly; call ``start()`` instead. """ while not self.halt: # print 'Heartbeat check...' self.lock_file.seek(0) self.lock_file.write(str(time.time())) self.lock_file.flush() # Check job status and save heartbeat timestamp. self.lock.acquire() Job.objects.update() job = Job.objects.get(id=self.job_id) job.last_heartbeat = datetime.now() force_stop = job.force_stop job.force_stop = False job.force_run = False job.save() self.lock.release() # If we noticed we're being forced to stop, then interrupt main. if force_stop: self.halt = True thread.interrupt_main() return time.sleep(10) set_current_heartbeat(None)
def wait(secs): for n in xrange(timeout_secs): time.sleep(1) if signal_finished: break else: thread.interrupt_main()
def receiverThread(cjdns): timeOfLastSend = time.time(); timeOfLastRecv = time.time(); try: while True: if (timeOfLastSend + KEEPALIVE_INTERVAL_SECONDS < time.time()): if (timeOfLastRecv + 10 < time.time()): raise Exception("ping timeout"); cjdns.socket.send('d1:q18:Admin_asyncEnabled4:txid8:keepalive'); timeOfLastSend = time.time(); try: data = cjdns.socket.recv(BUFFER_SIZE); except (socket.timeout): continue; try: benc = bdecode(data); except (KeyError, ValueError): print("error decoding [" + data + "]"); continue; if benc['txid'] == 'keepaliv': if benc['asyncEnabled'] == 0: raise Exception("lost session"); timeOfLastRecv = time.time(); else: #print "putting to queue " + str(benc); cjdns.queue.put(benc); except KeyboardInterrupt: print("interrupted"); import thread thread.interrupt_main();
def run(self): """ Run the poll loop. This method never returns. """ from _subprocess import WAIT_OBJECT_0, INFINITE # Build the list of handle to listen on. handles = [] if self.interrupt_handle: handles.append(self.interrupt_handle) if self.parent_handle: handles.append(self.parent_handle) # Listen forever. while True: result = ctypes.windll.kernel32.WaitForMultipleObjects( len(handles), # nCount (ctypes.c_int * len(handles))(*handles), # lpHandles False, # bWaitAll INFINITE) # dwMilliseconds if WAIT_OBJECT_0 <= result < len(handles): handle = handles[result - WAIT_OBJECT_0] if handle == self.interrupt_handle: raw_print('Interrupted by parent poller!') interrupt_main() elif handle == self.parent_handle: raw_print('Killed by parent poller!') os._exit(1)
def cleanup(self, state): try: state.app.session.stop() finally: super(MainProcess, self).cleanup(state) # signal the main thread that MainProcess exited thread.interrupt_main()
def decode_interrupthook(self): "interrupt awakened thread" global quitting quitting = True thread.interrupt_main()
def EOFhook(self): "Override SocketIO method - terminate wait on callback and exit thread" global quitting quitting = True thread.interrupt_main()
def run(self): """Starts the process when Process.start() is called. """ global JOB_CHECK_INTERVAL # make sure to catch sys.exit (which raises SystemExit) try: # Get directory where this module lives mod_dir = os.path.dirname(os.path.realpath(__file__)) # Try to connect to the database try: db = self._session.get_db() pilot_col = db["%s.p" % self._session.uid] logger.debug( "Connected to MongoDB. Serving requests for PilotManager %s." % self.pilot_manager_id) except Exception as e: logger.exception("Connection error: %s" % e) return last_job_check = time.time() while not self._terminate.is_set(): # Periodically, we pull up all ComputePilots that are pending # execution or were last seen executing and check if the corresponding # SAGA job is still pending in the queue. If that is not the case, # we assume that the job has failed for some reasons and update # the state of the ComputePilot accordingly. if last_job_check + JOB_CHECK_INTERVAL < time.time(): last_job_check = time.time() self.check_pilot_states(pilot_col) if self._disabled.is_set(): # don't process any new pilot start requests. # NOTE: this is not clean, in principle there could be other # launchers alive which want to still start those # pending pilots. In practice we only ever use one # pmgr though, and its during its shutdown that we get # here... ts = timestamp() compute_pilot = pilot_col.find_and_modify( query={ "pilotmanager": self.pilot_manager_id, "state": PENDING_LAUNCH }, update={ "$set": { "state": CANCELED }, "$push": { "statehistory": { "state": CANCELED, "timestamp": ts } } }) # run state checks more frequently. JOB_CHECK_INTERVAL = 3 time.sleep(1) continue # See if we can find a ComputePilot that is waiting to be launched. # If we find one, we use SAGA to create a job service, a job # description and a job that is then send to the local or remote # queueing system. If this succedes, we set the ComputePilot's # state to pending, otherwise to failed. compute_pilot = None ts = timestamp() compute_pilot = pilot_col.find_and_modify( query={ "pilotmanager": self.pilot_manager_id, "state": PENDING_LAUNCH }, update={ "$set": { "state": LAUNCHING }, "$push": { "statehistory": { "state": LAUNCHING, "timestamp": ts } } }) if not compute_pilot: time.sleep(IDLE_TIMER) else: try: # ------------------------------------------------------ # # LAUNCH THE PILOT AGENT VIA SAGA # logentries = [] pilot_id = str(compute_pilot["_id"]) logger.info("Launching ComputePilot %s" % pilot_id) # ------------------------------------------------------ # Database connection parameters session_id = self._session.uid database_url = self._session.dburl # ------------------------------------------------------ # pilot description and resource configuration number_cores = compute_pilot['description']['cores'] runtime = compute_pilot['description']['runtime'] queue = compute_pilot['description']['queue'] project = compute_pilot['description']['project'] cleanup = compute_pilot['description']['cleanup'] resource_key = compute_pilot['description']['resource'] schema = compute_pilot['description']['access_schema'] memory = compute_pilot['description']['memory'] pilot_sandbox = compute_pilot['sandbox'] global_sandbox = compute_pilot['global_sandbox'] # we expand and exchange keys in the resource config, # depending on the selected schema so better use a deep # copy.. resource_cfg = self._session.get_resource_config( resource_key, schema) # import pprint # pprint.pprint (resource_cfg) # ------------------------------------------------------ # get parameters from cfg, set defaults where needed agent_launch_method = resource_cfg.get( 'agent_launch_method') agent_dburl = resource_cfg.get( 'agent_mongodb_endpoint', database_url) agent_spawner = resource_cfg.get( 'agent_spawner', DEFAULT_AGENT_SPAWNER) agent_type = resource_cfg.get('agent_type', DEFAULT_AGENT_TYPE) rc_agent_config = resource_cfg.get( 'agent_config', DEFAULT_AGENT_CONFIG) agent_scheduler = resource_cfg.get('agent_scheduler') tunnel_bind_device = resource_cfg.get( 'tunnel_bind_device') default_queue = resource_cfg.get('default_queue') forward_tunnel_endpoint = resource_cfg.get( 'forward_tunnel_endpoint') js_endpoint = resource_cfg.get('job_manager_endpoint') lrms = resource_cfg.get('lrms') mpi_launch_method = resource_cfg.get( 'mpi_launch_method') pre_bootstrap_1 = resource_cfg.get('pre_bootstrap_1') pre_bootstrap_2 = resource_cfg.get('pre_bootstrap_2') python_interpreter = resource_cfg.get( 'python_interpreter') spmd_variation = resource_cfg.get('spmd_variation') task_launch_method = resource_cfg.get( 'task_launch_method') rp_version = resource_cfg.get('rp_version', DEFAULT_RP_VERSION) virtenv_mode = resource_cfg.get( 'virtenv_mode', DEFAULT_VIRTENV_MODE) virtenv = resource_cfg.get('virtenv', DEFAULT_VIRTENV) stage_cacerts = resource_cfg.get( 'stage_cacerts', 'False') cores_per_node = resource_cfg.get('cores_per_node') # Agent configuration that is not part of the public API. # The agent config can either be a config dict, or # a string pointing to a configuration name. If neither # is given, check if 'RADICAL_PILOT_AGENT_CONFIG' is # set. The last fallback is 'agent_default' agent_config = compute_pilot['description'].get( '_config') if not agent_config: agent_config = os.environ.get( 'RADICAL_PILOT_AGENT_CONFIG') if not agent_config: agent_config = rc_agent_config if isinstance(agent_config, dict): # nothing to do agent_cfg_dict = agent_config pass elif isinstance(agent_config, basestring): try: if os.path.exists(agent_config): # try to open as file name logger.info("Read agent config file: %s" % agent_config) agent_cfg_dict = ru.read_json(agent_config) else: # otherwise interpret as a config name # FIXME: load in session just like resource # configs, including user level overloads module_path = os.path.dirname( os.path.abspath(__file__)) config_path = "%s/../configs/" % module_path agent_cfg_file = os.path.join( config_path, "agent_%s.json" % agent_config) logger.info("Read agent config file: %s" % agent_cfg_file) agent_cfg_dict = ru.read_json( agent_cfg_file) except Exception as e: logger.exception( "Error reading agent config file: %s" % e) raise else: # we can't handle this type raise TypeError( 'agent config must be string (filename) or dict' ) # TODO: use booleans all the way? if stage_cacerts.lower() == 'true': stage_cacerts = True else: stage_cacerts = False # expand variables in virtenv string virtenv = virtenv % { 'pilot_sandbox': saga.Url(pilot_sandbox).path, 'global_sandbox': saga.Url(global_sandbox).path } # Check for deprecated global_virtenv global_virtenv = resource_cfg.get('global_virtenv') if global_virtenv: logger.warn( "'global_virtenv' keyword is deprecated -- use 'virtenv' and 'virtenv_mode'" ) virtenv = global_virtenv virtenv_mode = 'use' # Create a host:port string for use by the bootstrap_1. db_url = saga.Url(agent_dburl) if db_url.port: db_hostport = "%s:%d" % (db_url.host, db_url.port) else: db_hostport = "%s:%d" % (db_url.host, 27017 ) # mongodb default # Open the remote sandbox sandbox_tgt = saga.filesystem.Directory( pilot_sandbox, session=self._session, flags=saga.filesystem.CREATE_PARENTS) BOOTSTRAPPER_SCRIPT = "bootstrap_1.sh" LOCAL_SCHEME = 'file' # ------------------------------------------------------ # Copy the bootstrap shell script. This also creates # the sandbox. We use always "default_bootstrapper.sh" # TODO: Is this still configurable and/or in the resource configs? bootstrapper = "default_bootstrapper.sh" bootstrapper_path = os.path.abspath("%s/../bootstrapper/%s" \ % (mod_dir, bootstrapper)) msg = "Using bootstrapper %s" % bootstrapper_path logentries.append(Logentry(msg, logger=logger.info)) bs_script_url = saga.Url( "%s://localhost%s" % (LOCAL_SCHEME, bootstrapper_path)) msg = "Copying bootstrapper '%s' to agent sandbox (%s)." \ % (bs_script_url, sandbox_tgt) logentries.append(Logentry(msg, logger=logger.debug)) sandbox_tgt.copy(bs_script_url, BOOTSTRAPPER_SCRIPT) # ------------------------------------------------------ # the version of the agent is derived from # rp_version, which has the following format # and interpretation: # # case rp_version: # @<token>: # @tag/@branch/@commit: # no sdist staging # git clone $github_base radical.pilot.src # (cd radical.pilot.src && git checkout token) # pip install -t $VIRTENV/rp_install/ radical.pilot.src # rm -rf radical.pilot.src # export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH # # release: # no sdist staging # pip install -t $VIRTENV/rp_install radical.pilot # export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH # # local: # needs sdist staging # tar zxf $sdist.tgz # pip install -t $VIRTENV/rp_install $sdist/ # export PYTHONPATH=$VIRTENV/rp_install:$PYTHONPATH # # debug: # needs sdist staging # tar zxf $sdist.tgz # pip install -t $SANDBOX/rp_install $sdist/ # export PYTHONPATH=$SANDBOX/rp_install:$PYTHONPATH # # installed: # no sdist staging # true # esac # # virtenv_mode # private : error if ve exists, otherwise create, then use # update : update if ve exists, otherwise create, then use # create : use if ve exists, otherwise create, then use # use : use if ve exists, otherwise error, then exit # recreate: delete if ve exists, otherwise create, then use # # examples : # [email protected] # virtenv@devel # virtenv@release # virtenv@installed # stage@local # stage@/tmp/my_agent.py # # Note that some combinations may be invalid, # specifically in the context of virtenv_mode. If, for # example, virtenv_mode is 'use', then the 'virtenv:tag' # will not make sense, as the virtenv is not updated. # In those cases, the virtenv_mode is honored, and # a warning is printed. # # Also, the 'stage' mode can only be combined with the # 'local' source, or with a path to the agent (relative # to mod_dir, or absolute). # # A rp_version which does not adhere to the # above syntax is ignored, and the fallback stage@local # is used. if not rp_version.startswith('@') and \ not rp_version in ['installed', 'local', 'debug']: raise ValueError("invalid rp_version '%s'" % rp_version) stage_sdist = True if rp_version in ['installed', 'release']: stage_sdist = False if rp_version.startswith('@'): stage_sdist = False rp_version = rp_version[1:] # strip '@' # ------------------------------------------------------ # Copy the rp sdist if needed. We actually also stage # the sdists for radical.utils and radical.saga, so that # we have the complete stack to install... if stage_sdist: for sdist_path in [ ru.sdist_path, saga.sdist_path, rp_sdist_path ]: sdist_url = saga.Url( "%s://localhost%s" % (LOCAL_SCHEME, sdist_path)) msg = "Copying sdist '%s' to sandbox (%s)." % ( sdist_url, pilot_sandbox) logentries.append( Logentry(msg, logger=logger.debug)) sandbox_tgt.copy( sdist_url, os.path.basename(str(sdist_url))) # ------------------------------------------------------ # Some machines cannot run pip due to outdated CA certs. # For those, we also stage an updated certificate bundle if stage_cacerts: cc_path = os.path.abspath("%s/../bootstrapper/%s" \ % (mod_dir, 'cacert.pem.gz')) cc_url = saga.Url("%s://localhost/%s" % (LOCAL_SCHEME, cc_path)) msg = "Copying CA certificate bundle '%s' to sandbox (%s)." % ( cc_url, pilot_sandbox) logentries.append( Logentry(msg, logger=logger.debug)) sandbox_tgt.copy(cc_url, os.path.basename(str(cc_url))) # ------------------------------------------------------ # sanity checks if not agent_spawner: raise RuntimeError("missing agent spawner") if not agent_scheduler: raise RuntimeError("missing agent scheduler") if not lrms: raise RuntimeError("missing LRMS") if not agent_launch_method: raise RuntimeError("missing agentlaunch method") if not task_launch_method: raise RuntimeError("missing task launch method") if not mpi_launch_method: raise RuntimeError("missing mpi launch method") # massage some values if not queue: queue = default_queue if cleanup and isinstance(cleanup, bool): cleanup = 'luve' # l : log files # u : unit work dirs # v : virtualenv # e : everything (== pilot sandbox) # # we never cleanup virtenvs which are not private if virtenv_mode is not 'private': cleanup = cleanup.replace('v', '') sdists = ':'.join( [ru.sdist_name, saga.sdist_name, rp_sdist_name]) # if cores_per_node is set (!= None), then we need to # allocation full nodes, and thus round up if cores_per_node: cores_per_node = int(cores_per_node) number_cores = int(cores_per_node * math.ceil( float(number_cores) / cores_per_node)) # set mandatory args bootstrap_args = "" bootstrap_args += " -d '%s'" % sdists bootstrap_args += " -m '%s'" % virtenv_mode bootstrap_args += " -p '%s'" % pilot_id bootstrap_args += " -r '%s'" % rp_version bootstrap_args += " -s '%s'" % session_id bootstrap_args += " -v '%s'" % virtenv # set optional args if agent_type: bootstrap_args += " -a '%s'" % agent_type if lrms == "CCM": bootstrap_args += " -c" if pre_bootstrap_1: bootstrap_args += " -e '%s'" % "' -e '".join( pre_bootstrap_1) if pre_bootstrap_2: bootstrap_args += " -w '%s'" % "' -w '".join( pre_bootstrap_2) if forward_tunnel_endpoint: bootstrap_args += " -f '%s'" % forward_tunnel_endpoint if forward_tunnel_endpoint: bootstrap_args += " -h '%s'" % db_hostport if python_interpreter: bootstrap_args += " -i '%s'" % python_interpreter if tunnel_bind_device: bootstrap_args += " -t '%s'" % tunnel_bind_device if cleanup: bootstrap_args += " -x '%s'" % cleanup # set some agent configuration agent_cfg_dict['cores'] = number_cores agent_cfg_dict['debug'] = os.environ.get( 'RADICAL_PILOT_AGENT_VERBOSE', logger.getEffectiveLevel()) agent_cfg_dict['mongodb_url'] = str(agent_dburl) agent_cfg_dict['lrms'] = lrms agent_cfg_dict['spawner'] = agent_spawner agent_cfg_dict['scheduler'] = agent_scheduler agent_cfg_dict['runtime'] = runtime agent_cfg_dict['pilot_id'] = pilot_id agent_cfg_dict['session_id'] = session_id agent_cfg_dict[ 'agent_launch_method'] = agent_launch_method agent_cfg_dict[ 'task_launch_method'] = task_launch_method agent_cfg_dict['mpi_launch_method'] = mpi_launch_method if cores_per_node: agent_cfg_dict['cores_per_node'] = cores_per_node # ------------------------------------------------------ # Write agent config dict to a json file in pilot sandbox. cfg_tmp_handle, cf_tmp_file = tempfile.mkstemp( suffix='.json', prefix='rp_agent_cfg_') # Convert dict to json file msg = "Writing agent configuration to file '%s'." % cf_tmp_file logentries.append(Logentry(msg, logger=logger.debug)) ru.write_json(agent_cfg_dict, cf_tmp_file) cf_url = saga.Url("%s://localhost%s" % (LOCAL_SCHEME, cf_tmp_file)) msg = "Copying agent configuration file '%s' to sandbox (%s)." % ( cf_url, pilot_sandbox) logentries.append(Logentry(msg, logger=logger.debug)) sandbox_tgt.copy(cf_url, 'agent_0.cfg') # close and remove temp file os.close(cfg_tmp_handle) os.unlink(cf_tmp_file) # ------------------------------------------------------ # Done with all transfers to pilot sandbox, close handle sandbox_tgt.close() # ------------------------------------------------------ # now that the scripts are in place and configured, # we can launch the agent js_url = saga.Url(js_endpoint) logger.debug("saga.job.Service ('%s')" % js_url) if js_url in self._shared_worker_data['job_services']: js = self._shared_worker_data['job_services'][ js_url] else: js = saga.job.Service(js_url, session=self._session) self._shared_worker_data['job_services'][ js_url] = js # ------------------------------------------------------ # Create SAGA Job description and submit the pilot job jd = saga.job.Description() jd.executable = "/bin/bash" jd.arguments = [ "-l %s" % BOOTSTRAPPER_SCRIPT, bootstrap_args ] jd.working_directory = saga.Url(pilot_sandbox).path jd.project = project jd.output = "bootstrap_1.out" jd.error = "bootstrap_1.err" jd.total_cpu_count = number_cores jd.processes_per_host = cores_per_node jd.wall_time_limit = runtime jd.total_physical_memory = memory jd.queue = queue jd.environment = dict() # Set the SPMD variation only if required if spmd_variation: jd.spmd_variation = spmd_variation if 'RADICAL_PILOT_PROFILE' in os.environ: jd.environment['RADICAL_PILOT_PROFILE'] = 'TRUE' logger.debug("Bootstrap command line: %s %s" % (jd.executable, jd.arguments)) msg = "Submitting SAGA job with description: %s" % str( jd.as_dict()) logentries.append(Logentry(msg, logger=logger.debug)) pilotjob = js.create_job(jd) pilotjob.run() # do a quick error check if pilotjob.state == saga.FAILED: raise RuntimeError("SAGA Job state is FAILED.") saga_job_id = pilotjob.id self._shared_worker_data['job_ids'][pilot_id] = [ saga_job_id, js_url ] msg = "SAGA job submitted with job id %s" % str( saga_job_id) logentries.append(Logentry(msg, logger=logger.debug)) # # ------------------------------------------------------ log_dicts = list() for le in logentries: log_dicts.append(le.as_dict()) # Update the Pilot's state to 'PENDING_ACTIVE' if SAGA job submission was successful. ts = timestamp() ret = pilot_col.update( { "_id": pilot_id, "state": LAUNCHING }, { "$set": { "state": PENDING_ACTIVE, "saga_job_id": saga_job_id, "agent_config": agent_cfg_dict }, "$push": { "statehistory": { "state": PENDING_ACTIVE, "timestamp": ts } }, "$pushAll": { "log": log_dicts } }) if ret['n'] == 0: # could not update, probably because the agent is # running already. Just update state history and # jobid then # FIXME: make sure of the agent state! ret = pilot_col.update({"_id": pilot_id}, { "$set": { "saga_job_id": saga_job_id }, "$push": { "statehistory": { "state": PENDING_ACTIVE, "timestamp": ts } }, "$pushAll": { "log": log_dicts } }) except Exception as e: # Update the Pilot's state 'FAILED'. out, err, log = self._get_pilot_logs( pilot_col, pilot_id) ts = timestamp() # FIXME: we seem to be unable to bson/json handle saga # log messages containing an '#'. This shows up here. # Until we find a clean workaround, make log shorter and # rely on saga logging to reveal the problem. msg = "Pilot launching failed! (%s)" % e logentries.append(Logentry(msg)) log_dicts = list() log_messages = list() for le in logentries: log_dicts.append(le.as_dict()) log_messages.append(le.message) pilot_col.update( { "_id": pilot_id, "state": { "$ne": FAILED } }, { "$set": { "state": FAILED, "stdout": out, "stderr": err, "logfile": log }, "$push": { "statehistory": { "state": FAILED, "timestamp": ts } }, "$pushAll": { "log": log_dicts } }) logger.exception('\n'.join(log_messages)) except SystemExit as e: logger.exception( "pilot launcher thread caught system exit -- forcing application shutdown" ) import thread thread.interrupt_main()
PTP_GUID = "00:00:00:00:00:00:00:00:ff:ff:08:00:27:f5:16:4f" DEBUG = False #check for commandline args for i, arg in enumerate(sys.argv): #Let commandline arg override config if arg == '-d': DEBUG = True #Set logging file elif arg == '-l': if len(sys.argv) > i + 1: sys.stdout = open(sys.argv[i + 1], 'w') sys.stderr = open(sys.argv[i + 1], 'w') else: print "Please provide path the log file after -l" thread.interrupt_main() #exiting program #Setup Logging Output logFormatter = logging.Formatter("%(asctime)s [%(levelname)-5.5s] %(message)s") L = logging.getLogger() if DEBUG is True: L.setLevel(logging.DEBUG) else: L.setLevel(logging.INFO) consoleHandler = logging.StreamHandler(sys.stdout) consoleHandler.setFormatter(logFormatter) L.addHandler(consoleHandler) L.info("Server starting")
def traction_Orders_Callback(param): pass ### START XBEE CODE ### if rf_wifi_selector==1: try: xBeeSerial = serial.Serial(DIRECCION_XBEE, baudrate=BAUD_RATE) print("\t \t -- Conexion con XBee S8 inicializada --") except: print("\033[1;31mNo se pudo inicializar la conexion con la XBee S8 \033[0;0m") thread.interrupt_main() if rf_wifi_selector==2: s = socket.socket() s.settimeout(1) try: s.connect((HOST, PORT)) s.send(str.encode("C+RF")) except: print("\033[1;31mError en conexion Wi-Fi \033[0;0m") _thread.interrupt_main() s.close() if order_connect== 0: print("\033[1;31mNo estan activas todas las conexiones, encontrara funcionalidad limitada \033[0;0m") print("\t \t -- Modo de control RF --") ### Mensaje de inicializacion ###
def master_connection_init(conn): while 1: i=recvo(conn) if i==2: break thread.interrupt_main()
def handler(): debug("Shutting down HTTP server via timeout (%d) seconds." % (ARGS['timeout'])) thread.interrupt_main()
def command(com, args, tw_user=None): global e if (e.is_set()): # System Tray Icon exit must trigger this exit(0) if not com or not isinstance(com, str): return False original_com = com global inactive global user_full_name global user_prefix global config_file userin.twitter_user = tw_user com = com.upper() com = re.sub(r'([^\s\w]|_)+', '', com).strip() print("You: " + com) if inactive and com not in ("DRAGONFIRE", "DRAGON FIRE", "WAKE UP", "HEY"): return True if USER_ANSWERING['status']: if com.startswith("FIRST") or com.startswith( "THE FIRST") or com.startswith("SECOND") or com.startswith( "THE SECOND") or com.startswith( "THIRD") or com.startswith("THE THIRD"): USER_ANSWERING['status'] = False selection = None if com.startswith("FIRST") or com.startswith("THE FIRST"): selection = 0 elif com.startswith("SECOND") or com.startswith("THE SECOND"): selection = 1 elif com.startswith("THIRD") or com.startswith("THE THIRD"): selection = 2 if USER_ANSWERING['for'] == 'wikipedia': with nostderr(): search_query = USER_ANSWERING['options'][selection] try: wikiresult = wikipedia.search(search_query) if len(wikiresult) == 0: userin.say( "Sorry, " + user_prefix + ". But I couldn't find anything about " + search_query + " in Wikipedia.") return True wikipage = wikipedia.page(wikiresult[0]) wikicontent = "".join([ i if ord(i) < 128 else ' ' for i in wikipage.content ]) wikicontent = re.sub(r'\([^)]*\)', '', wikicontent) userin.define_and_execute( ["sensible-browser", wikipage.url], search_query) userin.say(wikicontent) return True except requests.exceptions.ConnectionError: userin.define_and_execute( [" "], "Wikipedia connection error.") userin.say( "Sorry, " + user_prefix + ". But I'm unable to connect to Wikipedia servers.") return True except Exception: return True if com in ("DRAGONFIRE", "DRAGON FIRE", "WAKE UP", "HEY"): tts_kill() inactive = False userin.define([" "], " ") userin.say(choice([ "Yes, " + user_prefix + ".", "Yes. I'm waiting.", "What is your order?", "Ready for the orders!", user_prefix + ", tell me your wish." ])) elif "GO TO SLEEP" == com: tts_kill() inactive = True userin.define_and_execute( ["echo"], "Dragonfire deactivated. To reactivate say 'Dragonfire!' or 'Wake Up!'") userin.say("I'm going to sleep") elif com in ("ENOUGH", "SHUT UP"): print("Dragonfire quiets.") tts_kill() elif VirtualAssistant.exact_match(com): return True # the request has been handled elif VirtualAssistant.in_match(com): return True # the request has been handled elif ("SEARCH" in com or "FIND" in com) and VirtualAssistant.search_command(com): pass # the request has been handled elif com in ("MY TITLE IS LADY", "I'M A LADY", "I'M A WOMAN", "I'M A GIRL"): tts_kill() config_file.update({ 'gender': 'female' }, Query().datatype == 'gender') user_prefix = "My Lady" userin.define([" "], " ") userin.say("Pardon, " + user_prefix + ".") elif com in ("MY TITLE IS SIR", "I'M A MAN", "I'M A BOY"): tts_kill() config_file.update({ 'gender': 'male' }, Query().datatype == 'gender') user_prefix = "Sir" userin.define([" "], " ") userin.say("Pardon, " + user_prefix + ".") elif com.startswith("CALL ME "): tts_kill() callme_config = config_file.search(Query().datatype == 'callme') if callme_config: config_file.update({ 'title': original_com[8:].lower() }, Query().datatype == 'callme') else: config_file.insert({ 'datatype': 'callme', 'title': original_com[8:].lower() }) user_prefix = original_com[8:].lower().encode("utf8") userin.define([" "], " ") userin.say("Pardon, " + user_prefix + ".") # only for The United States today but prepared for all countries. Also # only for celsius degrees today. --> by Radan Liska :-) elif "WHAT" in com and "TEMPERATURE" in com: tts_kill() capture = re.search( "(?:WHAT IS|WHAT'S) THE TEMPERATURE (?:IN|ON|AT|OF)? (?P<city>.*)", com) if capture: city = capture.group('city') owm = pyowm.OWM("16d66c84e82424f0f8e62c3e3b27b574") reg = owm.city_id_registry() weather = owm.weather_at_id( reg.ids_for(city)[0][0]).get_weather() fmt = "The temperature in {} is {} degrees celsius" msg = fmt.format(city, weather.get_temperature('celsius')['temp']) userin.define_and_execute([" "], msg) userin.say(msg) elif "FILE MANAGER" in com or "OPEN FILES" == com: tts_kill() userin.define_and_execute(["dolphin"], "File Manager") # KDE neon userin.define_and_execute(["pantheon-files"], "File Manager") # elementary OS userin.define_and_execute(["nautilus", "--browser"], "File Manager") # Ubuntu userin.say("File Manager") elif "OPEN CAMERA" == com: tts_kill() userin.define_and_execute(["kamoso"], "Camera") # KDE neon userin.define_and_execute(["snap-photobooth"], "Camera") # elementary OS userin.define_and_execute(["cheese"], "Camera") # Ubuntu userin.say("Camera") elif "OPEN CALENDAR" == com: tts_kill() userin.define_and_execute(["korganizer"], "Calendar") # KDE neon userin.define_and_execute(["maya-calendar"], "Calendar") # elementary OS userin.define_and_execute(["orage"], "Calendar") # Ubuntu userin.say("Calendar") elif "OPEN CALCULATOR" == com: tts_kill() userin.define_and_execute(["kcalc"], "Calculator") # KDE neon userin.define_and_execute(["pantheon-calculator"], "Calculator") # elementary OS userin.define_and_execute(["gnome-calculator"], "Calculator") # Ubuntu userin.say("Calculator") elif "SOFTWARE CENTER" in com: tts_kill() userin.define_and_execute(["plasma-discover"], "Software Center") # KDE neon userin.define_and_execute( ["software-center"], "Software Center") # elementary OS & Ubuntu userin.say("Software Center") elif com.startswith("KEYBOARD "): tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() for character in original_com[9:]: k.tap_key(character) k.tap_key(" ") elif com == "ENTER": tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.tap_key(k.enter_key) elif com == "NEW TAB": tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.press_keys([k.control_l_key, 't']) elif com == "SWITCH TAB": tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.press_keys([k.control_l_key, k.tab_key]) elif com in ("CLOSE", "ESCAPE"): tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.press_keys([k.control_l_key, 'w']) k.tap_key(k.escape_key) elif com == "GO BACK": tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.press_keys([k.alt_l_key, k.left_key]) elif com == "GO FORWARD": tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.press_keys([k.alt_l_key, k.right_key]) elif com == "SCROLL LEFT": tts_kill() with nostdout(): with nostderr(): m = PyMouse() m.scroll(0, -5) elif com == "SCROLL RIGHT": tts_kill() with nostdout(): with nostderr(): m = PyMouse() m.scroll(0, 5) elif com == "SCROLL UP": tts_kill() with nostdout(): with nostderr(): m = PyMouse() m.scroll(5, 0) elif com == "SCROLL DOWN": tts_kill() with nostdout(): with nostderr(): m = PyMouse() m.scroll(-5, 0) elif com in ("PLAY", "PAUSE", "SPACEBAR"): tts_kill() with nostdout(): with nostderr(): k = PyKeyboard() k.tap_key(" ") elif "SHUTDOWN THE COMPUTER" == com: tts_kill() userin.define(["sudo", "poweroff"], "Shutting down") userin.say("Shutting down") userin.execute(3) elif com in ("GOODBYE", "GOOD BYE", "BYE BYE", "SEE YOU LATER", "CATCH YOU LATER"): tts_kill() userin.define([" "], " ") userin.say("Goodbye, " + user_prefix) # raise KeyboardInterrupt thread.interrupt_main() elif "WIKIPEDIA" in com and ("SEARCH" in com or "FIND" in com): tts_kill() with nostderr(): capture = re.search( "(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? WIKIPEDIA", com) if capture: search_query = capture.group('query') try: wikiresult = wikipedia.search(search_query) if len(wikiresult) == 0: userin.say("Sorry, " + user_prefix + ". But I couldn't find anything about " + search_query + " in Wikipedia.") return True wikipage = wikipedia.page(wikiresult[0]) wikicontent = "".join([ i if ord(i) < 128 else ' ' for i in wikipage.content ]) wikicontent = re.sub(r'\([^)]*\)', '', wikicontent) userin.define_and_execute( ["sensible-browser", wikipage.url], search_query) userin.say(wikicontent) except requests.exceptions.ConnectionError: userin.define_and_execute( [" "], "Wikipedia connection error.") userin.say( "Sorry, " + user_prefix + ". But I'm unable to connect to Wikipedia servers." ) except wikipedia.exceptions.DisambiguationError as disambiguation: USER_ANSWERING['status'] = True USER_ANSWERING['for'] = 'wikipedia' USER_ANSWERING['reason'] = 'disambiguation' USER_ANSWERING['options'] = disambiguation.options[:3] notify = "Wikipedia disambiguation. Which one of these you meant?:\n - " + \ disambiguation.options[0] message = user_prefix + ", there is a disambiguation. Which one of these you meant? " + \ disambiguation.options[0] for option in disambiguation.options[1:3]: message += ", or " + option notify += "\n - " + option notify += '\nSay, for example: "THE FIRST ONE" to choose.' userin.define_and_execute([" "], notify) userin.say(message) except BaseException: pass elif "YOUTUBE" in com and ("SEARCH" in com or "FIND" in com): tts_kill() with nostdout(): with nostderr(): capture = re.search( "(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? YOUTUBE", com) if capture: search_query = capture.group('query') info = youtube_dl.YoutubeDL({}).extract_info( 'ytsearch:' + search_query, download=False, ie_key='YoutubeSearch') if len(info['entries']) > 0: youtube_title = info['entries'][0]['title'] youtube_url = "https://www.youtube.com/watch?v=%s" % ( info['entries'][0]['id']) userin.define(["sensible-browser", youtube_url], youtube_title) youtube_title = "".join([ i if ord(i) < 128 else ' ' for i in youtube_title ]) else: youtube_title = "No video found, " + user_prefix + "." userin.define(" ", " ") userin.execute(0) userin.say(youtube_title) time.sleep(5) k = PyKeyboard() k.tap_key(k.tab_key) k.tap_key(k.tab_key) k.tap_key(k.tab_key) k.tap_key(k.tab_key) k.tap_key('f') elif ("GOOGLE" in com or "WEB" in com) and "IMAGE" not in com and ("SEARCH" in com or "FIND" in com): tts_kill() with nostdout(): with nostderr(): capture = re.search( "(?:SEARCH|FIND) (?P<query>.*) (?:IN|ON|AT|USING)? (?:GOOGLE|WEB)?", com) if capture: search_query = capture.group('query') tab_url = "http://google.com/?#q=" + search_query userin.define_and_execute( ["sensible-browser", tab_url], search_query) userin.say(search_query) elif ("GOOGLE" in com or "WEB" in com) and "IMAGE" in com and ("SEARCH" in com or "FIND" in com): tts_kill() with nostdout(): with nostderr(): capture = re.search("(?:SEARCH IMAGES OF|FIND IMAGES OF|SEARCH|FIND) " "(?P<query>.*) (?:IN|ON|AT|USING)? " "(?:GOOGLE|WEB|GOOGLE IMAGES|WEB IMAGES)?", com) if capture: search_query = capture.group('query') tab_url = "http://google.com/?#q=" + search_query + "&tbm=isch" userin.define(["sensible-browser", tab_url], search_query) userin.execute(0) userin.say(search_query) else: tts_kill() arithmetic_response = arithmeticParser(com) if arithmetic_response: userin.define([" "], " ") userin.say(arithmetic_response) else: learn_response = learn_.respond(original_com) if learn_response: userin.define([" "], " ") userin.say(learn_response) else: if not omniscient_.respond(original_com, not args["silent"], userin, user_prefix, args["server"]): dc_response = dc.respond(original_com, user_prefix) if dc_response: userin.define([" "], " ") userin.say(dc_response)
def interrupt_the_server(self): if interruptable: thread.interrupt_main()
def handle_close(self): # Client disconnected thread.interrupt_main()
def runsubprocess(args,verbose=False,shell=False,polling=False,printstdout=True,preexec_fn=None): """takes a subprocess argument list and runs Popen/communicate or Popen/poll() (if polling=True); if verbose=True, processname (string giving command call) is printed to screen (processname is always printed if a process results in error); errors are handled at multiple levels i.e. subthread error handling; fuction can be used fruitfully (returns stdout)""" #function setup import subprocess,sys,signal try: import thread except: import _thread def subprocess_setup(): #see: https://github.com/vsbuffalo/devnotes/wiki/Python-and-SIGPIPE signal.signal(signal.SIGPIPE, signal.SIG_DFL) if preexec_fn=='sigpipefix': preexec_fn=subprocess_setup else: assert preexec_fn==None,'Error: unrecognised preexec_fn argument %s'%preexec_fn if shell==True: processname=args[0] processname=processname[0].split() processname=(" ".join(a for a in processname)) else: processname=(" ".join(a for a in args)) # if verbose==True: print('{0} {1}'.format(processname, 'processname')) try: if polling==True: p=subprocess.Popen(args, stdout=subprocess.PIPE,shell=shell,preexec_fn=preexec_fn) while True: stdout=p.stdout.readline() if p.poll() is not None: break if stdout: #if stdout not empty... if printstdout==True: print('{0}'.format(stdout.decode().strip())) else: p=subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE,shell=shell,preexec_fn=preexec_fn) stdout, stderr= p.communicate() if stdout: if printstdout==True: print('{0}'.format(stdout.decode())) if stderr: try: #want to output to stderr stream if (sys.version_info > (3, 0)): print('{0}'.format(stderr.decode()),file=sys.stderr) #Python3 else: print>>sys.stderr,stderr #Python2 except: #if above code block fails for some reason, print stderr (to stdout) print('{0}'.format(stderr.decode())) if p.returncode==0: if verbose==True: print('{0} {1}'.format(processname, 'code has run successfully')) else: sys.exit() #triggers except below except: print('{0} {1}'.format(processname, '#this pipeline step produced error')) print('unexpected error; exiting') sys.exit() if p.returncode!=0: print('unexpected error; exiting') try: thread.interrupt_main() except: _thread.interrupt_main() else: if stdout: return stdout.decode().strip()
def __playercheck( self): #Two underscores, mangles to _GridObject__playercheck if self.myinterface == None: return if self.level == None: return if self.game == None: return if self.vitality <= 0: self.die() return self.level.redraw() if self.status != "placed": return if len(self.level.child_interfaces) > 1: self.myinterface.push_message("Your turn.") self._lock.acquire() while 1: try: e = self.myinterface.get_key_event() except HTTPException: #Obviously using RPC so just print print("HTTP Exception in __playercheck() %r" % (sys.exc_info(), )) time.sleep(1) else: break self._lock.release() inexpensive = 0 if e in ("\x03", "\x04", "\x1a"): #ETX ^C, EOT ^D, and ^Z (on Windows). #Does not go through to Python otherwise, meaning that Linux main terminals #may be rendered otherwise out of order until someone kills the game #from a different terminal or over SSH (or rlogin). #This is relevant if running this on an RPi (long ago the intent). self.myinterface.interrupt() elif e in ("down", "up", "left", "right", "8", "4", "6", "2", "7", "9", "1", "3", "h", "j", "k", "l", "y", "u", "b", "n"): direction = self.conv_to_direction(e) target = self.direction_to_target(direction) if self.game.debug_ghost or self.level.handle_move(target, self): self.place(*target) elif e == "#": name = "#" + self.myinterface.ask_question("#") if name in ("#debug", "#debugon"): self.game.debug = 1 elif self.game.debug: inexpensive = 1 if name in ("#debugoff", ): self.game.debug = 0 elif name in ("#ghost", "#ghoston"): self.game.debug_ghost = 1 elif name in ("#ghostoff", ): self.game.debug_ghost = 0 elif name in ("#fovoff", "#fovoffon", "#clairvoyant", "#allsight", "#seeall"): self.game.debug_fov_off = 1 elif name in ("#fov", "#fovon", "#fovoffoff", "#clairvoyantoff", "#allsightoff", "#seealloff"): self.game.debug_fov_off = 0 elif name.startswith("#passthrough "): self.game.handle_command(name.split(" ", 1)[1], self) elif name in ("#bugreport", "#report", "#gurumeditation", "#guru"): self.game.dump_report() elif name in ("#save", "#s"): self.game.save( self.myinterface.ask_question("Save file: ")) elif name in ("#testerror", ): raise RuntimeError("testing error handler") elif name in ("#abort", "#abrt", "#kill"): import os os.abort() elif name in ("#quit", ): interrupt_main() else: inexpensive = self.level.handle_command(name, self) else: inexpensive = self.level.handle_command(name, self) else: inexpensive = self.level.handle_command(e, self) if inexpensive: # Did not expend the players go - give them the go then. self.__playercheck() return if len(self.level.child_interfaces) > 1: self.myinterface.push_message("Turn over.") self.myinterface.dump_messages()
def wait(secs): for n in xrange(timeout_secs): time.sleep(1) if signal_finished: break else: thread.interrupt_main()
def run(self): """run() is called when the process is started via PilotManagerController.start(). """ # make sure to catch sys.exit (which raises SystemExit) try : logger.debug("Worker thread (ID: %s[%s]) for PilotManager %s started." % (self.name, self.ident, self._pm_id)) while not self._stop.is_set(): # # Check if one or more startup requests have finished. # self.startup_results_lock.acquire() # new_startup_results = list() # for transfer_result in self.startup_results: # if transfer_result.ready(): # result = transfer_result.get() # self._db.update_pilot_state( # pilot_uid=result["pilot_uid"], # state=result["state"], # sagajobid=result["saga_job_id"], # pilot_sandbox=result["sandbox"], # global_sandbox=result["global_sandbox"], # submitted=result["submitted"], # logs=result["logs"] # ) # else: # new_startup_results.append(transfer_result) # self.startup_results = new_startup_results # self.startup_results_lock.release() # Check and update pilots. This needs to be optimized at # some point, i.e., state pulling should be conditional # or triggered by a tailable MongoDB cursor, etc. pilot_list = self._db.get_pilots(pilot_manager_id=self._pm_id) action = False for pilot in pilot_list: pilot_id = str(pilot["_id"]) new_state = pilot["state"] if pilot_id in self._shared_data: old_state = self._shared_data[pilot_id]["data"]["state"] else: old_state = None self._shared_data[pilot_id] = { 'data': pilot, 'callbacks': [], 'facade_object': None } self._shared_data[pilot_id]['data'] = pilot # FIXME: *groan* what a hack... The Canceling state is by # the nature of it not recorded in the database, but only in # the local cache. So if we see it as old state, we have to # avoid state transitions into non-final states in the cache # at all cost -- so we catch this here specifically no_cb = False if old_state == CANCELING : if new_state not in [DONE, FAILED, CANCELED] : # restore old state, making the cache explicitly # different than the DB recorded state self._shared_data[pilot_id]["data"]["state"] = old_state # do not tr igger a state cb! no_cb = True if new_state != old_state : action = True if not no_cb : # On a state change, we fire zee callbacks. logger.info("ComputePilot '%s' state changed from '%s' to '%s'." \ % (pilot_id, old_state, new_state)) # The state of the pilot has changed, We call all # pilot-level callbacks to propagate this. This also # includes communication to the unit scheduler which # may, or may not, cancel the pilot's units. self.call_callbacks(pilot_id, new_state) # If the state is 'DONE', 'FAILED' or 'CANCELED', we also # set the state of the compute unit accordingly (but only # for non-final units) if new_state in [FAILED, DONE, CANCELED]: unit_ids = self._db.pilot_list_compute_units(pilot_uid=pilot_id) self._db.set_compute_unit_state ( unit_ids=unit_ids, state=CANCELED, src_states=[ PENDING_INPUT_STAGING, STAGING_INPUT, PENDING_EXECUTION, SCHEDULING, EXECUTING, PENDING_OUTPUT_STAGING, STAGING_OUTPUT ], log="Pilot '%s' has terminated with state '%s'. CU canceled." % (pilot_id, new_state)) # After the first iteration, we are officially initialized! if not self._initialized.is_set(): self._initialized.set() # sleep a little if this cycle was idle if not action : time.sleep(IDLE_TIME) except SystemExit as e : logger.exception ("pilot manager controller thread caught system exit -- forcing application shutdown") import thread thread.interrupt_main () finally : # shut down the autonomous pilot launcher worker(s) for worker in self._pilot_launcher_worker_pool: logger.debug("pworker %s stops launcher %s" % (self.name, worker.name)) worker.stop () logger.debug("pworker %s stopped launcher %s" % (self.name, worker.name))
def _receiverThread(session): """Receiving messages from cjdns admin server""" timeOfLastSend = time.time() timeOfLastRecv = time.time() try: while True: if (timeOfLastSend + KEEPALIVE_INTERVAL_SECONDS < time.time()): if (timeOfLastRecv + 10 < time.time()): raise Exception("ping timeout") session.socket.send( 'd1:q18:Admin_asyncEnabled4:txid8:keepalive') timeOfLastSend = time.time() # Did we get data from the socket? got_data = False while True: # This can be interrupted and we need to loop it. try: data = session.socket.recv(BUFFER_SIZE) except (socket.timeout): # Stop retrying, but note we have no data break except socket.error as e: if e.errno != errno.EINTR: # Forward errors that aren't being interrupted raise # Otherwise it was interrupted so we try again. else: # Don't try again, we got data got_data = True break if not got_data: # Try asking again. continue try: benc = bdecode(data) except (KeyError, ValueError): print("error decoding [" + data + "]") continue if benc['txid'] == 'keepaliv': if benc['asyncEnabled'] == 0: raise Exception("lost session") timeOfLastRecv = time.time() else: # print "putting to queue " + str(benc) session.queue.put(benc) except KeyboardInterrupt: print("interrupted") import thread thread.interrupt_main() except Exception as e: # Forward along any errors, before killing the thread. session.queue.put(e)
def quit_function(fn_name): # print to stderr, unbuffered in Python 2. logging.error("{0} took too long".format(fn_name)) sys.stderr.flush() # Python 3 stderr is likely buffered. thread.interrupt_main()
def exit(self, value=0): print 'Exiting...' # sys.exit(value) self.loop.quit() # cleanup_stop_thread() thread.interrupt_main()
def _telnet_server(self, pstdin_w, pstdout_r, sock, orig_stdout): queue = PriorityQueue() inputbuffer = b'' self._telnet_seq = 0 try: t = threading.Thread(target=self._telnet_server_writer, args=(queue, sock)) t.daemon = True t.start() lock = threading.Lock() def writeall(data): start = 0 while start < len(data): size = os.write(pstdin_w, data[start:]) start += size def sendcontrol(t, data): lock.acquire() try: self._telnet_seq += 1 seq = self._telnet_seq finally: lock.release() queue.put((t, seq, data)) t2 = threading.Thread(target=self._telnet_server_writer2, args=(pstdout_r, queue, lock, orig_stdout)) t2.daemon = True t2.start() escaping = False option = None while True: newdata = sock.recv(1024) if newdata == b'': break for i in range(0, len(newdata)): c = newdata[i:i + 1] if escaping: if option == b'\xfd' and c == b'\x06': sendcontrol(1, b'\xff\xfb\x06') option = None escaping = False elif option == b'\xfd' or option == b'\xfe': sendcontrol(1, b'\xff\xfc' + c) option = None escaping = False elif option == b'\xfb' or option == b'\xfc': sendcontrol(1, b'\xff\xfe' + c) option = None escaping = False elif c in (b'\xfb', b'\xfc', b'\xfd', b'\xfe'): option = c else: option = None if c == b'\xf3' or c == b'\xf4': thread.interrupt_main() escaping = False else: if c == b'\x03': thread.interrupt_main() elif c == b'\x08': inputbuffer = inputbuffer[:-1] elif c == b'\x00': inputbuffer += b'\n' writeall(inputbuffer) inputbuffer = b'' elif c == b'\r' or c == b'\n': inputbuffer += c writeall(inputbuffer) inputbuffer = b'' elif c == b'\xff': escaping = True else: inputbuffer += c except OSError: pass except IOError: pass finally: try: os.close(pstdin_w) except: pass queue.put((-1, -1, -1))
def handler(): print("[DEBUG] Shutting down HTTP server via timeout (%d) seconds." % (ARGS.timeout)) thread.interrupt_main()
def f(): time.sleep(0.5) thread.interrupt_main()
def input_alert(self, title='', message='', input='', ok_button_title='ok', hide_cancel_button=False): #first, cover up view #then from threading import Timer print(threading.enumerate()) from thread import interrupt_main self.e.clear() for s in self.subviews: self.remove_subview(s) if self.superview: dim = ui.View(frame=(0, 0, self.superview.width, self.superview.height), bg_color=(0.5, 0.5, 0.5, 0.75)) inputbox = ui.View() inputbox.width = 300 inputbox.corner_radius = 10 inputbox.bg_color = 'white' inputbox.border_color = 'black' inputbox.center = (dim.width / 2, dim.height / 2) titlelabel = ui.Label() titlelabel.text = title titlelabel.alignment = ui.ALIGN_CENTER titlelabel.font = ('<system-bold>', 14) titlelabel.size_to_fit() titlelabel.y = 10 titlelabel.x = 0 messagelabel = ui.Label() messagelabel.text = message messagelabel.alignment = ui.ALIGN_CENTER messagelabel.font = ('<system>', 12) messagelabel.x = 0 messagelabel.y = titlelabel.height + titlelabel.y messagelabel.size_to_fit() inputfield = ui.TextField() inputfield.text = input inputfield.font = ('<system>', 12) inputfield.height = titlelabel.height * 1.5 inputfield.x = 0.1 * inputbox.width inputfield.y = messagelabel.height + messagelabel.y inputfield.width = .8 * inputbox.width inputfield.corner_radius = 0 inputfield.border_width = 1 inputfield.action = self.input_action buttons = ui.SegmentedControl() buttons.corner_radius = 0 buttons.height = 30 buttons.segments = ('Cancel', 'Ok') buttons.width = titlelabel.width = messagelabel.width = inputbox.width buttons.width += 10 buttons.x -= 5 buttons.y = inputfield.y + inputfield.height + 10 buttons.action = self.button_action inputbox.height = buttons.y + buttons.height - 1 dim.add_subview(inputbox) inputbox.add_subview(titlelabel) inputbox.add_subview(messagelabel) inputbox.add_subview(inputfield) inputbox.add_subview(buttons) self.superview.add_subview(dim) self.bring_to_front() dim.bring_to_front() self.width = dim.width self.height = dim.height def check_if_onscreen(): if not self.on_screen: self.e.set() else: Timer(0.25, check_if_onscreen).start() check_if_onscreen() self.e.wait() self.superview.remove_subview(dim) if not hide_cancel_button and buttons.selected_index == 0: rtnvalue = None self.width = 1 self.height = 1 interrupt_main() self.e.set() else: rtnvalue = inputfield.text self.width = 1 self.height = 1 return rtnvalue
def input_thread(): global bool_send_default_program while True: key_pressed = read_key() print("Key pressed-{}".format(key_pressed)) if key_pressed.lower() == "q": thread.interrupt_main() break else: res = '' if key_pressed.lower() == "i": res = command.init_seq_motor_1(6, 8, 7) if key_pressed.lower() == "o": res = command.init_seq_motor_2(10, 11, 12) if key_pressed.lower() == "a": res = command.move(100, 'R') if key_pressed.lower() == "b": res = command.move(200, 'L') if key_pressed.lower() == "0": res = command.moveto(0) if key_pressed.lower() == "1": res = command.moveto(90) if key_pressed.lower() == "2": res = command.moveto(180) if key_pressed.lower() == "3": res = command.moveto(-360) if key_pressed.lower() == "4": res = command.moveto(360 * 4) if key_pressed.lower() == "c": res = command.define_vel_acc(60, 400, 1) if key_pressed.lower() == "d": res = command.disable_pins(0) if key_pressed.lower() == "e": res = command.define_vel_acc(60, 400, 15) if key_pressed.lower() == "f": res = command.define_vel_acc(60, 400, 30) if key_pressed.lower() == "g": res = command.define_vel_acc(50, 120, 60) if key_pressed.lower() == "9": res = command.delay(300) if key_pressed.lower() == "8": res = command.delay(600) if key_pressed.lower() == "s": res = command.show_prog(0) if key_pressed.lower() == "[": res = command.program_start(0) if key_pressed.lower() == "]": res = command.program_end(0) if key_pressed.lower() == "l": res = command.show_prog(1) if key_pressed.lower() == "{": res = command.program_start(1) if key_pressed.lower() == "}": res = command.program_end(1) if key_pressed.lower() == "t": res = command.run_prog(0) if key_pressed.lower() == "y": res = command.run_prog(1) if key_pressed.lower() == "=": bool_send_default_program = True if not res == '': ser.write(res)
def do_GET(self): url = urlparse.urlsplit(self.path) query = dict(cgi.parse_qsl(url[3])) index_query = self.path.find('?') if index_query == -1: curr_path = self.path else: curr_path = self.path[:index_query] # non-command line HTTP clients, i.e. Internet browsers, may request a favorite icon if curr_path == '/favicon.ico': if ThreadedRequestHandler.does_favicon_exist: self.send_response(200) self.send_header('Content-Type', 'image/x-icon') self.end_headers() self.wfile.write(ThreadedRequestHandler.favicon_data) self.wfile.close() else: self.send_error(404, 'ERROR - ICON NOT FOUND: %s' % self.path) # stop the server, if only called from localhost elif curr_path == '/stop': if self.client_address[0] == LOCAL_HOST: body = 'stopped\n' self.send_response(200) self.send_header('Content-Type', 'text/plain') self.send_header('Content-Length', str(len(body))) self.end_headers() self.wfile.write(body) self.wfile.close() if sys.version_info < (2, 5): self.call_shell('./kill.sh') else: thread.interrupt_main() else: self.send_error(403, 'FORBIDDEN: not localhost') # get the directory listing as JSON {name, size} elif curr_path == '/list': file_dir = query.get('dir') file_prefix = query.get('prefix') file_suffix = query.get('suffix') check = False if not file_dir: self.send_error( 400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path) elif not does_dir_exist(file_dir): self.send_error( 400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path) else: # De Morgan's Laws # https://en.wikipedia.org/wiki/De_Morgan%27s_laws #if not file_prefix and not file_suffix: if not (file_prefix or file_suffix): check = False else: if not file_prefix: file_prefix = '' if not file_suffix: file_suffix = '' check = True file_info = [] curr_dir = EXPORT_DIR_ABS + os.sep + file_dir for file_name in os.listdir(curr_dir): add = False if os.path.isfile(curr_dir + os.sep + file_name): if check: add = (file_name.startswith(file_prefix) and file_name.endswith(file_suffix)) else: add = True if add: file_info.append( '{"name":"' + file_name + '","size":' + str(os.path.getsize(curr_dir + os.sep + file_name)) + '}') if len(file_info) > 0: list_json = '[' list_json += ','.join(file_info) list_json += ']' else: list_json = '[]' list_json += '\n' self.send_response(200) self.send_header('Content-Type', 'application/json') self.end_headers() self.wfile.write(list_json) self.wfile.close() # get the file, optionally as compressed elif curr_path == '/get': file_dir = query.get('dir') file_name = query.get('name') comp_ext = query.get('compress') compress = False if comp_ext: if comp_ext in ['zip', 'gz']: compress = True if not file_dir: self.send_error( 400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path) elif not does_dir_exist(file_dir): self.send_error( 400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path) elif not file_name: self.send_error(400, 'ERROR - FILE NAME IS MISSING: %s' % self.path) elif not does_file_exist(file_dir, file_name): self.send_error(400, 'ERROR - FILE DOES NOT EXIST: %s' % self.path) else: curr_dir = EXPORT_DIR_ABS + os.sep + file_dir try: sendReply = False if file_name.endswith(".html"): mime_type = 'text/html' sendReply = True elif file_name.endswith(".csv"): mime_type = 'text/csv' sendReply = True elif file_name.endswith(".gz"): mime_type = 'application/gzip' sendReply = True elif file_name.endswith(".zip"): mime_type = 'application/zip' sendReply = True else: mime_type = 'text/plain' sendReply = True if sendReply: f = open(curr_dir + os.sep + file_name, 'rb') if compress: ok = True base_name = os.path.splitext( os.path.basename(file_name))[0] file_content = f.read() file_size = 0 file_comp = '' mime_type = '' content = '' if comp_ext == 'gz': gz_file = self.gz_content( file_name, file_content) gz_size = self.get_size(gz_file) content = gz_file.getvalue() gz_file.close() file_comp = base_name + '.gz' file_size = gz_size mime_type = 'application/gzip' elif comp_ext == 'zip': zip_file = self.zip_content( file_name, file_content) zip_size = self.get_size(zip_file) content = zip_file.getvalue() zip_file.close() file_comp = base_name + '.zip' file_size = zip_size mime_type = 'application/zip' else: ok = False if ok: self.send_response(200) self.send_header('Content-Type', mime_type) self.send_header('Content-Length', file_size) self.send_header( 'Content-Disposition', 'attachment; filename=' + file_comp) self.end_headers() self.wfile.write(content) self.wfile.close() else: self.send_error( 400, 'ERROR - UNKNOWN COMPRESSION FORMAT: %s' % self.path) else: self.send_response(200) self.send_header('Content-Type', mime_type) self.send_header( 'Content-Length', str( os.path.getsize(curr_dir + os.sep + file_name))) self.send_header( 'Content-Disposition', 'attachment; filename=' + file_name) self.end_headers() while True: data = f.read(4096) self.wfile.write(data) if not data: break self.wfile.write('\n') self.wfile.close() f.close() else: self.send_error( 400, 'ERROR - BAD FILE REQUEST: %s' % self.path) return except IOError: self.send_error(404, 'ERROR - FILE NOT FOUND: %s' % self.path) # create a tarball which may take ages and then call back to inform the client elif curr_path == '/tar': file_dir = query.get('dir') file_name = query.get('name') if not file_dir: self.send_error( 400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path) elif not does_dir_exist(file_dir): self.send_error( 400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path) elif not file_name: self.send_error(400, 'ERROR - FILE NAME IS MISSING: %s' % self.path) elif not does_file_exist(file_dir, file_name): self.send_error(400, 'ERROR - FILE DOES NOT EXIST: %s' % self.path) else: cb_host = query.get('host') cb_port = query.get('port') cb_path = query.get('path') if cb_host and cb_port and cb_path: body = 'working ...\n' self.send_response(200) self.send_header('Content-Type', 'text/plain') self.send_header('Content-Length', str(len(body))) self.end_headers() self.wfile.write(body) self.wfile.close() base_name = os.path.splitext( os.path.basename(file_name))[0] curr_dir = file_dir tgz_file = base_name + '.tgz' file_comp = tgz_file cb_url = { 'host': cb_host, 'port': cb_port, 'path': cb_path, 'timeout': HTTP_CB_TIMEOUT, 'tar': tgz_file } if does_file_exist(file_dir, file_comp): logger.info('file already exists: %s' % file_dir + os.sep + file_comp) ret = 0 else: args = 'cd ' + curr_dir + '; tar -czf ' + file_comp + ' ' + file_name ret = self.call_shell(args) if ret == 0: cb_url['query'] = 'success=true' else: cb_url['query'] = 'success=false' self.http_get(cb_url) else: self.send_error( 400, 'ERROR - CALLBACK PARAMETERS ARE MISSING: %s' % self.path) # delete file in a given directory elif curr_path == '/delete': file_dir = query.get('dir') file_name = query.get('name') file_prefix = query.get('prefix') file_suffix = query.get('suffix') if not file_dir: self.send_error( 400, 'ERROR - DIRECTORY NAME IS MISSING: %s' % self.path) elif not does_dir_exist(file_dir): self.send_error( 400, 'ERROR - DIRECTORY DOES NOT EXIST: %s' % self.path) else: curr_dir = EXPORT_DIR_ABS + os.sep + file_dir if file_name: if does_file_exist('', curr_dir + os.sep + file_name): os.remove(curr_dir + os.sep + file_name) self.send_response(200) self.send_header('Content-Type', 'text/plain') self.end_headers() self.wfile.write('deleted: ' + curr_dir + os.sep + file_name) self.wfile.close() else: self.send_error( 400, 'ERROR - FILE DOES NOT EXIST: %s' % curr_dir + os.sep + file_name) else: if not (file_prefix or file_suffix): self.send_error( 400, 'ERROR - DELETE PARAMETERS ARE MISSING: %s' % self.path) else: del_file = 0 if not file_prefix: file_prefix = '*' if not file_suffix: file_suffix = '*' for file_name in os.listdir(curr_dir): if file_prefix == '*' and not file_suffix == '*': if file_name.endswith(file_suffix): del_file += 1 os.remove(curr_dir + os.sep + file_name) elif file_suffix == '*' and not file_prefix == '*': if file_name.startswith(file_prefix): del_file += 1 os.remove(curr_dir + os.sep + file_name) elif file_suffix == '*' and file_prefix == '*': del_file += 1 os.remove(curr_dir + os.sep + file_name) else: if file_name.startswith( file_prefix) and file_name.endswith( file_suffix): del_file += 1 os.remove(curr_dir + os.sep + file_name) self.send_response(200) self.send_header('Content-Type', 'text/plain') self.end_headers() if del_file > 0: self.wfile.write('deleted ' + str(del_file) + ' files\n') else: self.wfile.write('no file is deleted\n') self.wfile.close() else: self.send_error(400, 'ERROR - BAD RESOURCE REQUEST: %s' % self.path)
def handle_error(request, client_address): _type, value, tb = sys.exc_info() if isinstance(value, KeyboardInterrupt): interrupt_main()
def action(): time.sleep(2.0) thread.interrupt_main()
def quit_function(func): print('{0} TimeoutError'.format(func), file=sys.stderr) sys.stderr.flush() thread.interrupt_main()
def kill_proc(p): p.kill() # The timer is running in a separate thread, so # use interrupt_main() to throw a KeyboardInterrupt # back in the main thread. thread.interrupt_main()
def interrupter(): time.sleep(0.25) # Raises KeyboardInterrupt in the main thread thread.interrupt_main()
def process_output_parser(self, invoked_process): logger = logging.getLogger("ProcessBot.process_output_parser") logger.info("start parsing stdout of created process " + self.service) with self._lock_bus: if self.bus.state() == 'connected': self.bus.SendMessage( busIndipendentMessages.LogMessage( self._remote_wps_endpoint, "INFO", "start parsing stdout of created process " + self.service)) else: try: self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() # self.bus.xmpp.get_roster() if self.bus.state() == 'connected': self.bus.SendMessage( busIndipendentMessages.LogMessage( self._remote_wps_endpoint, "INFO", "start parsing stdout of created process " + self.service)) else: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) except: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) # Listen to stdout stdout_parser_compiled = [re.compile(r) for r in self._stdout_parser] stack_trace = [] while True: line = invoked_process.stdout.readline() if line != '': line = line.strip() logger.debug("Received line: " + line) stack_trace.append(line) match = False #todo: define a line to get pid of computational job process for rgx, action in zip(stdout_parser_compiled, self._stdout_action): res = rgx.match(line) if (res): if (action == "progress"): with self._lock_bus: if self.bus.state() != 'connected': try: self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() # self.bus.xmpp.get_roster() except: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) self.bus.SendMessage( busIndipendentMessages.ProgressMessage( self._remote_wps_endpoint, float(res.group(1).strip()))) match = True break elif (action == "log"): with self._lock_bus: if self.bus.state() != 'connected': try: self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() # self.bus.xmpp.get_roster() except: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) self.bus.SendMessage( busIndipendentMessages.LogMessage( self._remote_wps_endpoint, res.group(1).strip(), res.group(2).strip())) match = True break elif (action == "abort"): with self._lock_bus: if self.bus.state() != 'connected': try: self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() # self.bus.xmpp.get_roster() except: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) self.bus.SendMessage( busIndipendentMessages.ErrorMessage( self._remote_wps_endpoint, res.group(2).strip())) match = True break elif (action == "ignore"): match = True break else: pass else: #end of stream break logger.debug("process " + self.service + "stdout is over") #wait for process exit code return_code = invoked_process.wait() logger.info("process exit code is " + str(return_code)) if return_code == 0: #success logger.info("process exit code is " + str(return_code) + ": success") logger.info( "send job-completed message to WPS with output parameter") outputs = dict() try: for p in self._output_parameters_defs.parameters(): outputs[p.get_name()] = [ p.get_value(), p.get_description(), p.get_title(), p.get_type(), p.is_publish_as_layer(), p.get_publish_layer_name(), p.get_publish_default_style(), p.get_publish_target_workspace(), p.get_metadata() ] except: logging.exception("Process " + str(self._uniqueExeId) + " Exception: " + str(traceback.format_exc(sys.exc_info()))) error_message = "process exit code is " + str( return_code) + ": failure\n" + "\n".join( str(e) for e in stack_trace) self.send_error_message(error_message) #self.bus.disconnect() logger.info("after send job-error message to WPS") thread.interrupt_main() os._exit(return_code) logger.info("trying to acquire bus lock...") with self._lock_bus: logger.info("bus lock acquired...") if self.bus.state() != 'connected': try: self.bus.xmpp.reconnect() self.bus.xmpp.send_presence() # self.bus.xmpp.get_roster() except: logger.info( "[XMPP Disconnected]: Process " + str(self._uniqueExeId) + " Could not send info message to GeoServer Endpoint " + str(self._remote_wps_endpoint)) counter = 1 while not self._finished: logger.info("sending 'completed' message tentative #" + str(counter)) self.bus.SendMessage( busIndipendentMessages.CompletedMessage( self._remote_wps_endpoint, self._remote_wps_baseurl, outputs)) counter = counter + 1 if counter < 100: sleep(10) else: logger.error( "Could not contact Remote WPS with. Forcibly shutdown the process..." ) thread.interrupt_main() os._exit(-1) logger.info("after send job-completed message to WPS") else: error_message = "process exit code is " + str( return_code) + ": failure\n" + "\n".join( str(e) for e in stack_trace) logger.critical("process exit code is " + str(return_code) + ": failure") #todo: should i wait for finish message here as well? No self.send_error_message(error_message) #self.bus.disconnect() logger.info("after send job-error message to WPS") thread.interrupt_main() os._exit(return_code)
def end(): tb.stop() tb.close() thread.interrupt_main()