class AirpnpProcess(object): def __init__(self, config={}): self.config = config def create_config(self, config): f = tempfile.NamedTemporaryFile(delete=True) f.write("[airpnp]\n") for k, v in config.items(): f.write("%s=%s\n" % (k, v)) f.flush() return f.name def __enter__(self): configfn = self.create_config(self.config) args = ["twistd", "-n", "airpnp", "-c", configfn] cwd = os.path.join(os.path.dirname(__file__), "..") self.proc = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=cwd, close_fds=ON_POSIX, bufsize=1) q = Queue() t = Thread(target=enqueue_output, args=(self.proc.stdout, q)) t.daemon = True t.start() return q def __exit__(self, type, value, tb): self.proc.kill()
class DownloadThread(threading.Thread): def __init__(self, title, url): threading.Thread.__init__(self) self.progress = 'Preparing download...' self.updated = True self.killed = False # create the youtube-dl subprocess file = title + '.%(ext)s' output = '--output=' + file max_quality = '--max-quality=35' command = ['youtube-dl', '--no-part', '--continue', max_quality, output, url] self.download_process = Popen(command, stdout=PIPE, universal_newlines=True) def kill(self): self.download_process.kill() self.killed = True def run(self): while self.progress != '': self.progress = self.download_process.stdout.readline() self.updated = True if self.killed: self.progress = 'Aborted downloading' else: self.progress = 'Finished downloading'
def test_nanny_worker_ports(loop): try: worker = Popen(['dworker', '127.0.0.1:8989', '--host', '127.0.0.1', '--worker-port', '8788', '--nanny-port', '8789'], stdout=PIPE, stderr=PIPE) sched = Popen(['dscheduler', '--port', '8989'], stdout=PIPE, stderr=PIPE) with Executor('127.0.0.1:8989', loop=loop) as e: start = time() while True: d = sync(e.loop, e.scheduler.identity) if d['workers']: break else: assert time() - start < 5 sleep(0.1) assert d['workers']['127.0.0.1:8788']['services']['nanny'] == 8789 finally: with ignoring(Exception): w = rpc('127.0.0.1:8789') sync(loop, w.terminate) with ignoring(Exception): os.kill(sched.pid, signal.SIGINT) with ignoring(Exception): worker.kill()
def process(self, request, response, report): if response.is_html: referrer = str(request) pth = os.path.dirname(os.path.realpath(__file__)) proc = Popen([self.phantomJsPath, '{0}{1}scrape.js'.format(pth, os.sep), referrer], stdin=PIPE, stdout=PIPE, stderr=PIPE) try: out, err = proc.communicate(response.content.encode(), timeout=10) rc = proc.returncode except TimeoutExpired: proc.kill() out, err = proc.communicate() if len(err) > 0: report.add_error(err.decode('utf-8')) else: report.add_error(out.decode('utf-8')) else: if rc == 0: result = out.decode('utf-8').strip() urls = set() for url in set(result.splitlines()): if self._log_url(url): urls.add(url) self._add_request(url, referrer) self._report(report, urls) else: if len(err) > 0: report.add_error(err.decode('utf-8')) else: report.add_error(out.decode('utf-8'))
class OpenPostgreSQLsshTunnel(object): """ Class to let us open an ssh tunnel, then close it when done """ def __init__(self, port=5432): self.tunnel_process = 0 self.postgre_port = 5432 self.remote_port = port def __enter__(self): if HOSTNAME != 'dilepton-tower': import time import shlex self.postgre_port = self.remote_port _cmd = 'ssh -N -L localhost:%d' % self.remote_port + \ ':localhost:5432 [email protected]' args = shlex.split(_cmd) self.tunnel_process = Popen(args, shell=False) time.sleep(5) return self.postgre_port def __exit__(self, exc_type, exc_value, traceback): if self.tunnel_process: self.tunnel_process.kill() if exc_type or exc_value or traceback: return False else: return True
class MongodbPlugin(object): def __init__(self): self.mongo = None self.tmpdir = tempfile.mkdtemp() def pytest_sessionstart(self, session): port = session.config.getvalue('mongodb_port') self.mongo = Popen(["mongod", "--dbpath", self.tmpdir, "--port", str(port)], stdin=PIPE, stdout=PIPE, stderr=PIPE) for each in range(10): if 'waiting for connections' in self.mongo.stdout.readline(): break else: raise OSError('Mongodb start timeout.') def pytest_sessionfinish(self, session): if self.mongo is not None: try: self.mongo.kill() self.mongo.communicate() self.mongo.wait() finally: shutil.rmtree(self.tmpdir)
def load_shell(self, chunk): """Run shell commands from code chunks""" if chunk['evaluate']: lines = chunk['content'].lstrip().splitlines() result = "\n" for line in lines: command = line.split() major, minor = sys.version_info[:2] cmd = Popen(command, stdout=PIPE) if major == 2 or minor < 3: # Python 2 doesn't have timeout for subprocess try: content = cmd.communicate()[0].decode('utf-8').replace("\r", "") + "\n" except Exception as e: content = "Pweave ERROR can't execute shell command:\n %s\n" % command content += str(e) sys.stdout.write(" Pweave ERROR can't execute shell command:\n %s\n" % line) print(str(e)) else: try: content = cmd.communicate(timeout=20)[0].decode('utf-8').replace("\r", "") + "\n" except TimeoutExpired: cmd.kill() content, errs = cmd.communicate() sys.stdout.write("Shell command timeout:\n %s\n" % line) content = content.decode('utf-8').replace("\r", "") + "\n" if chunk['term']: result += "$ %s\n" % line result += content else: result = "" return result
def newjob(): choiceddevices = dict(request.form).get('choicedDevice') jobname = request.form.get('jobName') testtype = request.form.get('testType') monkeyconfig = {} monkeyconfig["actioncount"] = request.form.get("actioncount") monkeyconfig["actiondelay"] = request.form.get("actiondelay") monkeyconfig["touchpercent"] = request.form.get("touchpercent") monkeyconfig["motionpercent"] = request.form.get("motionpercent") monkeyconfig["pinchzoompercent"] = request.form.get("pinchzoompercent") monkeyconfig["majornavpercent"] = request.form.get("majornavpercent") monkeyconfig["syskeyspercent"] = request.form.get("syskeyspercent") monkeyconfig["appswitchpercent"] = request.form.get("appswitchpercent") order = request.args.get("order").split("|") f = request.files['file'] fname = secure_filename(f.filename) apk = os.path.join(Config.UPLOAD_FOLDER,fname) if not os.path.isdir(Config.UPLOAD_FOLDER): os.makedirs(Config.UPLOAD_FOLDER) f.save(apk) cmd_activity = "aapt d badging %s|%s launchable-activity" %(apk,"findstr" if system == "Windows" else "grep") cmd_package = "aapt d badging %s|%s package" %(apk,"findstr" if system == "Windows" else "grep") activity = Popen(cmd_activity,stdout=PIPE,shell=True) package = Popen(cmd_package,stdout=PIPE,shell=True) main_activity = activity.stdout.read().decode().split("name='")[1].split("'")[0] packageName = package.stdout.read().decode().split("name='")[1].split("'")[0] activity.kill() package.kill() testjob = Testjob(jobname,testtype,choiceddevices,apk,packageName,main_activity,order,monkeyconfig) db.session.add(testjob) db.session.commit() return redirect(url_for(".jobs"))
class Player: def __init__(self, name, cmd): self.name = name self.forts = set() self.marches = set() args = cmd.split(' ') self.process = Popen(list(args) + [name], stdin=PIPE, stdout=PIPE, universal_newlines=True) def capture(self, fort): if fort.owner: fort.owner.forts.remove(fort) self.forts.add(fort) fort.owner = self def is_defeated(self): return (not self.forts) and (not self.marches) def send_state(self): if not self.process.poll(): self.process.stdin.write(show_visible(self.forts)) self.process.stdin.write('\n') self.process.stdin.flush() def read_commands(self, game): if not self.process.poll(): try: read_commands(game, self, self.process.stdout) except StopIteration: self.process.kill()
def run(self): print "Starting asynchrone Thread with Command " + self.cmd p = Popen([self.cmd], stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False) if bDebug: print getTime() + ": PID of Thread = " + str(p.pid) # wrap p.stdout with a NonBlockingStreamReader object: nbsr = NonBlockingStreamReader(p.stdout) # get the output strTemp = '' while self.isrunning: output = nbsr.readline(0.1) # 0.1 secs to let the shell output the result if not output: if strTemp != '': # print strTemp try: self.connstream.write(strTemp) except: if bDebug: print getTime() + ": Connection was destroyed by Client!" print getTime() + ": PID = " + str(os.getpid()) print getTime() + ": Group PID = " + str(os.getpgid(os.getpid())) # p.kill() stops subprocess p.kill() self.isrunning = False break strTemp = '' continue else: strTemp = strTemp + output nbsr.stop() return
class MockServer(object): def __init__(self, http_port=None, https_port=None, proxy_port=None): self.http_port = http_port if http_port is not None else get_ephemeral_port() self.https_port = https_port if https_port is not None else get_ephemeral_port() self.proxy_port = proxy_port if proxy_port is not None else get_ephemeral_port() def __enter__(self): self.proc = Popen([ sys.executable, '-u', '-m', 'splash.tests.mockserver', '--http-port', str(self.http_port), '--https-port', str(self.https_port), '--proxy-port', str(self.proxy_port), ], env=get_testenv() ) for port in (self.http_port, self.https_port, self.proxy_port): _wait_for_port(port) return self def __exit__(self, exc_type, exc_value, traceback): self.proc.kill() self.proc.wait() def url(self, path, gzip=True, host='localhost'): gzip_path = '' if not gzip else '/gzip' return "http://%s:%s%s/%s" % ( host, self.http_port, gzip_path, path.lstrip('/') ) def https_url(self, path): return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))
def test_reload_wrapper_preservation(self): # This test verifies that when `python -m tornado.autoreload` # is used on an application that also has an internal # autoreload, the reload wrapper is preserved on restart. main = """\ import os import sys # This import will fail if path is not set up correctly import testapp if 'tornado.autoreload' not in sys.modules: raise Exception('started without autoreload wrapper') import tornado.autoreload print('Starting') sys.stdout.flush() if 'TESTAPP_STARTED' not in os.environ: os.environ['TESTAPP_STARTED'] = '1' # Simulate an internal autoreload (one not caused # by the wrapper). tornado.autoreload._reload() else: # Exit directly so autoreload doesn't catch it. os._exit(0) """ # Create temporary test application path = mkdtemp() os.mkdir(os.path.join(path, 'testapp')) self.addCleanup(shutil.rmtree, path) init_file = os.path.join(path, 'testapp', '__init__.py') open(init_file, 'w').close() main_file = os.path.join(path, 'testapp', '__main__.py') with open(main_file, 'w') as f: f.write(main) # Make sure the tornado module under test is available to the test # application pythonpath = os.getcwd() if 'PYTHONPATH' in os.environ: pythonpath += os.pathsep + os.environ['PYTHONPATH'] autoreload_proc = Popen( [sys.executable, '-m', 'tornado.autoreload', '-m', 'testapp'], stdout=subprocess.PIPE, cwd=path, env=dict(os.environ, PYTHONPATH=pythonpath), universal_newlines=True) for i in range(20): if autoreload_proc.poll() is not None: break time.sleep(0.1) else: autoreload_proc.kill() raise Exception("subprocess failed to terminate") out = autoreload_proc.communicate()[0] self.assertEqual(out, 'Starting\n' * 2)
def err_check(alpha,coord_vector,slope,space_type): #Runs OPIUM, checks test config errors and returns their sum if space_type == 'coeff': test_vector = coord_vector.copy() test_vector = test_vector - alpha*slope grid = fit_to_func(test_vector,'fourier_cos') setKB(grid) elif space_type == 'valley': test_vector = VtoC(coord_vector - alpha*slope) grid = fit_to_func(test_vector,'fourier_cos') setKB(grid) else: raise ValueError('space_type must be "coeff" or "valley"') #Use threading to cancel OPIUM if it takes too long proc = Popen('./opium ' + filename + ' ' + filename + '.log all rpt', shell=True) proc_thread = threading.Thread(target=proc.communicate) proc_thread.start() proc_thread.join(timeout_sec) if proc_thread.is_alive(): # Process still running - kill it and raise timeout error try: proc.kill() except OSError, e: # The process finished between the `is_alive()` and `kill()` return proc.returncode
def _main(): c = synth.Context() PrecomputeWaveTables(c).start() midi_name = sys.argv[1] file_name = sys.argv[2] command = './identifykey.sh -f %s' % file_name print(command) p = Popen(command, shell=True, stdout=PIPE) key = p.stdout.readline().strip() print('Detected key: [%s]' % key) t = None if midi_name: a = c.add(keep_alive = True) c.start(a) def on_note(note): shifted = note.key_shift('C', key) print('%s -> %s' % (note, shifted)) if shifted: a.add_module(copy(play.beep(c, shifted.shift_octave(-1)))) t = play.MidiListener(on_note, name=midi_name) t.start() else: c.start(scale(c, key)) m = None if file_name: command = 'aplay %s' % (file_name,) print(command) m = Popen(command.split(), stdin=PIPE) sys.stdin.readline() if t: t.stop() c.stop() if m: m.kill()
def execute_test(): retvalue = -1 envvars = os.environ.copy() if platform.system() == "Linux": libdir = envvars.get("CMAKE_LIBRARY_DIRECTORIES") if libdir != None: envvars["LD_LIBRARY_PATH"] = envvars["NDDSHOME"] + "/lib/" + example + ":" + libdir.replace(";", ":") else: envvars["LD_LIBRARY_PATH"] = envvars["NDDSHOME"] + "/lib/" + example + ":" + project_source_dir + "/lib" # Run server server = Popen([output_dir + "/bin/" + example + "/" + test_name + "ServerExample"], env=envvars) # Wait 5 seconds before run client sleep(5) client = Popen([output_dir + "/bin/" + example + "/" + test_name + "ClientExample"], env=envvars) client.wait() client.communicate() retvalue = client.returncode server.kill() return retvalue
class MockServer(object): def __init__(self, http_port=None, https_port=None, proxy_port=8990): self.http_port = http_port if http_port is not None else _ephemeral_port() self.https_port = https_port if https_port is not None else _ephemeral_port() self.proxy_port = proxy_port if proxy_port is not None else _ephemeral_port() def __enter__(self): self.proc = Popen([ sys.executable, '-u', '-m', 'splash.tests.mockserver', '--http-port', str(self.http_port), '--https-port', str(self.https_port), '--proxy-port', str(self.proxy_port), ], stdout=PIPE, env=get_testenv() ) for port in (self.http_port, self.https_port, self.proxy_port): _wait_for_port(port) print(_non_block_read(self.proc.stdout)) def __exit__(self, exc_type, exc_value, traceback): self.proc.kill() self.proc.wait() time.sleep(0.2) def url(self, path): return "http://localhost:%s/%s" % (self.http_port, path.lstrip('/')) def https_url(self, path): return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))
def call_command(*args): try: formatted_command = command.format(*args) except IndexError: raise WrongArgumentError( "'{0}' not enough arguments. Called with {1}".format(command, args)) process = Popen(formatted_command, stdout=PIPE, stderr=PIPE, shell=True, universal_newlines=True) if timeout: wait_time_remaining = timeout while process.poll() is None and wait_time_remaining > 0: time.sleep(wait_step) wait_time_remaining -= wait_step if wait_time_remaining <= 0: process.kill() raise OperatingSystemError( "{0} have not completed in {1} seconds".format( formatted_command, timeout)) (out, err) = process.communicate() if err != '': log = logging.getLogger('sh') log.warn( "'{0}' has written to stderr: {1}".format(formatted_command, err)) return out.rstrip()
def async_read(self, process): output = [] characters = [] pattern = '###################- 100.0% 0.0 kBps' pattern_repeat = 0 # set non-blocking flag while preserving old flags fl = fcntl(process.stdout, F_GETFL) fcntl(process.stdout, F_SETFL, fl | os.O_NONBLOCK) # read char until EOF hit while True: try: ch = os.read(process.stdout.fileno(), 1) # EOF if not ch: break ch = ch.decode('UTF-8') characters.append(ch) # New line - check the pattern and add to output string if ch == '\n': string = "".join(characters) if (string.find(pattern) != -1): pattern_repeat += 1 if (pattern_repeat == 2): # zsync is in loop (bug) Popen.kill(process) break output.append(string) characters = [] except OSError: # waiting for data be available on process.stdout pass return "".join(output)
class CrawlTestCase(TestCase): def setUp(self): self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'], stdout=PIPE) self.proc.stdout.readline() def tearDown(self): self.proc.kill() self.proc.wait() time.sleep(0.2) @defer.inlineCallbacks def test_follow_all(self): spider = FollowAllSpider() yield docrawl(spider) self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url @defer.inlineCallbacks def test_delay(self): spider = FollowAllSpider() yield docrawl(spider, {"DOWNLOAD_DELAY": 0.3}) t = spider.times[0] for t2 in spider.times[1:]: self.assertTrue(t2-t > 0.15, "download delay too small: %s" % (t2-t)) t = t2
def sendSMS(self,mobileno,text): if(self.notLogged): print("you are not logged in - call logIn()") return if(self.captchaNeeded): with open(self.captchaPath,"wb") as f: f.write(self.opener.open(self.captchaUrl).read()) p = Popen(["display",self.captchaPath]) self.dataDict['textcode'] = input("Captcha ? ") p.kill() if(len(text) <= 140): self.dataDict['mobNo'] = mobileno self.dataDict['text'] = text # print(self.postDataStr.format(**self.dataDict)) try: h = self.opener.open(self.sendSMSUrl,self.postDataStr.format(**self.dataDict).encode()) resp = h.read() try: msg = BS(resp).find("div",attrs={"id":"quicksms"}).find("div",attrs={"class":"quickname"}).text.strip() if msg.endswith("submitted successfully"): pass else: print("N : "+msg) except: print("N");self.captchaNeeded = True with open("successResp.html","wb") as f: f.write(resp) except urllib.error.HTTPError as error: pass
def run(self): try: if osflag: proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\ stderr=STDOUT,bufsize=0) else: from subprocess import STARTUPINFO si=STARTUPINFO() si.dwFlags|=1 si.wShowWindow=0 proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\ stderr=STDOUT,bufsize=0,startupinfo=si) while 1: if self.stop_flag: if osflag: proc.send_signal(signal.SIGKILL) else: proc.kill() break if osflag: if proc.stdout in select.select([proc.stdout],[],[],1)[0]: line=proc.stdout.readline() else: line=' \n' else: line=proc.stdout.readline() if not len(line): break else: if count(line,'ttl') or count(line,'TTL'): self.retries=0 else: self.retries=self.retries+1 line=' ' sleep(0.5) proc.poll() except: pass
class Engine: def __init__(self): self._proc = Popen(['stockfish'], stdin=PIPE, stdout=PIPE, stderr=PIPE) self._proc.stdin.write('setoption name Hash value 128\n') self._proc.stdin.write('setoption name Threads value 4\n') self._proc.stdin.write('setoption name Best Book Move value true\n') self._proc.stdin.write('setoption name Aggressiveness value 200\n') self._proc.stdin.write('setoption name Cowardice value 0\n') self._proc.stdin.write('setoption name Contempt Factor value 50\n') def update_pos(self, fen): self._proc.stdin.write('position fen %s\n' % fen) def get_next_move(self, fen=None, time_limit=50): if fen is not None: self.update_pos(fen) self._proc.stdin.write('go movetime %d\n' % time_limit) line = self._proc.stdout.readline() while not line.startswith('bestmove'): line = self._proc.stdout.readline() print line return line.split(' ')[1] def close(self): self._proc.kill()
def launch_spl(self): need_launch_in_shell = (os.name == "nt") # launch sploit proccess with team_ip as arg spl = Popen([self.sploit_name, self.team_ip], stdout=PIPE, stderr=STDOUT, bufsize=1, shell=need_launch_in_shell) q = Queue() # we are processing output in other thread to prevent blocking def enqueue_output(queue, out): while True: line = out.readline() queue.put(line) if not line: break t = Thread(target=enqueue_output, args=(q, spl.stdout)) t.daemon = True t.start() # get output by lines until EOF while True: try: remaining_time = MAX_RUNTIME - (time() - self.last_launch_time) line = q.get(timeout=remaining_time) except (Empty, ValueError): log("Killing %s sploit(tried to run for more than %d secs)" % ( self.team_name, MAX_RUNTIME)) break if not line: break line = line.strip() if not line: continue if self.cycle_num == 1: print("%s: %s" % (self.team_name, line)) flags = re.findall(FLAG_FORMAT, line) OWNER_LOCK.acquire() for flag in flags: if flag not in self.flags: if self.cycle_num == 1: log("Flag from %s: %s" % (self.team_name, flag)) with open(self.flag_filename, "ab", 0) as f: f.write(flag + b"\n") self.flags.add(flag) else: if self.cycle_num == 1: log("Flag from %s: %s (dup)" % (self.team_name, flag)) OWNER_LOCK.release() if os.name != "nt": spl.kill() else: spl.communicate()
def getLines(self): lines = [str(line) for line in vim.current.buffer] (row, col) = vim.current.window.cursor cur_line = lines[row-1] cur_line = cur_line[:col] + self.base + "AUTO332" + cur_line[col:] lines[row-1] = cur_line input_buffer = "\n".join(lines) # Send the buffer to hh_client and return the lines args = [ Const.CLIENT, r'--auto-complete' ] proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE) proc.stdin.write(input_buffer); out = proc.communicate()[0] base = self.base try: proc.kill(); except: pass proc = args = None #Util.debugf('getLines.txt', locals()) return [x for x in out.split("\n") if len(x) > 0]
def tesseract_ocr_and_render_pdf( input_files, output_file, log, pdfinfo, pdfinfo_lock): input_image = next((ii for ii in input_files if ii.endswith('.png')), '') input_pdf = next((ii for ii in input_files if ii.endswith('.pdf'))) if not input_image: # Skipping this page re_symlink(input_pdf, output_file) return args_tesseract = [ 'tesseract', '-l', '+'.join(options.language), input_image, os.path.splitext(output_file)[0], # Tesseract appends suffix 'pdf' ] + options.tesseract_config p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE, universal_newlines=True) try: stdout, stderr = p.communicate(timeout=options.tesseract_timeout) if stdout: log.info(stdout) if stderr: log.error(stderr) except TimeoutExpired: p.kill() log.info("Tesseract - page timed out") re_symlink(input_pdf, output_file)
class DisqueNode: def __init__(self, port, dir): self.port = port self.dir = dir self.proc = None self.socket = os.path.join(dir, 'disque.sock') def start(self): if not self.proc: cmd = ["disque-server", "--port", str(self.port), "--dir", self.dir, "--unixsocket", self.socket, "--unixsocketperm", "755"] self.proc = Popen(cmd, stdout=PIPE, stderr=PIPE) cmd = ['disque', '-p', str(self.port), 'info'] while True: sleep(.01) if self.proc.poll(): raise Exception('already stopped!', self.proc.stderr) resp = run(cmd, stdout=PIPE, stderr=PIPE) if not resp.returncode: break def stop(self): self.proc.kill() self.proc = None @property def configuration(self): return Configuration(port=self.port, dir=self.dir, socket=self.socket)
def wpa_run(self): dump_cmd = ['airodump-ng', '-c', self.channel, '--bssid', self.bssid, '-w', './log/' + self.bssid, self.iface] airodump_proc = Popen(dump_cmd, stdout=DN, stderr=DN) self.proc_list.append(airodump_proc) self.send_deauths() while self.key == '': output = Popen('tshark -r ./log/' + self.bssid + '-01.cap 2>/dev/null| grep "Message 4 of 4"',shell=True, stdout=PIPE).communicate()[0] if output.find('Message 4 of 4') != -1: execute('rm ./log/'+self.bssid+'.key') airodump_proc.kill() airodump_proc.communicate() crack_cmd = ['aircrack-ng', '-w', self.password_list, '-b', self.bssid, './log/' + self.bssid + '-01.cap','-l', './log/' + self.bssid + '.key'] crack_proc = Popen(crack_cmd, stdout=DN) self.proc_list.append(crack_proc) crack_proc.wait() try: f = open('./log/' + self.bssid + '.key') key = f.read() f.close() self.key = key self.crack_success = True self.stop() except: pass else: self.send_deauths() time.sleep(5) return self.key
def getDeviceState(): global deviceStatus cmd = "adb devices" devices = [] p = Popen(cmd,stdout=PIPE,shell=True) for info in p.stdout.readlines(): info = info.decode() if 'List' in info: continue elif 'offline' in info or 'unauthorized' in info or 'device' in info: device = {} name,state = [n.strip() for n in info.split('\t') if n.strip()] device["deviceName"] = name device["state"] = state if ":" in name: device["replacedName"] = name.replace(".","").replace(":","") else: device["replacedName"] = name devices.append(device) if name not in deviceStatus.keys(): deviceStatus[name] = False else: continue p.kill() return devices
def has_active_torrents(): """ Check for any active torrents on the local deluged server. :return: True if one or more torrents are active :rtype: Bool """ std_output, std_err_output = '', '' proc = None try: proc = Popen(["deluge-console", "info"], stdout=PIPE, stderr=PIPE) std_output, std_err_output = proc.communicate() # Decode std_output = std_output.decode("utf-8") std_err_output = std_err_output.decode("utf-8") except (OSError, ValueError): if proc is not None: proc.kill() raise RuntimeError("deluge-console could not be found, " + "make sure it is installed") if ('fatal' in std_output) or ( 'fatal' in std_err_output) or proc.returncode >= 1: raise RuntimeError("Status check failed, " + "make sure that deluged is running") else: # Success! return std_output.strip() != ""
def runLocalhostBenchmark(): serverName = 'async_server' serverPort = '22380' clientName = 'bench_client' hostName = 'localhost' localhostCount = count / 50 p1 = Popen(['./'+serverName, serverPort], stdout=PIPE, stderr=PIPE, env=env) then = time.time() p2 = Popen(['./'+clientName, hostName+':'+serverPort, str(localhostCount)], stdout=PIPE, stderr=PIPE, env=env) out2, err2 = p2.communicate() t = time.time() - then p1.kill() out1, err1 = p1.communicate() if p2.returncode != 0: print '%s error. returncode: %d' % (executableName, p2.returncode) print clientName, 'stdout:' print out2 print clientName, 'stderr:' print err2 print serverName, 'stdout:' print out1 print serverName, 'stderr:' print err1 sys.exit(1) print('%s: Made %s AMP calls (back-to-back in series - over ' 'localhost TCP socket) in %.4f seconds (%s calls/s)' % (clientName, locale.format("%d", localhostCount, grouping=True), t, locale.format("%d", int(localhostCount/t), grouping=True)))
class scp_proxy: def __init__(self,userhost,verbose=0): """Opens a connection to the remote host and establishes the remote client. userhost should be of the form "user@host""" self.verbose=verbose # Open the remote SSH process try : self.ssh=Popen(("ssh",userhost,"e2scp.py --client"),stdin=PIPE,stdout=PIPE) self.stdout=self.ssh.stdout # read from this self.stdin=self.ssh.stdin # write to this except: print("ssh to remote machine failed : ",("ssh",host,"e2ssh.py --client")) traceback.print_exc() sys.exit(2) while 1: ln=self.stdout.readline().strip() if len(ln)==0 : print("Error running e2scp.py on the remote machine. EMAN2 installed ?") sys.exit(3) if ln=="HELO" : if self.verbose : print("Connection established") break if self.verbose >1 : print("*** ",ln) atexit.register(self.close) def close(self): """Close the connection""" if self.ssh!=None : try: self.stdin.write("exit\n") # self.stdin.flush() self.ssh.kill() except: pass self.ssh=None def mkdir(self,path): """Create a path on the remote host, at all necessary levels""" self.stdin.write("mkdir\n%s\n"%path) self.stdin.flush() r=self.stdout.readline().strip() if r!="OK" : raise Exception("Error in creating remote path (%s)"%(r)) def listrecurse(self,path,basepath=""): """Recursively list the contents of a remote path, may be a directory or a BDB specifier. If specified will reference paths with respect to basepath.""" self.stdin.write("listrecurse\n%s\n%s\n"%(path,basepath)) r=int(self.stdout.readline().strip()) ret=[] for i in xrange(r): ret.append(self.stdout.readline().strip()) return ret def getheader(self,path,n): """Return the header of a remote image as a dictionary""" self.stdin.write("getheader\n%s\n%d\n"%(path,n)) self.stdin.flush() return read_obj(self.stdout,img) def getimage(self,path,n): """Return a single reomote EMData object from a file""" self.stdin.write("getimage\n%s\n%d\n"%(path,n)) self.stdin.flush() return read_obj(self.stdout,img) def putimage(self,path,n,img): """Write a single EMData object into a remote file""" self.stdin.write("putimage\n%s\n%d\n"%(path,n)) self.write_obj(self.stdin,img) self.stdin.flush() def getfile(self,remotepath,localpath): """Retrieve a single binary remote file and write to path. Streams for minimal memory usage.""" self.stdin.write("getfile\n%s\n"%(remotepath)) self.stdin.flush() recv_file(self.stdout,localpath) def putfile(self,localpath,remotepath): self.stdin.write("putfile\n%s\n"%(remotepath)) self.stdin.flush() send_file(self.stdin,localpath) def getbdb(self,remotepath,localpath): """Copyies a remote BDB to a local machine""" self.stdin.write("getbdb\n%s\n"%(remotepath)) self.stdin.flush() recv_bdb(self.stdout,localpath) def putbdb(self,localpath,remotepath): """Copies a local BDB to a remote machine""" self.stdin.write("putbdb\n%s\n"%(remotepath)) self.stdin.flush() send_bdb(self.stdin,localpath)
class TestUDST(TestCase): def setUp(self): # Application, minimum, to define three trace points app_text = b""" #include <unistd.h> #include <stdint.h> #include <stdio.h> #include <string.h> #include "folly/tracing/StaticTracepoint.h" int main() { char s[100]; int i, a = 200, b = 40; for (i = 0; i < 100; i++) s[i] = (i & 7) + (i & 6); uint64_t j = 0; char s1[64]; const char* str = "str"; size_t len = strlen(str); while (1) { FOLLY_SDT(test, probe_point_1, s[7], b); FOLLY_SDT(test, probe_point_3, a, b); FOLLY_SDT(test, probe_point_1, s[4], a); FOLLY_SDT(test, probe_point_2, 5, s[10]); FOLLY_SDT(test, probe_point_3, s[4], s[7]); memset(&s1, '\0', sizeof(s1)); strncpy(s1, str, len); snprintf(s1 + len, sizeof(s1) - len, "%d", j); FOLLY_SDT(test, probe_point_4, j++, &s1); memset(&s1, '\0', sizeof(s1)); strncpy(s1, str, len); snprintf(s1 + len, sizeof(s1) - len, "%d", j); FOLLY_SDT(test, probe_point_5, &s1, j++); sleep(1); } return 1; } """ # BPF program self.bpf_text = """ #include <linux/blkdev.h> #include <uapi/linux/ptrace.h> struct probe_result_t1 { char v1; int v2; }; struct probe_result_t2 { int v1; char v2; }; struct probe_result_t3 { int v1; int v2; }; struct probe_result_t4 { u64 v1; char v2[8]; }; struct probe_result_t5 { char v1[8]; u64 v2; }; BPF_PERF_OUTPUT(event1); BPF_PERF_OUTPUT(event2); BPF_PERF_OUTPUT(event3); BPF_PERF_OUTPUT(event4); BPF_PERF_OUTPUT(event5); int do_trace1(struct pt_regs *ctx) { struct probe_result_t1 result = {}; bpf_usdt_readarg(1, ctx, &result.v1); bpf_usdt_readarg(2, ctx, &result.v2); event1.perf_submit(ctx, &result, sizeof(result)); return 0; }; int do_trace2(struct pt_regs *ctx) { struct probe_result_t2 result = {}; bpf_usdt_readarg(1, ctx, &result.v1); bpf_usdt_readarg(2, ctx, &result.v2); event2.perf_submit(ctx, &result, sizeof(result)); return 0; } int do_trace3(struct pt_regs *ctx) { struct probe_result_t3 result = {}; bpf_usdt_readarg(1, ctx, &result.v1); bpf_usdt_readarg(2, ctx, &result.v2); event3.perf_submit(ctx, &result, sizeof(result)); return 0; } int do_trace4(struct pt_regs *ctx) { struct probe_result_t4 result = {}; bpf_usdt_readarg(1, ctx, &result.v1); bpf_usdt_readarg_p(2, ctx, &result.v2, sizeof(result.v2)); event4.perf_submit(ctx, &result, sizeof(result)); return 0; } int do_trace5(struct pt_regs *ctx) { struct probe_result_t5 result = {}; bpf_usdt_readarg_p(1, ctx, &result.v1, sizeof(result.v1)); bpf_usdt_readarg(2, ctx, &result.v2); event5.perf_submit(ctx, &result, sizeof(result)); return 0; } """ # Compile and run the application self.ftemp = NamedTemporaryFile(delete=False) self.ftemp.close() comp = Popen([ "gcc", "-I", "%s/include" % os.path.dirname( os.path.abspath(inspect.getfile(inspect.currentframe()))), "-x", "c", "-o", self.ftemp.name, "-" ], stdin=PIPE) comp.stdin.write(app_text) comp.stdin.close() self.assertEqual(comp.wait(), 0) self.app = Popen([self.ftemp.name]) def test_attach1(self): # enable USDT probe from given PID and verifier generated BPF programs u = USDT(pid=int(self.app.pid)) u.enable_probe(probe="probe_point_1", fn_name="do_trace1") u.enable_probe(probe="probe_point_2", fn_name="do_trace2") u.enable_probe(probe="probe_point_3", fn_name="do_trace3") u.enable_probe(probe="probe_point_4", fn_name="do_trace4") u.enable_probe(probe="probe_point_5", fn_name="do_trace5") b = BPF(text=self.bpf_text, usdt_contexts=[u], debug=4) # Event states for each event: # 0 - probe not caught, 1 - probe caught with correct value, # 2 - probe caught with incorrect value self.evt_st_1 = 0 self.evt_st_2 = 0 self.evt_st_3 = 0 # define output data structure in Python class Data1(ct.Structure): _fields_ = [("v1", ct.c_char), ("v2", ct.c_int)] class Data2(ct.Structure): _fields_ = [("v1", ct.c_int), ("v2", ct.c_char)] class Data3(ct.Structure): _fields_ = [("v1", ct.c_int), ("v2", ct.c_int)] class Data4(ct.Structure): _fields_ = [("v1", ct.c_ulonglong), ("v2", ct.c_char * 64)] class Data5(ct.Structure): _fields_ = [("v1", ct.c_char * 64), ("v2", ct.c_ulonglong)] def check_event_val(event, event_state, v1, v2, v3, v4): if ((event.v1 == v1 and event.v2 == v2) or (event.v1 == v3 and event.v2 == v4)): if (event_state == 0 or event_state == 1): return 1 return 2 def print_event1(cpu, data, size): event = ct.cast(data, ct.POINTER(Data1)).contents self.evt_st_1 = check_event_val(event, self.evt_st_1, b'\x0d', 40, b'\x08', 200) def print_event2(cpu, data, size): event = ct.cast(data, ct.POINTER(Data2)).contents # pretend we have two identical probe points to simplify the code self.evt_st_2 = check_event_val(event, self.evt_st_2, 5, b'\x04', 5, b'\x04') def print_event3(cpu, data, size): event = ct.cast(data, ct.POINTER(Data3)).contents self.evt_st_3 = check_event_val(event, self.evt_st_3, 200, 40, 8, 13) def print_event4(cpu, data, size): event = ct.cast(data, ct.POINTER(Data4)).contents print("%s" % event.v2) def print_event5(cpu, data, size): event = ct.cast(data, ct.POINTER(Data5)).contents print("%s" % event.v1) # loop with callback to print_event b["event1"].open_perf_buffer(print_event1) b["event2"].open_perf_buffer(print_event2) b["event3"].open_perf_buffer(print_event3) b["event4"].open_perf_buffer(print_event4) b["event5"].open_perf_buffer(print_event5) # three iterations to make sure we get some probes and have time to process them for i in range(3): b.perf_buffer_poll() self.assertTrue(self.evt_st_1 == 1 and self.evt_st_2 == 1 and self.evt_st_3 == 1) def tearDown(self): # kill the subprocess, clean the environment self.app.kill() self.app.wait() os.unlink(self.ftemp.name)
def __load_tar(filename, mode, allowTmpFile, transformDataRawData, tarMember, extractAll, eraseTmpTarMembers, ignore_zeros, logger=None): """ Internal method for reading tarfiles """ #useSubprocess = False useSubprocess = True if tarMember is None: f = tarfile.open(filename, mode, ignore_zeros=ignore_zeros) if not extractAll: if logger: logger.info("Retrieving tar file members (%s)...", "full" if ignore_zeros else "fast") memberList = f.getmembers() elif type(tarMember) in (tarfile.TarInfo, str): useSubprocess = True memberList = [tarMember] else: raise TypeError("tarMember argument must be TarInfo or None.") for entry in memberList if not extractAll else [None]: if allowTmpFile: tmpFolderPath = tempfile.mkdtemp() if useSubprocess: from subprocess import Popen, PIPE, CalledProcessError from RingerCore import is_tool tar_cmd = 'gtar' if is_tool('gtar') else 'tar' # TODO This will crash if someday someone uses a member in file that is # not in root path at the tarfile. if extractAll: start = time() logger.info("Proceeding to untar all members.") process_args = ( tar_cmd, '--verbose', '-xvzif', filename, ) untar_ps = Popen(process_args, stdout=PIPE, bufsize=1, cwd=tmpFolderPath) memberList = [] with untar_ps.stdout: while True: outputLine = untar_ps.stdout.readline().strip('\n') if outputLine == '': if untar_ps.poll() is not None: break else: memberList.append(outputLine) logger.debug(outputLine) return_code = untar_ps.wait() if return_code != 0: raise CalledProcessError(return_code, process_args) from re import compile rexp = compile('\s+') memberList = [ (int(size), name) for _, _, size, _, _, name in map( lambda member: rexp.split(member), memberList) ] end = time() logger.info("Untar file content took %.2fs", end - start) else: memberName = entry.name if type( entry) is tarfile.TarInfo else entry untar_ps = Popen(( tar_cmd, '--verbose', '-xvzif', filename, memberName, ), stdout=PIPE, bufsize=1, cwd=tmpFolderPath) with untar_ps.stdout: from re import compile rexp = compile('\s+') for line in iter(untar_ps.stdout.readline, b''): line = line.strip('\n') _, _, size, _, _, name = rexp.split(line) memberList = [(int(size), name)] break for entry in memberList: memberSize, memberName = ( entry.size, entry.name, ) if type(entry) is tarfile.TarInfo else entry oFile = os.path.join(tmpFolderPath, memberName) while not os.path.isfile(oFile): sleep(0.001) while os.path.getsize(oFile) != memberSize: sleep(0.001) if not extractAll: untar_ps.kill() untar_ps.wait() os.listdir(tmpFolderPath) with open(oFile) as f_member: data = transformDataRawData( cPickle.load(f_member), oFile if extractAll else filename, memberName) yield data if extractAll: break #else: # if extractAll: # logger.info("Untaring all members to %s...", tmpFolderPath) # f.extractall(path=tmpFolderPath, ) # else: # f.extractall(path=tmpFolderPath, members=(entry,)) # for entry in memberList if extractAll else [entry]: # memberName = entry.name if type(entry) is tarfile.TarInfo else entry # oFile = os.path.join(tmpFolderPath, memberName) # print oFile # with open(oFile) as f_member: # yield transformDataRawData( cPickle.load(f_member), oFile if extractAll else filename, entry ) # if extractAll: # break if eraseTmpTarMembers: shutil.rmtree(tmpFolderPath) else: fileobj = f.extractfile(entry) if checkExtension(entry.name, 'gz|gzip'): fio = StringIO.StringIO(fileobj.read()) fileobj = gzip.GzipFile(fileobj=fio) yield transformDataRawData(cPickle.load(fileobj), filename, memberName) if not useSubprocess: f.close()
def li_calc(calc_op, simnode, simacc, **kwargs): scene = bpy.context.scene pfs, epfs = [], [] context = simnode['coptions']['Context'] subcontext = simnode['coptions']['Type'] scene['liparams']['maxres'], scene['liparams']['minres'], scene['liparams']['avres'] = {}, {}, {} frames = range(scene['liparams']['fs'], scene['liparams']['fe'] + 1) if not kwargs.get('genframe') else [kwargs['genframe']] os.chdir(scene['viparams']['newdir']) rtcmds, rccmds = [], [] builddict = {'0': ('School', 'Higher Education', 'Healthcare', 'Residential', 'Retail', 'Office & Other'), '2': ('School', 'Higher Education', 'Healthcare', 'Residential', 'Retail', 'Office & Other'), '3': ('Office/Education/Commercial', 'Healthcare')} for f, frame in enumerate(frames): if context == 'Basic' or (context == 'CBDM' and subcontext == '0') or (context == 'Compliance' and int(subcontext) < 3): if os.path.isfile("{}-{}.af".format(scene['viparams']['filebase'], frame)): os.remove("{}-{}.af".format(scene['viparams']['filebase'], frame)) if simnode.pmap: pmappfile = open(os.path.join(scene['viparams']['newdir'], 'viprogress'), 'w') pmappfile.close() errdict = {'fatal - too many prepasses, no global photons stored\n': "Too many prepasses have ocurred. Make sure light sources can see your geometry", 'fatal - too many prepasses, no global photons stored, no caustic photons stored\n': "Too many prepasses have ocurred. Turn off caustic photons and encompass the scene", 'fatal - zero flux from light sources\n': "No light flux, make sure there is a light source and that photon port normals point inwards", 'fatal - no light sources\n': "No light sources. Photon mapping does not work with HDR skies"} amentry, pportentry, cpentry, cpfileentry = retpmap(simnode, frame, scene) pmcmd = ('mkpmap -e {1}.pmapmom -bv+ +fo -apD 0.001 {0} -apg {1}-{2}.gpm {3} {4} {5} {1}-{2}.oct'.format(pportentry, scene['viparams']['filebase'], frame, simnode.pmapgno, cpentry, amentry)) pmrun = Popen(pmcmd.split(), stderr = PIPE, stdout = PIPE) while pmrun.poll() is None: with open(os.path.join(scene['viparams']['newdir'], 'viprogress'), 'r') as pfile: if 'CANCELLED' in pfile.read(): pmrun.kill() return 'CANCELLED' sleep(1) with open('{}.pmapmon'.format(scene['viparams']['filebase']), 'r') as pmapfile: for line in pmapfile.readlines(): if line in errdict: calc_op.report({'ERROR'}, errdict[line]) return # for line in pmrun.stdout: # print(line) # for line in pmrun.stderr: # print(line) # if line.decode() in errdict: # calc_op.report({'ERROR'}, errdict[line.decode()]) # return rtcmds.append("rtrace -n {0} -w {1} -ap {2}-{3}.gpm 50 {4} -faa -h -ov -I {2}-{3}.oct".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame, cpfileentry)) #+" | tee "+lexport.newdir+lexport.fold+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res" else: # if context == 'Compliance': # rtcmds.append("rcontrib -w -n {} {} -m sky_glow -I+ -V+ {}-{}.oct ".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame)) # # else: rtcmds.append("rtrace -n {0} -w {1} -faa -h -ov -I {2}-{3}.oct".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame)) #+" | tee "+lexport.newdir+lexport.fold+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res" else: rccmds.append("rcontrib -w -h -I -fo -bn 146 {} -n {} -f tregenza.cal -b tbin -m sky_glow {}-{}.oct".format(simnode['radparams'], scene['viparams']['nproc'], scene['viparams']['filebase'], frame)) # rccmds.append("rcontrib -w -h -I- -fo -bn 146 {} -n {} -f tregenza.cal -b tbin -m env_glow {}-{}.oct".format(simnode['radparams'], scene['viparams']['nproc'], scene['viparams']['filebase'], frame)) try: tpoints = [bpy.data.objects[lc]['rtpnum'] for lc in scene['liparams']['livic']] except: calc_op.report({'ERROR'}, 'Re-export the LiVi geometry') return calcsteps = sum(tpoints) * len(frames) pfile = progressfile(scene, datetime.datetime.now(), calcsteps) kivyrun = progressbar(os.path.join(scene['viparams']['newdir'], 'viprogress')) reslists = [] obs = [scene.objects[on] for on in scene['liparams']['livic']] for oi, o in enumerate(obs): curres = sum(tpoints[:oi]) selobj(scene, o) o['omax'], o['omin'], o['oave'], totsensearea, totsdaarea, totasearea = {}, {}, {}, 0, 0, 0 if context == 'Basic': bccout = o.basiccalcapply(scene, frames, rtcmds, simnode, curres, pfile) if bccout == 'CANCELLED': return 'CANCELLED' else: reslists += bccout elif context == 'CBDM' and subcontext == '0': lhout = o.lhcalcapply(scene, frames, rtcmds, simnode, curres, pfile) if lhout == 'CANCELLED': return 'CANCELLED' else: reslists += lhout elif (context == 'CBDM' and subcontext in ('1', '2')) or (context == 'Compliance' and subcontext == '3'): udiout = o.udidacalcapply(scene, frames, rccmds, simnode, curres, pfile) if udiout == 'CANCELLED': return 'CANCELLED' else: reslists += udiout[2] pfs.append(udiout[0]) epfs.append(udiout[1]) elif context == 'Compliance': compout = o.compcalcapply(scene, frames, rtcmds, simnode, curres, pfile) if compout == 'CANCELLED': return 'CANCELLED' else: reslists += compout[2] pfs.append(compout[0]) epfs.append(compout[1]) for f, frame in enumerate(frames): if context == 'Compliance': tpf = 'FAIL' if 'FAIL' in pfs[f] or 'FAIL*' in pfs[f] else 'PASS' if simnode['coptions']['canalysis'] == '0': tpf = 'EXEMPLARY' if tpf == 'PASS' and ('FAIL' not in epfs[f] and 'FAIL*' not in epfs[f]) else tpf cred = '0' if tpf == 'FAIL' else ('1', '2', '2', '1', '1', '1')[int(simnode['coptions']['buildtype'])] ecred = '1' if tpf == 'EXEMPLARY' else '0' simnode['tablecomp{}'.format(frame)] = [['Standard: BREEAM HEA1'], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Standard credits: ' + cred], ['Exemplary credits: '+ ecred]] for o in obs: o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])]), 'Standard credits: ' + cred, 'Exemplary credits: '+ ecred, '']] elif simnode['coptions']['canalysis'] == '1': cfshcred = 0 for pf in pfs[f]: for stype in [0, 1, 2]: if all([p[1] == 'Pass' for p in pf if p[0] == stype]) and [p for p in pf if p[0] == stype]: cfshcred += 1 simnode['tablecomp{}'.format(frame)] = [['Standard: CfSH'], ['Build type: Residential'], [''], ['Standard credits: {}'.format(cfshcred)]] for o in obs: o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: Residential', 'Standard credits: {}'.format(cfshcred), '', '']] elif simnode['coptions']['canalysis'] == '2': gscred = max(len(p) for p in pfs[f]) - max([sum([(0, 1)[p == 'Fail'] for p in pf]) for pf in pfs[f]]) simnode['tablecomp{}'.format(frame)] = [['Standard: Green Star'], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Standard credits: {}'.format(gscred)]] for o in obs: o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']]), 'Standard credits: {}'.format(gscred), '', '']] elif simnode['coptions']['canalysis'] == '3': cred = 0 for z in list(zip(pfs[f], epfs[f])): if all([pf == 'Pass' for pf in z[:-1]]): cred += int(z[-1]) simnode['tablecomp{}'.format(frame)] = [['Standard: LEEDv4'], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Credits: {}'.format(cred)]] for o in obs: o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])]), 'Credits: {}'.format(cred), '', '']] if kivyrun.poll() is None: kivyrun.kill() return reslists
class SmtlibProc: def __init__(self, command: str, debug: bool = False): """Single smtlib interactive process :param command: the shell command to execute :param debug: log all messaging """ self._proc: Optional[Popen] = None self._command = command self._debug = debug def start(self): """Spawns POpen solver process""" if self._proc is not None: return self._proc = Popen( shlex.split(self._command), stdin=PIPE, stdout=PIPE, bufsize=0, universal_newlines=True, close_fds=True, ) def stop(self): """ Stops the solver process by: - sending a SIGKILL signal, - waiting till the process terminates (so we don't leave a zombie process) """ if self._proc is None: return # if it did not finished already if self._proc.returncode is None: self._proc.stdin.close() self._proc.stdout.close() # Kill the process self._proc.kill() self._proc.wait() # No need to wait for termination, zombies avoided. self._proc = None def __readline_and_count(self): assert self._proc assert self._proc.stdout buf = self._proc.stdout.readline() # No timeout enforced here # If debug is enabled check if the solver reports a syntax error # Error messages may contain an unbalanced parenthesis situation if self._debug: if "(error" in buf: raise SolverException(f"Error in smtlib: {buf}") lparen, rparen = map(sum, zip(*((c == "(", c == ")") for c in buf))) return buf, lparen, rparen def send(self, cmd: str) -> None: """ Send a string to the solver. :param cmd: a SMTLIBv2 command (ex. (check-sat)) """ if self._debug: logger.debug(">%s", cmd) self._proc.stdout.flush() # type: ignore self._proc.stdin.write(f"{cmd}\n") # type: ignore def recv(self) -> str: """Reads the response from the smtlib solver""" buf, left, right = self.__readline_and_count() bufl = [buf] while left != right: buf, l, r = self.__readline_and_count() bufl.append(buf) left += l right += r buf = "".join(bufl).strip() if self._debug: logger.debug("<%s", buf) return buf def _restart(self) -> None: """Auxiliary to start or restart the external solver""" self.stop() self.start() def is_started(self): return self._proc is not None
class MCDReforgedServer: process: Optional[Popen] def __init__(self): # TODO console args self.mcdr_state = MCDReforgedState.INITIALIZING self.server_state = ServerState.STOPPED self.process = None # type: Optional[PIPE] self.flags = MCDReforgedFlag.NONE self.starting_server_lock = Lock( ) # to prevent multiple start_server() call self.stop_lock = Lock() # to prevent multiple stop() call # will be assigned in on_config_changed() self.encoding_method = None # type: Optional[str] self.decoding_method = None # type: Optional[str] # --- Constructing fields --- self.logger = MCDReforgedLogger(self) self.logger.set_file(constant.LOGGING_FILE) self.server_interface = ServerInterface(self) self.task_executor = TaskExecutor(self) self.console_handler = ConsoleHandler(self) self.watch_dog = WatchDog(self) self.update_helper = UpdateHelper(self) self.language_manager = LanguageManager(self.logger) self.config = Config(self.logger) self.rcon_manager = RconManager(self) self.server_handler_manager = ServerHandlerManager(self) self.reactor_manager = InfoReactorManager(self) self.command_manager = CommandManager(self) self.plugin_manager = PluginManager(self) self.permission_manager = PermissionManager(self) # --- Initialize fields instance --- file_missing = False try: self.load_config( allowed_missing_file=False) # loads config, language, handlers except FileNotFoundError: self.logger.error('Config is missing, default config generated') self.config.save_default() file_missing = True try: self.permission_manager.load_permission_file( allowed_missing_file=False) except FileNotFoundError: self.logger.error( 'Permission file is missing, default permission file generated' ) self.permission_manager.save_default() file_missing = True if file_missing: self.on_first_start() return self.plugin_manager.register_permanent_plugins() self.set_mcdr_state(MCDReforgedState.INITIALIZED) def __del__(self): try: if self.process and self.process.poll() is None: self.kill_server() except: pass def on_first_start(self): self.logger.info( 'Some of the user files are missing, check them before launch MCDR again' ) default_config = self.config.get_default() file_util.touch_directory(default_config['working_directory']) self.plugin_manager.set_plugin_directories( default_config['plugin_directories']) # to touch the directory # -------------------------- # Translate info strings # -------------------------- def tr(self, text, *args, allow_failure=True): result = self.language_manager.translate(text, allow_failure).strip('\r\n') if len(args) > 0: result = result.format(*args) return result # -------------------------- # Loaders # -------------------------- def load_config(self, *, allowed_missing_file=True): has_missing = self.config.read_config(allowed_missing_file) # load the language first to make sure tr() is available self.on_config_changed() if has_missing: for line in self.tr('config.missing_config').splitlines(): self.logger.warning(line) def on_config_changed(self): logger.console_color_disabled = self.config['disable_console_color'] self.logger.set_debug_options(self.config['debug']) if self.config.is_debug_on(): self.logger.info( self.tr('mcdr_server.on_config_changed.debug_mode_on')) self.language_manager.set_language(self.config['language']) self.logger.info( self.tr('mcdr_server.on_config_changed.language_set', self.config['language'])) self.encoding_method = self.config['encoding'] if self.config[ 'encoding'] is not None else sys.getdefaultencoding() self.decoding_method = self.config['decoding'] if self.config[ 'decoding'] is not None else locale.getpreferredencoding() self.logger.info( self.tr('mcdr_server.on_config_changed.encoding_decoding_set', self.encoding_method, self.decoding_method)) self.plugin_manager.set_plugin_directories( self.config['plugin_directories']) self.logger.info( self.tr('mcdr_server.on_config_changed.plugin_directories_set', self.encoding_method, self.decoding_method)) for directory in self.plugin_manager.plugin_directories: self.logger.info('- {}'.format(directory)) self.reactor_manager.register_reactors( self.config['custom_info_reactors']) self.server_handler_manager.register_handlers( self.config['custom_handlers']) self.server_handler_manager.set_handler(self.config['handler']) self.logger.info( self.tr('mcdr_server.on_config_changed.handler_set', self.config['handler'])) self.connect_rcon() def load_plugins(self): self.plugin_manager.refresh_all_plugins() self.logger.info( self.plugin_manager.last_operation_result.to_rtext(show_path=True)) def on_plugin_changed(self): self.command_manager.clear_command() self.plugin_manager.registry_storage.export_commands( self.command_manager.register_command) # --------------------------- # State Getters / Setters # --------------------------- def is_server_running(self): return self.server_state.in_state( {ServerState.RUNNING, ServerState.STOPPING}) # Flags def is_server_startup(self): return MCDReforgedFlag.SERVER_STARTUP in self.flags def is_server_rcon_ready(self): return MCDReforgedFlag.SERVER_RCON_READY in self.flags def is_interrupt(self): return MCDReforgedFlag.INTERRUPT in self.flags def is_mcdr_exit(self): return self.mcdr_in_state(MCDReforgedState.STOPPED) def is_exit_naturally(self): return MCDReforgedFlag.EXIT_NATURALLY in self.flags def with_flag(self, flag: MCDReforgedFlag): self.flags |= flag def remove_flag(self, flag: MCDReforgedFlag): self.flags &= ~flag def set_exit_naturally(self, flag): if flag: self.with_flag(MCDReforgedFlag.EXIT_NATURALLY) else: self.remove_flag(MCDReforgedFlag.EXIT_NATURALLY) self.logger.debug('flag_exit_naturally has set to "{}"'.format(flag), option=DebugOption.MCDR) # State def server_in_state(self, states): return self.server_state.in_state(states) def mcdr_in_state(self, states): return self.mcdr_state.in_state(states) def is_initialized(self): return self.mcdr_in_state(MCDReforgedState.INITIALIZED) def set_server_state(self, state): self.server_state = state self.logger.debug('Server state has set to "{}"'.format(state), option=DebugOption.MCDR) def set_mcdr_state(self, state): self.mcdr_state = state self.logger.debug('MCDR state has set to "{}"'.format(state), option=DebugOption.MCDR) def should_keep_looping(self): """ A criterion for sub threads to determine if it should keep looping :rtype: bool """ if self.server_in_state(ServerState.STOPPED): if self.is_interrupt(): # if interrupted and stopped return False return not self.is_exit_naturally( ) # if the sever exited naturally, exit MCDR return not self.is_mcdr_exit() # -------------------------- # Server Controls # -------------------------- def start_server(self): """ try to start the server process return True if the server process has started successfully return False if the server is not able to start :return: a bool as above :rtype: bool """ with self.starting_server_lock: if self.is_interrupt(): self.logger.warning( self.tr('mcdr_server.start_server.already_interrupted')) return False if self.is_server_running(): self.logger.warning( self.tr('mcdr_server.start_server.start_twice')) return False cwd = self.config['working_directory'] if not os.path.isdir(cwd): self.logger.error( self.tr('mcdr_server.start_server.cwd_not_existed', cwd)) return False try: start_command = self.config['start_command'] self.logger.info( self.tr('mcdr_server.start_server.starting', start_command)) self.process = Popen(start_command, cwd=self.config['working_directory'], stdin=PIPE, stdout=PIPE, stderr=STDOUT, shell=True) except: self.logger.exception( self.tr('mcdr_server.start_server.start_fail')) return False else: self.on_server_start() return True def kill_server(self): """ Kill the server process group """ if self.process and self.process.poll() is None: self.logger.info(self.tr('mcdr_server.kill_server.killing')) try: for child in psutil.Process( self.process.pid).children(recursive=True): child.kill() self.logger.info( self.tr('mcdr_server.kill_server.process_killed', child.pid)) except psutil.NoSuchProcess: pass self.process.kill() self.logger.info( self.tr('mcdr_server.kill_server.process_killed', self.process.pid)) else: raise IllegalCallError( "Server process has already been terminated") def interrupt(self): """ Interrupt MCDR The first call will softly stop the server and the later calls will kill the server Return if it's the first try :rtype: bool """ self.logger.info( 'Interrupting, first strike = {}'.format(not self.is_interrupt())) self.stop(forced=self.is_interrupt()) ret = self.is_interrupt() self.with_flag(MCDReforgedFlag.INTERRUPT) return ret def stop(self, forced=False): """ Stop the server :param forced: an optional bool. If it's False (default) MCDR will stop the server by sending the STOP_COMMAND from the current handler. If it's True MCDR will just kill the server process group """ with self.stop_lock: if not self.is_server_running(): self.logger.warning( self.tr('mcdr_server.stop.stop_when_stopped')) return self.set_server_state(ServerState.STOPPING) if not forced: try: self.send(self.server_handler_manager.get_current_handler( ).get_stop_command()) except: self.logger.error(self.tr('mcdr_server.stop.stop_fail')) forced = True if forced: try: self.kill_server() except IllegalCallError: pass # -------------------------- # Server Logics # -------------------------- def on_server_start(self): self.set_exit_naturally(True) self.logger.info( self.tr('mcdr_server.start_server.pid_info', self.process.pid)) self.plugin_manager.dispatch_event(MCDRPluginEvents.SERVER_START, ()) self.set_server_state(ServerState.RUNNING) def on_server_stop(self): return_code = self.process.poll() self.logger.info( self.tr('mcdr_server.on_server_stop.show_stopcode', return_code)) self.process.stdin.close() self.process.stdout.close() self.process = None self.remove_flag( MCDReforgedFlag.SERVER_STARTUP | MCDReforgedFlag.SERVER_RCON_READY) # removes this two self.plugin_manager.dispatch_event(MCDRPluginEvents.SERVER_STOP, (return_code, )) self.set_server_state(ServerState.STOPPED) def send(self, text, ending='\n', encoding=None): """ Send a text to server's stdin if the server is running :param text: A str or a bytes you want to send. if text is a str then it will attach the ending parameter to its back :param str ending: The suffix of a command with a default value \n :param str encoding: The encoding method for the text. If it's not given used the method in config """ if encoding is None: encoding = self.encoding_method if type(text) is str: text = (text + ending).encode(encoding) if self.is_server_running(): self.process.stdin.write(text) self.process.stdin.flush() else: self.logger.warning(self.tr('mcdr_server.send.send_when_stopped')) def receive(self): """ Try to receive a str from server's stdout. This will block the current thread If server has stopped it will wait up to 10s for the server process to exit, then raise a ServerStopped exception :rtype: str :raise: ServerStopped """ while True: try: text = next(iter(self.process.stdout)) except StopIteration: # server process has stopped for i in range(constant.WAIT_TIME_AFTER_SERVER_STDOUT_END_SEC * 10): if self.process.poll() is not None: break time.sleep(0.1) if i % 10 == 0: self.logger.info( self.tr('mcdr_server.receive.wait_stop')) raise ServerStopped() else: try: text = text.decode(self.decoding_method) except: self.logger.error( self.tr('mcdr_server.receive.decode_fail', text)) raise return text.rstrip('\n\r').lstrip('\n\r') def tick(self): """ ticking MCDR: try to receive a new line from server's stdout and parse / display / process the text """ try: text = self.receive() except ServerStopped: self.on_server_stop() return try: text = self.server_handler_manager.get_current_handler( ).pre_parse_server_stdout(text) except: self.logger.warning(self.tr('mcdr_server.tick.pre_parse_fail')) parsed_result: Info try: parsed_result = self.server_handler_manager.get_current_handler( ).parse_server_stdout(text) except: if self.logger.should_log_debug( option=DebugOption.HANDLER ): # traceback.format_exc() is costly self.logger.debug( 'Fail to parse text "{}" from stdout of the server, using raw handler' .format(text)) for line in traceback.format_exc().splitlines(): self.logger.debug(' {}'.format(line)) parsed_result = self.server_handler_manager.get_basic_handler( ).parse_server_stdout(text) else: if self.logger.should_log_debug(option=DebugOption.HANDLER): self.logger.debug('Parsed text from server stdin:') for line in parsed_result.format_text().splitlines(): self.logger.debug(' {}'.format(line)) self.server_handler_manager.detect_text(text) self.reactor_manager.put_info(parsed_result) def on_mcdr_start(self): self.task_executor.start() self.task_executor.enqueue_regular_task(self.load_plugins) self.task_executor.wait_till_finish_all_task() self.plugin_manager.dispatch_event(MCDRPluginEvents.MCDR_START, ()) if not self.config['disable_console_thread']: self.console_handler.start() else: self.logger.info( self.tr('mcdr_server.on_mcdr_start.console_disabled')) if not self.start_server(): raise ServerStartError() self.update_helper.start() self.watch_dog.start() self.server_handler_manager.start_handler_detection() self.set_mcdr_state(MCDReforgedState.RUNNING) def on_mcdr_stop(self): try: self.set_mcdr_state(MCDReforgedState.PRE_STOPPED) if self.is_interrupt(): self.logger.info( self.tr('mcdr_server.on_mcdr_stop.user_interrupted')) else: self.logger.info( self.tr('mcdr_server.on_mcdr_stop.server_stop')) self.watch_dog.stop() # it's ok for plugins to take some time self.watch_dog.join() self.plugin_manager.dispatch_event(MCDRPluginEvents.MCDR_STOP, ()) self.task_executor.wait_till_finish_all_task() self.logger.info(self.tr('mcdr_server.on_mcdr_stop.bye')) except KeyboardInterrupt: # I don't know why there sometimes will be a KeyboardInterrupt if MCDR is stopped by ctrl-c pass except: self.logger.exception( self.tr('mcdr_server.on_mcdr_stop.stop_error')) finally: self.set_mcdr_state(MCDReforgedState.STOPPED) def start(self): """ The entry method to start MCDR Try to start the server. if succeeded the console thread will start and MCDR will start ticking :raise: IllegalStateError if MCDR is in wrong state :raise: ServerStartError if the server is already running or start_server has been called by other """ if not self.mcdr_in_state(MCDReforgedState.INITIALIZED): if self.mcdr_in_state(MCDReforgedState.INITIALIZING): raise IllegalStateError( 'This instance is not fully initialized') else: raise IllegalStateError('MCDR can only start once') self.main_loop() return self.process def main_loop(self): """ The main loop of MCDR """ self.on_mcdr_start() while self.should_keep_looping(): try: if self.is_server_running(): self.tick() else: time.sleep(0.01) except KeyboardInterrupt: self.interrupt() except: if self.is_interrupt(): break else: self.logger.critical(self.tr('mcdr_server.run.error'), exc_info=True) self.on_mcdr_stop() def connect_rcon(self): self.rcon_manager.disconnect() if self.config['rcon']['enable'] and self.is_server_rcon_ready(): self.rcon_manager.connect(self.config['rcon']['address'], self.config['rcon']['port'], self.config['rcon']['password'])
class Z3Solver(Solver): def __init__(self): ''' Build a Z3 solver instance. This is implemented using an external z3 solver (via a subprocess). ''' super(Z3Solver, self).__init__() self._proc = None self._log = '' # this should be enabled only if we are debugging self.version = self._solver_version() self.support_maximize = False self.support_minimize = False self.support_reset = True logger.debug('Z3 version: %s', self.version) if self.version >= Version(4, 5, 0): self.support_maximize = False self.support_minimize = False self.support_reset = True elif self.version >= Version(4, 4, 1): self.support_maximize = True self.support_minimize = True self.support_reset = False else: logger.debug( ' Please install Z3 4.4.1 or newer to get optimization support' ) self._command = 'z3 -t:120000 -smt2 -in' self._init = [ '(set-logic QF_AUFBV)', '(set-option :global-decls false)' ] self._get_value_fmt = ( re.compile('\(\((?P<expr>(.*))\ #x(?P<value>([0-9a-fA-F]*))\)\)'), 16) @staticmethod def _solver_version(): ''' If we fail to parse the version, we assume z3's output has changed, meaning it's a newer version than what's used now, and therefore ok. Anticipated version_cmd_output format: 'Z3 version 4.4.2' 'Z3 version 4.4.5 - 64 bit - build hashcode $Z3GITHASH' ''' their_version = Version(0, 0, 0) try: version_cmd_output = check_output('z3 -version'.split()) except OSError: raise Z3NotFoundError try: version = version_cmd_output.split()[2] their_version = Version(*map(int, version.split('.'))) except (IndexError, ValueError, TypeError): pass return their_version def _start_proc(self): ''' Auxiliary method to spawn the external solver process''' assert '_proc' not in dir(self) or self._proc is None try: self._proc = Popen(self._command.split(' '), stdin=PIPE, stdout=PIPE) except OSError: # Z3 was removed from the system in the middle of operation raise Z3NotFoundError # TODO(mark) don't catch this exception in two places # run solver specific initializations for cfg in self._init: self._send(cfg) def _stop_proc(self): ''' Auxiliary method to stop the external solver process''' if self._proc is not None: self._send("(exit)") self._proc.stdin.close() self._proc.stdout.close() self._proc.wait() try: self._proc.kill() except BaseException: pass self._proc = None # marshaling/pickle def __getstate__(self): raise Exception() def __setstate__(self, state): raise Exception() def __del__(self): try: self._proc.stdin.writelines(('(exit)\n', )) self._proc.wait() except Exception: pass def _reset(self, constraints=None): ''' Auxiliary method to reset the smtlib external solver to initial defaults''' if self._proc is None: self._start_proc() else: if self.support_reset: self._send("(reset)") for cfg in self._init: self._send(cfg) else: self._stop_proc() self._start_proc() if constraints is not None: self._send(constraints) def _send(self, cmd): ''' Send a string to the solver. :param cmd: a SMTLIBv2 command (ex. (check-sat)) ''' logger.debug('>%s', cmd) self._log += str(cmd) + '\n' try: buf = str(cmd) self._proc.stdin.write(buf + '\n') except IOError as e: raise SolverException(e) def _recv(self): ''' Reads the response from the solver ''' def readline(): buf = self._proc.stdout.readline() return buf, buf.count('('), buf.count(')') bufl = [] left = 0 right = 0 buf, l, r = readline() bufl.append(buf) left += l right += r while left != right: buf, l, r = readline() bufl.append(buf) left += l right += r buf = ''.join(bufl).strip() logger.debug('<%s', buf) if '(error' in bufl[0]: raise Exception("Error in smtlib: {}".format(bufl[0])) return buf # UTILS: check-sat get-value def _check(self): ''' Check the satisfiability of the current state ''' logger.debug("Solver.check() ") start = time.time() self._send('(check-sat)') _status = self._recv() logger.debug("Check took %s seconds (%s)", time.time() - start, _status) if _status not in ('sat', 'unsat', 'unknown'): raise SolverException(_status) if consider_unknown_as_unsat: if _status == 'unknown': logger.warning( 'Found an unknown core, probably a solver timeout') _status = 'unsat' if _status == 'unknown': raise SolverUnknown(_status) return _status def _assert(self, expression): ''' Auxiliary method to send an assert ''' assert isinstance(expression, Bool) smtlib = translate_to_smtlib(expression) self._send('(assert %s)' % smtlib) def _getvalue(self, expression): ''' Ask the solver for one possible assignment for val using current set of constraints. The current set of assertions must be sat. :param val: an expression or symbol ''' if not issymbolic(expression): return expression assert isinstance(expression, Variable) self._send('(get-value (%s))' % expression.name) ret = self._recv() assert ret.startswith('((') and ret.endswith('))') if isinstance(expression, Bool): return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]] elif isinstance(expression, BitVec): pattern, base = self._get_value_fmt m = pattern.match(ret) expr, value = m.group('expr'), m.group('value') return int(value, base) raise NotImplementedError( "_getvalue only implemented for Bool and BitVec") # push pop def _push(self): ''' Pushes and save the current constraint store and state.''' self._send('(push 1)') def _pop(self): ''' Recall the last pushed constraint store and state. ''' self._send('(pop 1)') #@memoized def can_be_true(self, constraints, expression): ''' Check if two potentially symbolic values can be equal ''' if isinstance(expression, bool): if not expression: return expression else: expression = BoolConstant(expression) assert isinstance(expression, Bool) with constraints as temp_cs: temp_cs.add(expression) self._reset(temp_cs.related_to(expression)) return self._check() == 'sat' # get-all-values min max minmax #@memoized def get_all_values(self, constraints, expression, maxcnt=3000, silent=False): ''' Returns a list with all the possible values for the symbol x''' if not isinstance(expression, Expression): return [expression] assert isinstance(constraints, ConstraintSet) assert isinstance(expression, Expression) with constraints as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) else: raise NotImplementedError( "get_all_values only implemted for Bool and BitVec") temp_cs.add(var == expression) self._reset(temp_cs.related_to(var)) result = [] val = None while self._check() == 'sat': value = self._getvalue(var) result.append(value) # Reset the solver to avoid the incremental mode # Triggered with two consecutive calls to check-sat # Yet, if the number of solution is large, sending back # the whole formula is more expensive if len(result) < 50: self._reset(temp_cs.related_to(var)) for value in result: self._assert(var != value) else: self._assert(var != value) if len(result) >= maxcnt: if silent: # do not throw an exception if set to silent # Default is not silent, assume user knows # what they are doing and will check the size # of returned vals list (previous smtlib behavior) break else: raise TooManySolutions(result) return result #@memoized def optimize(self, constraints, x, goal, M=10000): ''' Iterativelly finds the maximum or minimal value for the operation (Normally Operators.UGT or Operators.ULT) :param X: a symbol or expression :param M: maximum number of iterations allowed ''' assert goal in ('maximize', 'minimize') assert isinstance(x, BitVec) operation = { 'maximize': Operators.UGE, 'minimize': Operators.ULE }[goal] with constraints as temp_cs: X = temp_cs.new_bitvec(x.size) temp_cs.add(X == x) aux = temp_cs.new_bitvec(X.size, name='optimized_') self._reset(temp_cs) self._send(aux.declaration) if getattr(self, 'support_{}'.format(goal)): self._push() try: self._assert(operation(X, aux)) self._send('(%s %s)' % (goal, aux.name)) self._send('(check-sat)') _status = self._recv() if _status not in ('sat', 'unsat', 'unknown'): # Minimize (or Maximize) sometimes prints the objective before the status # This will be a line like NAME |-> VALUE maybe_sat = self._recv() if maybe_sat == 'sat': pattern = re.compile( '(?P<expr>.*?)\s+\|->\s+(?P<value>.*)', re.DOTALL) m = pattern.match(_status) expr, value = m.group('expr'), m.group('value') assert expr == aux.name return int(value) elif _status == 'sat': ret = self._recv() if not (ret.startswith('(') and ret.endswith(')')): raise SolverException( 'bad output on max, z3 may have been killed') pattern = re.compile( '\(objectives.*\((?P<expr>.*) (?P<value>\d*)\).*\).*', re.MULTILINE | re.DOTALL) m = pattern.match(ret) expr, value = m.group('expr'), m.group('value') assert expr == aux.name return int(value) finally: self._pop() self._reset(temp_cs) self._send(aux.declaration) operation = { 'maximize': Operators.UGT, 'minimize': Operators.ULT }[goal] self._assert(aux == X) last_value = None i = 0 while self._check() == 'sat': last_value = self._getvalue(aux) self._assert(operation(aux, last_value)) i = i + 1 if (i > M): raise SolverException( "Optimizing error, maximum number of iterations was reached" ) if last_value is not None: return last_value raise SolverException("Optimizing error, unsat or unknown core") #@memoized def get_value(self, constraints, expression): ''' Ask the solver for one possible assignment for val using current set of constraints. The current set of assertions must be sat. :param val: an expression or symbol ''' if not issymbolic(expression): return expression assert isinstance(expression, (Bool, BitVec, Array)) with constraints as temp_cs: if isinstance(expression, Bool): var = temp_cs.new_bool() elif isinstance(expression, BitVec): var = temp_cs.new_bitvec(expression.size) elif isinstance(expression, Array): var = [] result = '' for i in xrange(expression.index_max): subvar = temp_cs.new_bitvec(expression.value_bits) var.append(subvar) temp_cs.add(subvar == expression[i]) self._reset(temp_cs) if self._check() != 'sat': raise SolverException('Model is not available') for i in xrange(expression.index_max): self._send('(get-value (%s))' % var[i].name) ret = self._recv() assert ret.startswith('((') and ret.endswith('))') pattern, base = self._get_value_fmt m = pattern.match(ret) expr, value = m.group('expr'), m.group('value') result += chr(int(value, base)) return result temp_cs.add(var == expression) self._reset(temp_cs) if self._check() != 'sat': raise SolverException('Model is not available') self._send('(get-value (%s))' % var.name) ret = self._recv() if not (ret.startswith('((') and ret.endswith('))')): raise SolverException('SMTLIB error parsing response: %s' % ret) if isinstance(expression, Bool): return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]] if isinstance(expression, BitVec): pattern, base = self._get_value_fmt m = pattern.match(ret) expr, value = m.group('expr'), m.group('value') return int(value, base) raise NotImplementedError( "get_value only implemented for Bool and BitVec")
class CloudSqlProxyRunner(LoggingMixin): """ Downloads and runs cloud-sql-proxy as subprocess of the Python process. The cloud-sql-proxy needs to be downloaded and started before we can connect to the Google Cloud SQL instance via database connection. It establishes secure tunnel connection to the database. It authorizes using the GCP credentials that are passed by the configuration. More details about the proxy can be found here: https://cloud.google.com/sql/docs/mysql/sql-proxy """ def __init__(self, path_prefix, instance_specification, gcp_conn_id='google_cloud_default', project_id=None, sql_proxy_version=None, sql_proxy_binary_path=None): """ Creates the proxy runner class. :param path_prefix: Unique path prefix where proxy will be downloaded and directories created for unix sockets. :type path_prefix: str :param instance_specification: Specification of the instance to connect the proxy to. It should be specified in the form that is described in https://cloud.google.com/sql/docs/mysql/sql-proxy#multiple-instances in -instances parameter (typically in the form of ``<project>:<region>:<instance>`` for UNIX socket connections and in the form of ``<project>:<region>:<instance>=tcp:<port>`` for TCP connections. :type instance_specification: str :param gcp_conn_id: Id of Google Cloud Platform connection to use for authentication :type gcp_conn_id: str :param project_id: Optional id of the GCP project to connect to - it overwrites default project id taken from the GCP connection. :type project_id: str :param sql_proxy_version: Specific version of SQL proxy to download (for example 'v1.13'). By default latest version is downloaded. :type sql_proxy_version: str :param sql_proxy_binary_path: If specified, then proxy will be used from the path specified rather than dynamically generated. This means that if the binary is not present in that path it will also be downloaded. :type sql_proxy_binary_path: str """ super(CloudSqlProxyRunner, self).__init__() self.path_prefix = path_prefix if not self.path_prefix: raise AirflowException("The path_prefix must not be empty!") self.sql_proxy_was_downloaded = False self.sql_proxy_version = sql_proxy_version self.download_sql_proxy_dir = None self.sql_proxy_process = None self.instance_specification = instance_specification self.project_id = project_id self.gcp_conn_id = gcp_conn_id self.command_line_parameters = [] self.cloud_sql_proxy_socket_directory = self.path_prefix self.sql_proxy_path = sql_proxy_binary_path if sql_proxy_binary_path \ else self.path_prefix + "_cloud_sql_proxy" self.credentials_path = self.path_prefix + "_credentials.json" self._build_command_line_parameters() def _build_command_line_parameters(self): self.command_line_parameters.extend( ['-dir', self.cloud_sql_proxy_socket_directory]) self.command_line_parameters.extend( ['-instances', self.instance_specification]) @staticmethod def _is_os_64bit(): return platform.machine().endswith('64') def _download_sql_proxy_if_needed(self): if os.path.isfile(self.sql_proxy_path): self.log.info("cloud-sql-proxy is already present") return system = platform.system().lower() processor = "amd64" if CloudSqlProxyRunner._is_os_64bit() else "386" if not self.sql_proxy_version: download_url = CLOUD_SQL_PROXY_DOWNLOAD_URL.format( system, processor) else: download_url = CLOUD_SQL_PROXY_VERSION_DOWNLOAD_URL.format( self.sql_proxy_version, system, processor) proxy_path_tmp = self.sql_proxy_path + ".tmp" self.log.info("Downloading cloud_sql_proxy from %s to %s", download_url, proxy_path_tmp) r = requests.get(download_url, allow_redirects=True) # Downloading to .tmp file first to avoid case where partially downloaded # binary is used by parallel operator which uses the same fixed binary path with open(proxy_path_tmp, 'wb') as f: f.write(r.content) if r.status_code != 200: raise AirflowException( "The cloud-sql-proxy could not be downloaded. Status code = {}. " "Reason = {}".format(r.status_code, r.reason)) self.log.info("Moving sql_proxy binary from %s to %s", proxy_path_tmp, self.sql_proxy_path) shutil.move(proxy_path_tmp, self.sql_proxy_path) os.chmod(self.sql_proxy_path, 0o744) # Set executable bit self.sql_proxy_was_downloaded = True @provide_session def _get_credential_parameters(self, session): connection = session.query(Connection). \ filter(Connection.conn_id == self.gcp_conn_id).first() session.expunge_all() if GCP_CREDENTIALS_KEY_PATH in connection.extra_dejson: credential_params = [ '-credential_file', connection.extra_dejson[GCP_CREDENTIALS_KEY_PATH] ] elif GCP_CREDENTIALS_KEYFILE_DICT in connection.extra_dejson: credential_file_content = json.loads( connection.extra_dejson[GCP_CREDENTIALS_KEYFILE_DICT]) self.log.info("Saving credentials to %s", self.credentials_path) with open(self.credentials_path, "w") as f: json.dump(credential_file_content, f) credential_params = ['-credential_file', self.credentials_path] else: self.log.info( "The credentials are not supplied by neither key_path nor " "keyfile_dict of the gcp connection %s. Falling back to " "default activated account", self.gcp_conn_id) credential_params = [] if not self.instance_specification: project_id = connection.extra_dejson.get( 'extra__google_cloud_platform__project') if self.project_id: project_id = self.project_id if not project_id: raise AirflowException( "For forwarding all instances, the project id " "for GCP should be provided either " "by project_id extra in the GCP connection or by " "project_id provided in the operator.") credential_params.extend(['-projects', project_id]) return credential_params def start_proxy(self): """ Starts Cloud SQL Proxy. You have to remember to stop the proxy if you started it! """ self._download_sql_proxy_if_needed() if self.sql_proxy_process: raise AirflowException( "The sql proxy is already running: {}".format( self.sql_proxy_process)) else: command_to_run = [self.sql_proxy_path] command_to_run.extend(self.command_line_parameters) try: self.log.info("Creating directory %s", self.cloud_sql_proxy_socket_directory) os.makedirs(self.cloud_sql_proxy_socket_directory) except OSError: # Needed for python 2 compatibility (exists_ok missing) pass command_to_run.extend(self._get_credential_parameters()) self.log.info("Running the command: `%s`", " ".join(command_to_run)) self.sql_proxy_process = Popen(command_to_run, stdin=PIPE, stdout=PIPE, stderr=PIPE) self.log.info("The pid of cloud_sql_proxy: %s", self.sql_proxy_process.pid) while True: line = self.sql_proxy_process.stderr.readline().decode('utf-8') return_code = self.sql_proxy_process.poll() if line == '' and return_code is not None: self.sql_proxy_process = None raise AirflowException( "The cloud_sql_proxy finished early with return code {}!" .format(return_code)) if line != '': self.log.info(line) if "googleapi: Error" in line or "invalid instance name:" in line: self.stop_proxy() raise AirflowException( "Error when starting the cloud_sql_proxy {}!".format( line)) if "Ready for new connections" in line: return def stop_proxy(self): """ Stops running proxy. You should stop the proxy after you stop using it. """ if not self.sql_proxy_process: raise AirflowException("The sql proxy is not started yet") else: self.log.info("Stopping the cloud_sql_proxy pid: %s", self.sql_proxy_process.pid) self.sql_proxy_process.kill() self.sql_proxy_process = None # Cleanup! self.log.info("Removing the socket directory: %s", self.cloud_sql_proxy_socket_directory) shutil.rmtree(self.cloud_sql_proxy_socket_directory, ignore_errors=True) if self.sql_proxy_was_downloaded: self.log.info("Removing downloaded proxy: %s", self.sql_proxy_path) # Silently ignore if the file has already been removed (concurrency) try: os.remove(self.sql_proxy_path) except OSError as e: if not e.errno == errno.ENOENT: raise else: self.log.info("Skipped removing proxy - it was not downloaded: %s", self.sql_proxy_path) if os.path.isfile(self.credentials_path): self.log.info("Removing generated credentials file %s", self.credentials_path) # Here file cannot be delete by concurrent task (each task has its own copy) os.remove(self.credentials_path) def get_proxy_version(self): """ Returns version of the Cloud SQL Proxy. """ self._download_sql_proxy_if_needed() command_to_run = [self.sql_proxy_path] command_to_run.extend(['--version']) command_to_run.extend(self._get_credential_parameters()) result = subprocess.check_output(command_to_run).decode('utf-8') pattern = re.compile("^.*[V|v]ersion ([^;]*);.*$") m = pattern.match(result) if m: return m.group(1) else: return None def get_socket_path(self): """ Retrieves UNIX socket path used by Cloud SQL Proxy. :return: The dynamically generated path for the socket created by the proxy. :rtype: str """ return self.cloud_sql_proxy_socket_directory + "/" + self.instance_specification
def importdata(self): env = os.environ.copy() env["PROJ_LIB"] = "c:\\OSGeo4W64\\share\\proj" sql = "select id from datafile where filename=? and md5=?" self.db.cursor.execute(sql, [self.filename, self.md5]) fileid = self.db.cursor.fetchall()[0][0] args = [self.proj] args.extend(self.projparams) with open(self.filename, 'r') as csvfile: breaking = True breaking = False csvreader = csv.reader(csvfile, delimiter=",") i = 0 header = [] insmeas = "insert into detectormeasure (measurementid,vd,ndet, livetime, rawdose, dose,totcount,spectra) values(?,?,?,?,?,?,?,?) " vds = ['VD1', 'VD2', 'VD3', 'VD4'] for row in csvreader: i = i + 1 if i == 2: top = row if not "GPS Sample" in top: print(top) print("No GPS data") break # Makes no sense to continue if no GPS data roistarts = [ i for i, s in enumerate(top) if 'ROI for Virtual Detector' in s ] spectrumstarts = [ i for i, s in enumerate(top) if 'Spectrum VD' in s ] try: specchs = spectrumstarts[1] - spectrumstarts[0] except IndexError as error: specchs = 1025 # Must be one extra vdstart = [idx for idx, s in enumerate(top) if '[1]' in s][0] print(roistarts) print(spectrumstarts) print(vdstart) print("===") if i == 3: header = row meashead = dict(zip(header[0:vdstart], range(0, vdstart))) print(meashead) insert = "insert into measurement(datafileid,seqnum,systemtime,aqutimeus,GPSSmplFlags,GpsError,GpsTime,PDOP,longitude,latitude,GPSAltitude,LineNum" values = " values(?,?,convert(datetime,?,103),?,?,?,convert(datetime,?,103),?,?,?,?,?" adcs = False pres = False if "ADC 1" in header: # ADCs - for altitude measurement May or may not be exported insert = insert + ",adc1,adc2" values = values + ",?,?" adcs = True if "Pres" in header: # Pressure and temperature insert = insert + ",Pres,Temp" values = values + ",?,?" pres = True #insert=insert+",location,location_utm) "+values+",geometry::STGeomFromText('POINT ({} {})',4326),geometry::STGeomFromText('POINT ({})',32633)) " insert = insert + ",location_utm) " + values + ",geometry::STGeomFromText('POINT ({})',32633)) " # Analyze header to see that all that is needed is there... if i > 3: store = False for vd in vds: if self.req[vd][0] == '': continue vd = int(self.req[vd][0]) store = store or int(row[roistarts[vd - 1]]) > 0 # no reason to store data where the detector not yet is up if not store: continue lon = row[meashead["Long"]] lat = row[meashead["Lat"]] projcoord = lon + " " + lat + "" projcoord = projcoord.encode() process = Popen(args, env=env, stdin=PIPE, stdout=PIPE) utmcoord = process.communicate(input=projcoord)[0].decode() if breaking: print(projcoord) print(utmcoord) process.kill() params = [fileid] params.append(row[meashead["SeqNum"]]) params.append(row[meashead["UtcDate"]] + " " + row[meashead["UtcTime"]]) params.append(row[meashead["µs"]]) # nothing in row[4] params.append(row[meashead["SmplFlags"]]) params.append(row[meashead["GpsError"]]) params.append(row[meashead["GpsUtcDate"]] + " " + row[meashead["GpsUtcTime"]]) params.append(row[meashead["PDOP"]]) params.append(lon) params.append(lat) params.append(row[meashead["Alt[m]"]]) params.append(row[meashead["LineNum"]]) if adcs: params.append(row[meashead["ADC 1"]]) params.append(row[meashead["ADC 2"]]) if pres: params.append(row[meashead["Pres"]]) params.append(row[meashead["Temp"]]) params = list(map(lambda x: 0 if x == '' else x, params)) #self.db.cursor.execute(insert.format(lon,lat,utmcoord),params) self.db.cursor.execute( insert.format(utmcoord), params ) # Using format to insert coordinates in the Geom from text - normal parameters do not work here self.db.cursor.execute( "select IDENT_CURRENT('measurement')" ) # Get last id - may fail id = self.db.cursor.fetchall() measid = id[0][0] if i % 100 == 0: print(i, measid) for vd in vds: if self.req[vd][0] == '': continue vd = int(self.req[vd][0]) roi = roistarts[vd - 1] params = [measid, vd] params.extend( row[roi:roi + 5] ) # DetCount, LiveTime, Raw Doserate, Doserate TotCount[cps] specst = spectrumstarts[vd - 1] spectrum = ":".join(row[specst:specst + specchs]) params.append(spectrum) # print("VD"+str(vd)) self.db.cursor.execute(insmeas, params) #break # Jumps out if no data for this VD if i > 10 and breaking: break sql = "update datafile set finished = 1 where id = ?" self.db.cursor.execute(sql, fileid) self.db.cursor.commit() message = "Uploaded OK" return (message)
class Pqos(object): """ The Pqos class defines methods to interact with pqos_wrapper cli. """ CAP_SYS_RAWIO = "cap_sys_rawio" def __init__(self, show_warnings=False): self.reset_required = False self.show_warnings = show_warnings self.mon_process = None self.executable_path = find_executable2("pqos_wrapper") if self.executable_path is None: if self.show_warnings: logging.info( "Unable to find pqos_wrapper, please install it for " "cache allocation and monitoring if your CPU supports Intel RDT " "(cf. https://gitlab.com/sosy-lab/software/pqos-wrapper).") def execute_command(self, __type, function, suppress_warning, *args): """ Execute a given pqos_wrapper command and log the output @__type: The type of command being executed (monitoring or l3ca) @function_name: The name of the function being executed in pqos_wrapper @suppress_warning: A boolean to decide wether to print warning on failing execution """ if self.executable_path: args_list = [self.executable_path] + list(args) try: if "-m" in args_list: self.mon_process = Popen(args_list, stdout=PIPE, stderr=PIPE) else: ret = json.loads( check_output(args_list, stderr=STDOUT).decode()) logging.debug(ret[function]["message"]) return True except CalledProcessError as e: if self.show_warnings and (not suppress_warning): self.print_error_message(e.output.decode(), __type, args_list) return False def print_error_message(self, err, __type, args_list): """ Prints error message returned from pqos_wrapper @err: The error output returned by pqos_wrapper @__type: The type of command being executed (monitoring or l3ca) @args_list: The command being executed as a list """ msg_prefix = { "mon": "Could not monitor events", "l3ca": "Could not set cache allocation", } try: ret = json.loads(err) logging.warning("{0}...{1}".format(msg_prefix[__type], ret["message"])) self.check_for_errors() except ValueError: logging.warning("{0}...Unable to execute command {1}".format( msg_prefix[__type], " ".join(args_list))) def check_capacity(self, technology): """ Check if given intel rdt is supported. @technology: The intel rdt to be tested """ return self.execute_command(technology, "check_capability", False, "-c", technology) @staticmethod def convert_core_list(core_assignment): """ Convert a double list to a string. @core_assignment: The double list of cores """ ret = [] for benchmark in core_assignment: ret.append("[" + ",".join(str(core) for core in benchmark) + "]") return "[" + ",".join(ret) + "]" def allocate_l3ca(self, core_assignment): """ This method checks if L3CAT is available and calls pqos_wrapper to allocate equal cache to each thread. @core_assignment: The list of cores assigned to each run """ if self.check_capacity("l3ca"): core_string = self.convert_core_list(core_assignment) if self.execute_command("l3ca", "allocate_resource", False, "-a", "l3ca", core_string): self.reset_required = True else: self.reset_resources() def start_monitoring(self, core_assignment): """ This method checks if monitoring capability is available and calls pqos_wrapper to monitor events on given lists of cores. @core_assignment: The list of cores assigned to each run """ if self.check_capacity("mon"): core_string = self.convert_core_list(core_assignment) self.execute_command("mon", "monitor_events", False, "-m", core_string) def stop_monitoring(self): """ This method stops monitoring by sending SIGINT to the monitoring process and resets the RMID for monitored cores to 0 """ ret = {} if self.mon_process: self.mon_process.send_signal(SIGINT) mon_output = self.mon_process.communicate() if self.mon_process.returncode == 0: mon_data = json.loads(mon_output[0].decode()) logging.debug(mon_data["monitor_events"]["message"]) ret = self.flatten_mon_data( mon_data["monitor_events"]["function_output"] ["monitoring_data"]) else: if self.show_warnings: self.print_error_message(mon_output[1].decode(), "mon", self.mon_process.args) self.mon_process.kill() self.mon_process = None else: if self.show_warnings: logging.warning("No monitoring process started") return ret def reset_monitoring(self): """ Reset monitoring RMID to 0 for all cores """ self.execute_command("mon", "reset_monitoring", True, "-rm") @staticmethod def flatten_mon_data(mon_data): """ Converts the monitoring data array received from pqos_wrapper to a flattened dictionary @mon_data: The array of data received from pqos_wrapper monitoring cli """ flatten_dict = {} for data in mon_data: core_str = ",".join(str(core) for core in data["cores"]) data.pop("cores", None) for key, val in data.items(): if isinstance(val, dict): for sub_key, sub_val in val.items(): if len(mon_data) > 1: flatten_key = "{0}_{1}_cpus{2}".format( key, sub_key, core_str) else: flatten_key = "{0}_{1}".format(key, sub_key) flatten_dict[flatten_key] = sub_val else: if len(mon_data) > 1: flatten_key = "{0}_cpus{1}".format(key, core_str) else: flatten_key = key flatten_dict[flatten_key] = val return flatten_dict def reset_resources(self): """ This method resets all resources to default. """ if self.reset_required: self.execute_command("l3ca", "reset_resources", True, "-r") self.reset_required = False def check_for_errors(self): """ This method logs a detailed error on a failed pqos_error command. """ cap = get_capability(self.executable_path) if cap["error"] is False: if self.CAP_SYS_RAWIO in cap["capabilities"]: if not all(x in cap["set"] for x in ["e", "p"]): logging.warning( "Insufficient capabilities for pqos_wrapper, Please add e,p in cap_sys_rawio capability set of pqos_wrapper" ) else: logging.warning( "Insufficient capabilities for pqos_wrapper, Please set capabilitiy cap_sys_rawio with e,p for pqos_wrapper" ) msr = check_msr() if msr["loaded"]: current_user = grp.getgrgid(os.getegid()).gr_name if msr["read"]: if not msr["write"]: logging.warning( "Add write permissions for msr module for {}".format( current_user)) else: logging.warning( "Add read and write permissions for msr module for {}". format(current_user)) else: logging.warning( "Load msr module for using cache allocation/monitoring")
class ORIGAMIMS(object): def __init__(self, *args, **kwargs): self.__wx_app = wx.App(redirect=False) self.run = None self.view = None self.init(*args, **kwargs) def start(self): self.view.Show() self.__wx_app.MainLoop() def quit(self): self.__wx_app.ProcessIdle() self.__wx_app.ExitMainLoop() self.view.Destroy() return True def endSession(self): wx.CallAfter(self.quit, force=True) def init(self, *args, **kwargs): self.config = config.config() self.icons = config.IconContainer() self.view = mainWindow.MyFrame(self, config=self.config, icons=self.icons, title="ORIGAMI-MS") self.wrensCMD = None self.wrensRun = None self.wrensReset = None self.currentPath = "" self.wrensInput = { 'polarity': None, 'activationZone': None, 'method': None, 'command': None } self.logging = True self.config.startTime = (strftime("%H-%M-%S_%d-%m-%Y", gmtime())) self.__wx_app.SetTopWindow(self.view) self.__wx_app.SetAppName("ORIGAMI-MS") self.__wx_app.SetVendorName("Lukasz G Migas, University of Manchester") # Log all events to if self.logging == True: sys.stdin = self.view.panelPlots.log sys.stdout = self.view.panelPlots.log sys.stderr = self.view.panelPlots.log self.importConfigFileOnStart(evt=None) #=============================================================================== # ACQUISITION FUNCTIONS #=============================================================================== def onLibraryLink(self, evt): """Open selected webpage.""" # set link links = { ID_helpHomepage: 'home', ID_helpGitHub: 'github', ID_helpCite: 'cite', ID_helpNewVersion: 'newVersion' } link = self.config.links[links[evt.GetId()]] # open webpage try: webbrowser.open(link, autoraise=1) except: pass def onCheckParameters(self, evt): """ This function checks that all variables are in correct format """ print("version", self.config.version) if isinstance(self.config.iSPV, (int, long)): pass else: msg = ("SPV value should be an integer!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if isinstance(self.config.iScanTime, (int, float)): pass else: msg = ("Scan time value should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if isinstance(self.config.iStartVoltage, (int, float)): pass else: msg = ("Start voltage should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if isinstance(self.config.iEndVoltage, (int, float)): pass else: msg = ("End voltage should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if isinstance(self.config.iStepVoltage, (int, float)): pass else: msg = ("Step voltage should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if self.config.iActivationMode == "Exponential": if isinstance(self.config.iExponentPerct, (int, float)): pass else: msg = ("Exponential % value should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if isinstance(self.config.iExponentIncre, (int, float)): pass else: msg = ("Exponential increment value should be an float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False elif self.config.iActivationMode == "Boltzmann": if isinstance(self.config.iBoltzmann, (int, float)): pass else: msg = ("Boltzmann offset value should be an integer or float!") dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return False if (abs(self.config.iEndVoltage) <= abs(self.config.iStartVoltage)): msg = ('End voltage has to be larger than starting voltage') dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if (abs(self.config.iEndVoltage) > 200): msg = ( 'The highest possible voltage is 200 V. Set to default: 200') dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iEndVoltage = 200 self.view.panelControls.endVoltage_input.SetValue( str(self.config.iEndVoltage)) if (abs(self.config.iStartVoltage) < 0): msg = ('The lowest possible voltage is 0 V. Set to default: 0') dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iStartVoltage = 0 self.view.panelControls.startVoltage_input.SetValue( str(self.config.iStartVoltage)) if self.config.iSPV <= 0: msg = ('SPV must be larger than 0! Set to default: 3') dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iSPV = 3 self.view.panelControls.spv_input.SetValue(str(self.config.iSPV)) if self.config.iScanTime <= 0: msg = ('Scan time must be larger than 0! Set to default: 5') dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iScanTime = 5 self.view.panelControls.scanTime_input.SetValue( str(self.config.iScanTime)) if self.config.iActivationMode == "Exponential": if self.config.iExponentPerct < 0: msg = ( 'Exponential % must be larger or equal to 0! Set to default: 0' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iExponentPerct = 0 elif self.config.iExponentPerct >= 100: msg = ( 'Exponential % must be smaller than 100! Set to default: 0' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iExponentPerct = 0 self.view.panelControls.exponentialPerct_input.SetValue( str(self.config.iExponentPerct)) if self.config.iExponentIncre <= 0: msg = ( 'Exponential increment must be larger than 0! Set to default: 0.01' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iExponentIncre = 0.01 elif self.config.iExponentIncre > 0.075: msg = ( 'Exponential increment must be smaller than 0.075! Set to default: 0.075' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iExponentIncre = 0.075 self.view.panelControls.exponentialIncrm_input.SetValue( str(self.config.iExponentIncre)) elif self.config.iActivationMode == "Boltzmann": if self.config.iBoltzmann < 10: msg = ( 'Boltzmann offset must be larger than 10! Set to default: 10' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iBoltzmann = 10 elif self.config.iBoltzmann >= 100: msg = ( 'Boltzmann offset must be smaller than 100! Set to default: 25' ) dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") self.config.iBoltzmann = 25 self.view.panelControls.boltzmann_input.SetValue( str(self.config.iBoltzmann)) # All good return True def onCalculateParameters(self, evt): """ This function is to be used to setup path to save origami parameters """ if not self.config.iActivationMode == "User-defined": if self.onCheckParameters(evt=None) == False: print('Please fill in all necessary fields first!') return divisibleCheck = abs( self.config.iEndVoltage - self.config.iStartVoltage) / self.config.iStepVoltage divisibleCheck2 = divisibleCheck % 1 if divisibleCheck2 != 0: msg = "Are you sure your collision voltage range is divisible by your increment?" dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return else: if self.config.iScanTime == None or self.config.iScanTime == "": msg = 'Please make sure you to fill in the scan time input box.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if self.config.iActivationMode == "Linear": self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareLinearMethod( ) elif self.config.iActivationMode == "Exponential": self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareExponentialMethod( ) elif self.config.iActivationMode == "Boltzmann": self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareBoltzmannMethod( ) elif self.config.iActivationMode == "User-defined": self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareListMethod( ) self.wrensCMD = self.wrensInput.get('command', None) # Setup status: self.view.SetStatusText(''.join( ['Acq. time: ', str(totalAcqTime), ' mins']), number=0) self.view.SetStatusText(''.join( [str(len(scanPerVoltageList)), ' steps']), number=1) # Add wrensCMD to config file self.config.wrensCMD = self.wrensCMD self.onPlotSPV(ColEnergyX, scanPerVoltageList) self.onPlotTime(ColEnergyX, timeList) print(''.join(["Your submission code: ", self.wrensCMD])) def onPrepareLinearMethod(self): startScansPerVoltage = self.config.iSPV scanPerVoltageList, timeList = [], [] timeFit = 3 numberOfCEsteps = (self.config.iEndVoltage - self.config.iStartVoltage ) / self.config.iStepVoltage + 1 ColEnergyX = np.linspace(self.config.iStartVoltage, self.config.iEndVoltage, numberOfCEsteps) for i in range(int(numberOfCEsteps)): scanPerVoltageFit = startScansPerVoltage scanPerVoltageList.append(scanPerVoltageFit) timeFit = timeFit + scanPerVoltageFit timeList.append(timeFit * self.config.iScanTime) numberOfCEScans = (numberOfCEsteps * self.config.iSPV) timeOfCEScans = (numberOfCEScans + self.config.iSPV) * self.config.iScanTime totalAcqTime = round( float((6 + numberOfCEScans + self.config.iSPV) * self.config.iScanTime) / 60, 2) if (totalAcqTime > 300): msg = "The acquisition will take more than 5 hours. Consider reducing " \ "your collision voltage range or adjusting the parameters!" dialogs.dlgBox(exceptionTitle='Very long acquisition warning', exceptionMsg=msg, type="Warning") wrensCMD = ''.join([ self.config.wrensLinearPath, self.config.iActivationZone, ',', self.config.iPolarity, ',', str(self.config.iSPV), ',', str(self.config.iScanTime), ',', str(self.config.iStartVoltage), ',', str(self.config.iEndVoltage), ',', str(self.config.iStepVoltage), ',', str(totalAcqTime) ]) wrensInput = { 'polarity': self.config.iPolarity, 'activationZone': self.config.iActivationZone, 'method': self.config.iActivationMode, 'command': wrensCMD } if self.config.iPolarity == 'POSITIVE': polarity = "+VE" else: polarity = "-VE" self.view.SetStatusText(''.join([ "Current method: ", polarity, " mode in the ", self.config.iActivationZone, " using the ", self.config.iActivationMode, " method" ]), number=2) return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime def onPrepareExponentialMethod(self): startScansPerVoltage = self.config.iSPV scanPerVoltageList, timeList = [], [] timeFit = 3 expAccumulator = 0 numberOfCEsteps = (self.config.iEndVoltage - self.config.iStartVoltage ) / self.config.iStepVoltage + 1 ColEnergyX = np.linspace(self.config.iStartVoltage, self.config.iEndVoltage, numberOfCEsteps) for i in xrange(int(numberOfCEsteps)): if abs(ColEnergyX[i]) >= abs(self.config.iEndVoltage * self.config.iExponentPerct / 100): expAccumulator = expAccumulator + self.config.iExponentIncre scanPerVoltageFit = np.round( startScansPerVoltage * np.exp(expAccumulator), 0) else: scanPerVoltageFit = startScansPerVoltage scanPerVoltageList.append(scanPerVoltageFit) timeFit = timeFit + scanPerVoltageFit timeList.append(timeFit * self.config.iScanTime) numberOfCEScans = sum(scanPerVoltageList) timeOfCEScans = (numberOfCEScans + scanPerVoltageList[-1]) * self.config.iScanTime totalAcqTime = round( float((6 + numberOfCEScans + self.config.iSPV) * self.config.iScanTime) / 60, 2) if (totalAcqTime > 300): msg = "The acquisition will take more than 5 hours. Consider reducing " \ "your collision voltage range or adjusting the parameters!" dialogs.dlgBox(exceptionTitle='Very long acquisition warning', exceptionMsg=msg, type="Warning") wrensCMD = ''.join([ self.config.wrensExponentPath, self.config.iActivationZone, ',', self.config.iPolarity, ',', str(self.config.iSPV), ',', str(self.config.iScanTime), ',', str(self.config.iStartVoltage), ',', str(self.config.iEndVoltage), ',', str(self.config.iStepVoltage), ',', str(self.config.iExponentPerct), ',', str(self.config.iExponentIncre), ',', str(totalAcqTime) ]) wrensInput = { 'polarity': self.config.iPolarity, 'activationZone': self.config.iActivationZone, 'method': self.config.iActivationMode, 'command': wrensCMD } if self.config.iPolarity == 'POSITIVE': polarity = "+VE" else: polarity = "-VE" self.view.SetStatusText(''.join([ "Current method: ", polarity, " mode in the ", self.config.iActivationZone, " using the ", self.config.iActivationMode, " method" ]), number=2) return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime def onPrepareBoltzmannMethod(self): startScansPerVoltage = self.config.iSPV scanPerVoltageList, timeList = [], [] timeFit = 3 A1 = 2 A2 = 0.07 x0 = 47 numberOfCEsteps = int( (self.config.iEndVoltage - self.config.iStartVoltage) / self.config.iStepVoltage + 1) ColEnergyX = np.linspace(self.config.iStartVoltage, self.config.iEndVoltage, numberOfCEsteps) for i in range(int(numberOfCEsteps)): scanPerVoltageFit = np.round( 1 / (A2 + (A1 - A2) / (1 + np.exp( (ColEnergyX[i] - x0) / self.config.iBoltzmann))), 0) scanPerVoltageList.append(scanPerVoltageFit * startScansPerVoltage) timeFit = timeFit + scanPerVoltageFit timeList.append(timeFit * self.config.iScanTime) numberOfCEScans = sum(scanPerVoltageList) timeOfCEScans = (numberOfCEScans + scanPerVoltageList[-1]) * self.config.iScanTime totalAcqTime = round( float((6 + numberOfCEScans + self.config.iSPV) * self.config.iScanTime) / 60, 2) if (totalAcqTime > 300): msg = "The acquisition will take more than 5 hours. Consider reducing " \ "your collision voltage range or adjusting the parameters!" dialogs.dlgBox(exceptionTitle='Very long acquisition warning', exceptionMsg=msg, type="Warning") wrensCMD = ''.join([ self.config.wrensBoltzmannPath, self.config.iActivationZone, ',', self.config.iPolarity, ',', str(self.config.iSPV), ',', str(self.config.iScanTime), ',', str(self.config.iStartVoltage), ',', str(self.config.iEndVoltage), ',', str(self.config.iStepVoltage), ',', str(self.config.iBoltzmann), ',', str(totalAcqTime) ]) wrensInput = { 'polarity': self.config.iPolarity, 'activationZone': self.config.iActivationZone, 'method': self.config.iActivationMode, 'command': wrensCMD } if self.config.iPolarity == 'POSITIVE': polarity = "+VE" else: polarity = "-VE" self.view.SetStatusText(''.join([ "Current method: ", polarity, " mode in the ", self.config.iActivationZone, " using the ", self.config.iActivationMode, " method" ]), number=2) return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime def onPrepareListMethod(self): if self.config.CSVFilePath == None: msg = 'Please load a CSV file first.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if self.config.iScanTime == None or self.config.iScanTime == "": msg = 'Please fill in appropriate fields. The scan time is empty or incorrect' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return print(self.config.CSVFilePath) try: spvCVlist = np.genfromtxt(self.config.CSVFilePath, skip_header=1, delimiter=',', filling_values=0) except: return # Read table scanPerVoltageList = spvCVlist[:, 0].astype(int) ColEnergyX = spvCVlist[:, 1] timeFit = 3 timeList = [] for i in scanPerVoltageList: timeFit = timeFit + i timeList.append(timeFit * self.config.iScanTime) if len(scanPerVoltageList) != len(ColEnergyX): return totalAcqTime = np.round( (sum(scanPerVoltageList) * self.config.iScanTime) / 60, 2) if (totalAcqTime > 300): msg = "The acquisition will take more than 5 hours. Consider reducing " \ "your collision voltage range or adjusting the parameters!" dialogs.dlgBox(exceptionTitle='Very long acquisition warning', exceptionMsg=msg, type="Warning") SPV_list = ' '.join(str(spv) for spv in scanPerVoltageList.tolist()) SPV_list = ''.join(["[", SPV_list, "]"]) CV_list = ' '.join(str(cv) for cv in ColEnergyX.tolist()) CV_list = ''.join(["[", CV_list, "]"]) self.config.SPVsList = scanPerVoltageList self.config.CVsList = ColEnergyX print(SPV_list, CV_list) wrensCMD = ''.join([ self.config.wrensUserDefinedPath, self.config.iActivationZone, ',', self.config.iPolarity, ',', str(self.config.iScanTime), ',', SPV_list, ',', CV_list ]) wrensInput = { 'polarity': self.config.iPolarity, 'activationZone': self.config.iActivationZone, 'method': self.config.iActivationMode, 'command': wrensCMD } if self.config.iPolarity == 'POSITIVE': polarity = "+VE" else: polarity = "-VE" self.view.SetStatusText(''.join([ "Current method: ", polarity, " mode in the ", self.config.iActivationZone, " using the ", self.config.iActivationMode, " method" ]), number=2) return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime def onStartWREnSRun(self, evt): if self.wrensCMD == None: msg = "Are you sure you filled in correct details or pressed calculate?" dialogs.dlgBox( exceptionTitle= 'Please complete all necessary fields and press Calculate', exceptionMsg=msg, type="Error") return # A couple of checks to ensure the method in the settings is the one # currently available in memory.. if self.wrensInput.get("polarity", None) != self.config.iPolarity: msg = 'The polarity of the current method and the one in the window do not agree. Consider re-calculating.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if self.wrensInput.get("activationZone", None) != self.config.iActivationZone: msg = 'The activation zone of the current method and the one in the window do not agree. Consider re-calculating.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if self.wrensInput.get("method", None) != self.config.iActivationMode: msg = 'The acquisition mode of the current method and the one in the window do not agree. Consider re-calculating.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return if self.wrensInput.get("command", None) != self.wrensCMD: msg = 'The command in the memory and the current method and the one in the window do not agree. Consider re-calculating.' dialogs.dlgBox(exceptionTitle='Mistake in the input', exceptionMsg=msg, type="Error") return print(''.join(["Your code: ", self.wrensCMD])) self.wrensRun = Popen(self.wrensCMD) def onStopWREnSRun(self, evt): if self.wrensRun: print('Stopped acquisition and reset the property banks') self.wrensRun.kill() self.wrensReset = Popen(self.config.wrensResetPath) self.view.panelControls.goBtn.Enable() else: print('You have to start acquisition first!') def onFillInDefaults(self, evt=None): """ This function fills in default values in case you are being lazy! """ pass def onGetMassLynxPath(self, evt): """ Select path to the MassLynx folder """ dlg = wx.DirDialog(self.view, "Select MassLynx directory", style=wx.DD_DEFAULT_STYLE) if dlg.ShowModal() == wx.ID_OK: print "You chose %s" % dlg.GetPath() self.view.panelControls.path_value.SetValue(dlg.GetPath()) self.currentPath = dlg.GetPath() def importConfigFile(self, evt): """ This function imports configuration file """ dlg = wx.FileDialog(self.view, "Open Configuration File", wildcard="*.ini", style=wx.FD_DEFAULT_STYLE | wx.FD_CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: fileName = dlg.GetPath() self.config.importConfig(fileName=fileName, e=None) print(''.join(["WREnS runner path: ", self.config.wrensRunnerPath])) print(''.join(["Linear path: ", self.config.wrensLinearPath])) print(''.join(["Exponent path: ", self.config.wrensExponentPath])) print(''.join(["Boltzmann path: ", self.config.wrensBoltzmannPath])) print(''.join(["List path: ", self.config.wrensUserDefinedPath])) print(''.join(["Reset path: ", self.config.wrensResetPath])) def importConfigFileOnStart(self, evt): print("Importing origamiConfig.ini") self.config.importConfig(fileName="origamiConfig.ini", e=None) print(''.join(["WREnS runner path: ", self.config.wrensRunnerPath])) print(''.join(["Linear path: ", self.config.wrensLinearPath])) print(''.join(["Exponent path: ", self.config.wrensExponentPath])) print(''.join(["Boltzmann path: ", self.config.wrensBoltzmannPath])) print(''.join(["List path: ", self.config.wrensUserDefinedPath])) print(''.join(["Reset path: ", self.config.wrensResetPath])) def onExportConfig(self, evt): """ This function exports configuration file """ dlg = wx.FileDialog(self.view, "Save As Configuration File", wildcard="*.ini", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) if dlg.ShowModal() == wx.ID_OK: fileName = dlg.GetPath() self.config.exportConfig(fileName=fileName, e=None) def onExportParameters(self, evt): dlg = wx.FileDialog(self.view, "Save As ORIGAMI configuration File", wildcard="*.conf", style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT) dlg.SetFilename('origami.conf') if dlg.ShowModal() == wx.ID_OK: fileName = dlg.GetPath() self.config.exportOrigamiConfig(fileName=fileName, e=None) def onLoadCSVList(self, evt): """ This function loads a two column list with Collision voltage | number of scans """ dlg = wx.FileDialog(self.view, "Choose a file:", wildcard="*.txt; *.csv", style=wx.FD_DEFAULT_STYLE | wx.FD_CHANGE_DIR) if dlg.ShowModal() == wx.ID_OK: print "You chose %s" % dlg.GetPath() self.config.CSVFilePath = dlg.GetPath() def onPlotSPV(self, xvals, yvals): self.view.panelPlots.plot1.clearPlot() self.view.panelPlots.plot1.plot1Ddata(xvals=xvals, yvals=yvals, title="", xlabel="Collision Voltage (V)", ylabel="SPV") # Show the mass spectrum self.view.panelPlots.plot1.repaint() def onPlotTime(self, xvals, yvals): self.view.panelPlots.plot2.clearPlot() self.view.panelPlots.plot2.plot1Ddata(xvals=xvals, yvals=yvals, title="", xlabel="Collision Voltage (V)", ylabel="Accumulated Time (s)") # Show the mass spectrum self.view.panelPlots.plot2.repaint()
#!/usr/bin/env python3 from subprocess import Popen from socket import * tcp = Popen(['./cqotdd-tcp', 'This is a test purely intended to test.']) udp = Popen(['./cqotdd-udp', 'This is a test purely intended to test.']) sock = socket(AF_INET, SOCK_STREAM) sock.connect(('127.0.0.1', 17)) print(sock.recv(512).decode('UTF-8'), end='') sock.close() sock = socket(AF_INET, SOCK_DGRAM) sock.sendto(b'\0' * 22, ('127.0.0.1', 17)) print(sock.recv(512).decode('UTF-8'), end='') sock.close() tcp.kill() udp.kill()
def test_reload_wrapper_preservation(self): # This test verifies that when `python -m tornado.autoreload` # is used on an application that also has an internal # autoreload, the reload wrapper is preserved on restart. main = """\ import os import sys # This import will fail if path is not set up correctly import testapp if 'tornado.autoreload' not in sys.modules: raise Exception('started without autoreload wrapper') import tornado.autoreload print('Starting') sys.stdout.flush() if 'TESTAPP_STARTED' not in os.environ: os.environ['TESTAPP_STARTED'] = '1' # Simulate an internal autoreload (one not caused # by the wrapper). tornado.autoreload._reload() else: # Exit directly so autoreload doesn't catch it. os._exit(0) """ # Create temporary test application os.mkdir(os.path.join(self.path, 'testapp')) init_file = os.path.join(self.path, 'testapp', '__init__.py') open(init_file, 'w').close() main_file = os.path.join(self.path, 'testapp', '__main__.py') with open(main_file, 'w') as f: f.write(main) # Make sure the tornado module under test is available to the test # application pythonpath = os.getcwd() if 'PYTHONPATH' in os.environ: pythonpath += os.pathsep + os.environ['PYTHONPATH'] autoreload_proc = Popen( [sys.executable, '-m', 'tornado.autoreload', '-m', 'testapp'], stdout=subprocess.PIPE, cwd=self.path, env=dict(os.environ, PYTHONPATH=pythonpath), universal_newlines=True) # This timeout needs to be fairly generous for pypy due to jit # warmup costs. for i in range(40): if autoreload_proc.poll() is not None: break time.sleep(0.1) else: autoreload_proc.kill() raise Exception("subprocess failed to terminate") out = autoreload_proc.communicate()[0] self.assertEqual(out, 'Starting\n' * 2)
class LocalhostRun: def __init__(self, port, id=None, interval=30, retries=30): import os filePath = "/usr/local/sessionSettings/localhostDB.json" if not os.path.exists(filePath): os.makedirs(filePath[:-16], exist_ok=True) open(filePath, 'w').close() installAutoSSH() if not id: id = str(uuid.uuid4())[:8] self.connection = None self.id = id self.port = port self.interval = interval self.retries = retries def start(self): if self.connection: self.connection.kill() self.connection = Popen( f"ssh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no" .split(), stdout=PIPE, stdin=PIPE) try: return re.findall( "http://(.*?.localhost.run)", self.connection.stdout.readline().decode("utf-8"))[0] except: raise Exception(self.connection.stdout.readline().decode("utf-8")) def keep_alive(self): # if self.connection:self.connection.kill() import urllib try: localhostOpenDB = dict( accessSettingFile("localhostDB.json", v=False)) except TypeError: localhostOpenDB = dict() if findProcess("autossh", f"80:localhost:{self.port}"): try: oldAddr = localhostOpenDB[str(self.port)] urllib.request.urlopen("http://" + oldAddr) return oldAddr except: pass for _ in range(2): self.connection = Popen( f"autossh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no -o ServerAliveInterval={self.interval} -o ServerAliveCountMax={self.retries}" .split(), stdout=PIPE, stdin=PIPE, stderr=PIPE) #print("ssh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no -o ServerAliveInterval={self.interval} -o ServerAliveCountMax={self.retries}") try: newAddr = re.findall( "http://(.*?.localhost.run)", self.connection.stdout.readline().decode("utf-8"))[0] localhostOpenDB[str(self.port)] = newAddr accessSettingFile("localhostDB.json", localhostOpenDB, v=False) return newAddr except: outs, errs = self.connection.communicate(timeout=15) self.connection.kill() # print(outs) # print(errs) if re.search(r"Permission\sdenied\s\(publickey\)", errs.decode("utf-8")): os.system("ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa") continue raise Exception(errs.decode("utf-8")) break def kill(self): self.connection.kill()
def run_attach(cmd, output_cb=None): """ run command and attach output to cb Arguments: cmd: shell command output_cb: where to display output """ stdoutmaster, stdoutslave = pty.openpty() subproc = Popen(cmd, shell=True, stdout=stdoutslave, stderr=PIPE) os.close(stdoutslave) decoder = codecs.getincrementaldecoder('utf-8')() def last_ten_lines(s): chunk = s[-1500:] lines = chunk.splitlines(True) return ''.join(lines[-10:]).replace('\r', '') decoded_output = "" try: while subproc.poll() is None: try: b = os.read(stdoutmaster, 512) except OSError as e: if e.errno != errno.EIO: raise break else: final = False if not b: final = True decoded_chars = decoder.decode(b, final) if decoded_chars is None: continue decoded_output += decoded_chars if output_cb: ls = last_ten_lines(decoded_output) output_cb(ls) if final: break finally: os.close(stdoutmaster) if subproc.poll() is None: subproc.kill() subproc.wait() errors = [l.decode('utf-8') for l in subproc.stderr.readlines()] if output_cb: output_cb(last_ten_lines(decoded_output)) errors = ''.join(errors) if subproc.returncode == 0: return decoded_output.strip() else: raise Exception("Problem running {0} " "{1}:{2}".format(cmd, subproc.returncode))
DEPENDENCIES = ('requests', 'bs4', 'cryptography', 'git', 'psutil') assert all(map(find_spec, DEPENDENCIES)),\ 'Missing dependencies are required to continue. Check to make sure you have {} installed.'.format(', '.join(DEPENDENCIES[:-1]).title()+', and '+DEPENDENCIES[-1].title()) print('Dependency check passed.') HERE = os.path.dirname(os.path.realpath(__file__)) os.chdir(HERE) for folder in ('logs', 'data', 'config', 'web/assets'): try: os.mkdir(folder) except FileExistsError: pass assert os.path.exists('.env'), 'Toplevel env file not found.' while True: process = Popen([sys.executable, '_main.py'], stdin=sys.stdin, stdout=sys.stdout, stderr=sys.stderr) process.wait() if process.returncode == 37: process.kill() print('Rebooting.') continue elif process.returncode == 0: print('Shut down gracefully.') process.kill() break
class W3MImageDisplayer(ImageDisplayer, FileManagerAware): """Implementation of ImageDisplayer using w3mimgdisplay, an utilitary program from w3m (a text-based web browser). w3mimgdisplay can display images either in virtual tty (using linux framebuffer) or in a Xorg session. Does not work over ssh. w3m need to be installed for this to work. """ is_initialized = False def __init__(self): self.binary_path = None self.process = None def initialize(self): """start w3mimgdisplay""" self.binary_path = None self.binary_path = self._find_w3mimgdisplay_executable() # may crash self.process = Popen([self.binary_path] + W3MIMGDISPLAY_OPTIONS, stdin=PIPE, stdout=PIPE, universal_newlines=True) self.is_initialized = True @staticmethod def _find_w3mimgdisplay_executable(): paths = [os.environ.get(W3MIMGDISPLAY_ENV, None)] + W3MIMGDISPLAY_PATHS for path in paths: if path is not None and os.path.exists(path): return path raise ImageDisplayError( "No w3mimgdisplay executable found. Please set " "the path manually by setting the %s environment variable. (see " "man page)" % W3MIMGDISPLAY_ENV) def _get_font_dimensions(self): # Get the height and width of a character displayed in the terminal in # pixels. if self.binary_path is None: self.binary_path = self._find_w3mimgdisplay_executable() farg = struct.pack("HHHH", 0, 0, 0, 0) fd_stdout = sys.stdout.fileno() fretint = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, farg) rows, cols, xpixels, ypixels = struct.unpack("HHHH", fretint) if xpixels == 0 and ypixels == 0: process = Popen([self.binary_path, "-test"], stdout=PIPE, universal_newlines=True) output, _ = process.communicate() output = output.split() xpixels, ypixels = int(output[0]), int(output[1]) # adjust for misplacement xpixels += 2 ypixels += 2 return (xpixels // cols), (ypixels // rows) def draw(self, path, start_x, start_y, width, height): if not self.is_initialized or self.process.poll() is not None: self.initialize() try: input_gen = self._generate_w3m_input(path, start_x, start_y, width, height) except ImageDisplayError: raise # Mitigate the issue with the horizontal black bars when # selecting some images on some systems. 2 milliseconds seems # enough. Adjust as necessary. if self.fm.settings.w3m_delay > 0: from time import sleep sleep(self.fm.settings.w3m_delay) self.process.stdin.write(input_gen) self.process.stdin.flush() self.process.stdout.readline() self.quit() self.is_initialized = False def clear(self, start_x, start_y, width, height): if not self.is_initialized or self.process.poll() is not None: self.initialize() fontw, fonth = self._get_font_dimensions() cmd = "6;{x};{y};{w};{h}\n4;\n3;\n".format( x=int((start_x - 0.2) * fontw), y=start_y * fonth, # y = int((start_y + 1) * fonth), # (for tmux top status bar) w=int((width + 0.4) * fontw), h=height * fonth + 1, # h = (height - 1) * fonth + 1, # (for tmux top status bar) ) try: self.process.stdin.write(cmd) except IOError as ex: if ex.errno == errno.EPIPE: return raise self.process.stdin.flush() self.process.stdout.readline() def _generate_w3m_input(self, path, start_x, start_y, max_width, max_height): """Prepare the input string for w3mimgpreview start_x, start_y, max_height and max_width specify the drawing area. They are expressed in number of characters. """ fontw, fonth = self._get_font_dimensions() if fontw == 0 or fonth == 0: raise ImgDisplayUnsupportedException max_width_pixels = max_width * fontw max_height_pixels = max_height * fonth - 2 # (for tmux top status bar) # max_height_pixels = (max_height - 1) * fonth - 2 # get image size cmd = "5;{}\n".format(path) self.process.stdin.write(cmd) self.process.stdin.flush() output = self.process.stdout.readline().split() if len(output) != 2: raise ImageDisplayError('Failed to execute w3mimgdisplay', output) width = int(output[0]) height = int(output[1]) # get the maximum image size preserving ratio if width > max_width_pixels: height = (height * max_width_pixels) // width width = max_width_pixels if height > max_height_pixels: width = (width * max_height_pixels) // height height = max_height_pixels return "0;1;{x};{y};{w};{h};;;;;{filename}\n4;\n3;\n".format( x=int((start_x - 0.2) * fontw), y=start_y * fonth, # y = (start_y + 1) * fonth, # (for tmux top status bar) w=width, h=height, filename=path, ) def quit(self): if self.is_initialized and self.process and self.process.poll( ) is None: self.process.kill()
def remote_access_service( local_host="127.0.0.1", local_port=22267, server="app.pywebio.online", server_port=1022, remote_port="/", setup_timeout=60, ): """ Wait at most one minute to get the ssh output, if it gets a normal out, the connection is successfully established. Otherwise report error and kill ssh process. :param local_port: ssh local listen port :param server: ssh server domain :param server_port: ssh server port :param setup_timeout: If the service can't setup successfully in `setup_timeout` seconds, then exit. """ global _ssh_process, _ssh_notfound bin = State.deploy_config.SSHExecutable cmd = f"{bin} -oStrictHostKeyChecking=no -R {remote_port}:{local_host}:{local_port} -p {server_port} {server} -- --output json" args = shlex.split(cmd) logger.debug(f"remote access service command: {cmd}") if _ssh_process is not None and _ssh_process.poll() is None: logger.warning(f"Kill previous ssh process [{_ssh_process.pid}]") _ssh_process.kill() try: _ssh_process = Popen(args, stdout=PIPE, stderr=PIPE) except FileNotFoundError as e: logger.critical( f"Cannot find SSH executable {bin}, please install OpenSSH or specify SSHExecutable in deploy.yaml" ) _ssh_notfound = True return logger.info(f"remote access process pid: {_ssh_process.pid}") success = False def timeout_killer(wait_sec): time.sleep(wait_sec) if not success and _ssh_process.poll() is None: logger.info("Connection timeout, kill ssh process") _ssh_process.kill() threading.Thread(target=timeout_killer, kwargs=dict(wait_sec=setup_timeout), daemon=True).start() stdout = _ssh_process.stdout.readline().decode("utf8") logger.info(f"ssh server stdout: {stdout}") connection_info = {} try: connection_info = json.loads(stdout) success = True except json.decoder.JSONDecodeError: if not success and _ssh_process.poll() is None: _ssh_process.kill() if success: if connection_info.get("status", "fail") != "success": logger.info( f"Failed to establish remote access, this is the error message from service provider: {connection_info.get('message', '')}" ) new_username = connection_info.get("change_username", None) if new_username: logger.info( f"Server requested to change username, change it to: {new_username}" ) State.deploy_config.SSHUser = new_username else: global address address = connection_info["address"] logger.debug(f"Remote access url: {address}") # wait ssh or main thread exit while not am_i_the_only_thread() and _ssh_process.poll() is None: # while _ssh_process.poll() is None: time.sleep(1) if _ssh_process.poll() is None: # main thread exit, kill ssh process logger.info("App process exit, killing ssh process") _ssh_process.kill() else: # ssh process exit by itself or by timeout killer stderr = _ssh_process.stderr.read().decode("utf8") if stderr: logger.error( f"PyWebIO application remote access service error: {stderr}") else: logger.info("PyWebIO application remote access service exit.") address = None
class Py3: """ Helper object that gets injected as ``self.py3`` into Py3status modules that have not got that attribute set already. This allows functionality like: - User notifications - Forcing module to update (even other modules) - Triggering events for modules Py3 is also used for testing in which case it does not get a module when being created. All methods should work in this situation. """ CACHE_FOREVER = PY3_CACHE_FOREVER """ Special constant that when returned for ``cached_until`` will cause the module to not update unless externally triggered. """ LOG_ERROR = PY3_LOG_ERROR """Show as Error""" LOG_INFO = PY3_LOG_INFO """Show as Informational""" LOG_WARNING = PY3_LOG_WARNING """Show as Warning""" # Shared by all Py3 Instances _formatter = None _gradients = Gradients() _none_color = NoneColor() _storage = Storage() # Exceptions Py3Exception = exceptions.Py3Exception CommandError = exceptions.CommandError RequestException = exceptions.RequestException RequestInvalidJSON = exceptions.RequestInvalidJSON RequestTimeout = exceptions.RequestTimeout RequestURLError = exceptions.RequestURLError def __init__(self, module=None): self._audio = None self._config_setting = {} self._english_env = dict(os.environ) self._english_env["LC_ALL"] = "C" self._english_env["LANGUAGE"] = "C" self._format_placeholders = {} self._format_placeholders_cache = {} self._is_python_2 = sys.version_info < (3, 0) self._module = module self._report_exception_cache = set() self._thresholds = None self._threshold_gradients = {} self._uid = uuid4() if module: self._i3s_config = module._py3_wrapper.config["py3_config"][ "general"] self._module_full_name = module.module_full_name self._output_modules = module._py3_wrapper.output_modules self._py3status_module = module.module_class self._py3_wrapper = module._py3_wrapper # create formatter we only if need one but want to pass py3_wrapper so # that we can do logging etc. if not self._formatter: self.__class__._formatter = Formatter(module._py3_wrapper) def __getattr__(self, name): """ Py3 can provide COLOR constants eg COLOR_GOOD, COLOR_BAD, COLOR_DEGRADED but also any constant COLOR_XXX we find this color in the config if it exists """ if not name.startswith("COLOR_"): raise AttributeError("Attribute `%s` not in Py3" % name) return self._get_config_setting(name.lower()) def _get_config_setting(self, name, default=None): try: return self._config_setting[name] except KeyError: fn = self._py3_wrapper.get_config_attribute param = fn(self._module_full_name, name) # colors are special we want to make sure that we treat a color # that was explicitly set to None as a True value. Ones that are # not set should be treated as None if name.startswith("color_"): if hasattr(param, "none_setting"): # see if named color and use if it is param = COLOR_NAMES.get(name[6:].lower()) elif param is None: param = self._none_color # if a non-color parameter and was not set then set to default elif hasattr(param, "none_setting"): param = default self._config_setting[name] = param return self._config_setting[name] def _get_color(self, color): if not color: return # fix any hex colors so they are #RRGGBB if color.startswith("#"): color = color.upper() if len(color) == 4: color = ("#" + color[1] + color[1] + color[2] + color[2] + color[3] + color[3]) return color name = "color_%s" % color return self._get_config_setting(name) def _thresholds_init(self): """ Initiate and check any thresholds set """ thresholds = getattr(self._py3status_module, "thresholds", []) self._thresholds = {} if isinstance(thresholds, list): try: thresholds.sort() except TypeError: pass self._thresholds[None] = [(x[0], self._get_color(x[1])) for x in thresholds] return elif isinstance(thresholds, dict): for key, value in thresholds.items(): if isinstance(value, list): try: value.sort() except TypeError: pass self._thresholds[key] = [(x[0], self._get_color(x[1])) for x in value] def _get_module_info(self, module_name): """ THIS IS PRIVATE AND UNSUPPORTED. Get info for named module. Info comes back as a dict containing. 'module': the instance of the module, 'position': list of places in i3bar, usually only one item 'type': module type py3status/i3status """ return self._output_modules.get(module_name) def _report_exception(self, msg, frame_skip=2): """ THIS IS PRIVATE AND UNSUPPORTED. logs an exception that occurs inside of a Py3 method. We only log the exception once to prevent spamming the logs and we do not notify the user. frame_skip is used to change the place in the code that the error is reported as coming from. We want to show it as coming from the py3status module where the Py3 method was called. """ # We use a hash to see if the message is being repeated. msg_hash = hash(msg) if msg_hash in self._report_exception_cache: return self._report_exception_cache.add(msg_hash) # If we just report the error the traceback will end in the try # except block that we are calling from. # We want to show the traceback originating from the module that # called the Py3 method so get the correct error frame and pass this # along. error_frame = sys._getframe(0) while frame_skip: error_frame = error_frame.f_back frame_skip -= 1 self._py3_wrapper.report_exception(msg, notify_user=False, error_frame=error_frame) def error(self, msg, timeout=None): """ Raise an error for the module. :param msg: message to be displayed explaining the error :param timeout: how long before we should retry. For permanent errors `py3.CACHE_FOREVER` should be returned. If not supplied then the modules `cache_timeout` will be used. """ raise ModuleErrorException(msg, timeout) def flatten_dict(self, d, delimiter="-", intermediates=False, parent_key=None): """ Flatten a dictionary. Values that are dictionaries are flattened using delimiter in between (eg. parent-child) Values that are lists are flattened using delimiter followed by the index (eg. parent-0) example: .. code-block:: python { 'fish_facts': { 'sharks': 'Most will drown if they stop moving', 'skates': 'More than 200 species', }, 'fruits': ['apple', 'peach', 'watermelon'], 'number': 52 } # becomes { 'fish_facts-sharks': 'Most will drown if they stop moving', 'fish_facts-skates': 'More than 200 species', 'fruits-0': 'apple', 'fruits-1': 'peach', 'fruits-2': 'watermelon', 'number': 52 } # if intermediates is True then we also get unflattened elements # as well as the flattened ones. { 'fish_facts': { 'sharks': 'Most will drown if they stop moving', 'skates': 'More than 200 species', }, 'fish_facts-sharks': 'Most will drown if they stop moving', 'fish_facts-skates': 'More than 200 species', 'fruits': ['apple', 'peach', 'watermelon'], 'fruits-0': 'apple', 'fruits-1': 'peach', 'fruits-2': 'watermelon', 'number': 52 } """ items = [] if isinstance(d, list): d = dict(enumerate(d)) for k, v in d.items(): if parent_key: k = u"{}{}{}".format(parent_key, delimiter, k) if intermediates: items.append((k, v)) if isinstance(v, list): v = dict(enumerate(v)) if isinstance(v, collections.Mapping): items.extend( self.flatten_dict(v, delimiter, intermediates, str(k)).items()) else: items.append((str(k), v)) return dict(items) def format_units(self, value, unit="B", optimal=5, auto=True, si=False): """ Takes a value and formats it for user output, we can choose the unit to use eg B, MiB, kbits/second. This is mainly for use with bytes/bits it converts the value into a human readable form. It has various additional options but they are really only for special cases. The function returns a tuple containing the new value (this is a number so that the user can still format it if required) and a unit that is the units that we have been converted to. By supplying unit to the function we can force those units to be used eg ``unit=KiB`` would force the output to be in Kibibytes. By default we use non-si units but if the unit is si eg kB then we will switch to si units. Units can also be things like ``Mbit/sec``. If the auto parameter is False then we use the unit provided. This only makes sense when the unit is singular eg 'Bytes' and we want the result in bytes and not say converted to MBytes. optimal is used to control the size of the output value. We try to provide an output value of that number of characters (including decimal point), it may also be less due to rounding. If a fixed unit is used the output may be more than this number of characters. """ UNITS = "KMGTPEZY" DECIMAL_SIZE = 1000 BINARY_SIZE = 1024 CUTOFF = 1000 can_round = False if unit: # try to guess the unit. Do we have a known prefix too it? if unit[0].upper() in UNITS: index = UNITS.index(unit[0].upper()) + 1 post = unit[1:] si = len(unit) > 1 and unit[1] != "i" if si: post = post[1:] auto = False else: index = 0 post = unit if si: size = DECIMAL_SIZE else: size = BINARY_SIZE if auto: # we will try to use an appropriate prefix if value < CUTOFF: unit_out = post else: value /= size for prefix in UNITS: if abs(value) < CUTOFF: break value /= size if si: # si kilo is lowercase if prefix == "K": prefix = "k" else: post = "i" + post unit_out = prefix + post can_round = True else: # we are using a fixed unit unit_out = unit size = pow(size, index) if size: value /= size can_round = True if can_round and optimal and value: # we will try to make the output value the desired size # we need to keep out value as a numeric type places = int(log10(abs(value))) if places >= optimal - 2: value = int(value) else: value = round(value, max(optimal - places - 2, 0)) return value, unit_out def is_color(self, color): """ Tests to see if a color is defined. Because colors can be set to None in the config and we want this to be respected in an expression like. color = self.py3.COLOR_MUTED or self.py3.COLOR_BAD The color is treated as True but sometimes we want to know if the color has a value set in which case the color should count as False. This function is a helper for this second case. """ return not (color is None or hasattr(color, "none_setting")) def i3s_config(self): """ returns the i3s_config dict. """ return self._i3s_config def is_python_2(self): """ True if the version of python being used is 2.x Can be helpful for fixing python 2 compatibility issues """ return self._is_python_2 def is_my_event(self, event): """ Checks if an event triggered belongs to the module receiving it. This is mainly for containers who will also receive events from any children they have. Returns True if the event name and instance match that of the module checking. """ return (event.get("name") == self._module.module_name and event.get("instance") == self._module.module_inst) def log(self, message, level=LOG_INFO): """ Log the message. The level must be one of LOG_ERROR, LOG_INFO or LOG_WARNING """ assert level in [ self.LOG_ERROR, self.LOG_INFO, self.LOG_WARNING, ], "level must be LOG_ERROR, LOG_INFO or LOG_WARNING" # nicely format logs if we can using pretty print if isinstance(message, (dict, list, set, tuple)): message = pformat(message) # start on new line if multi-line output try: if "\n" in message: message = "\n" + message except: # noqa e722 pass message = "Module `{}`: {}".format(self._module.module_full_name, message) self._py3_wrapper.log(message, level) def update(self, module_name=None): """ Update a module. If module_name is supplied the module of that name is updated. Otherwise the module calling is updated. """ if not module_name: return self._module.force_update() else: module_info = self._get_module_info(module_name) if module_info: module_info["module"].force_update() def get_output(self, module_name): """ Return the output of the named module. This will be a list. """ output = [] module_info = self._get_module_info(module_name) if module_info: output = module_info["module"].get_latest() # we do a deep copy so that any user does not change the actual output # of the module. return deepcopy(output) def trigger_event(self, module_name, event): """ Trigger an event on a named module. """ if module_name: self._py3_wrapper.events_thread.process_event(module_name, event) def prevent_refresh(self): """ Calling this function during the on_click() method of a module will request that the module is not refreshed after the event. By default the module is updated after the on_click event has been processed. """ self._module.prevent_refresh = True def notify_user(self, msg, level="info", rate_limit=5, title=None, icon=None): """ Send a notification to the user. level must be 'info', 'error' or 'warning'. rate_limit is the time period in seconds during which this message should not be repeated. icon must be an icon path or icon name. """ module_name = self._module.module_full_name if isinstance(msg, Composite): msg = msg.text() if title is None: title = "py3status: {}".format(module_name) elif isinstance(title, Composite): title = title.text() # force unicode for python2 str if self._is_python_2: if isinstance(msg, str): msg = msg.decode("utf-8") if isinstance(title, str): title = title.decode("utf-8") if msg: self._py3_wrapper.notify_user( msg=msg, level=level, rate_limit=rate_limit, module_name=module_name, title=title, icon=icon, ) def register_function(self, function_name, function): """ Register a function for the module. The following functions can be registered .. py:function:: content_function() Called to discover what modules a container is displaying. This is used to determine when updates need passing on to the container and also when modules can be put to sleep. the function must return a set of module names that are being displayed. .. note:: This function should only be used by containers. .. py:function:: urgent_function(module_names) This function will be called when one of the contents of a container has changed from a non-urgent to an urgent state. It is used by the group module to switch to displaying the urgent module. ``module_names`` is a list of modules that have become urgent .. note:: This function should only be used by containers. """ my_info = self._get_module_info(self._module.module_full_name) my_info[function_name] = function def time_in(self, seconds=None, sync_to=None, offset=0): """ Returns the time a given number of seconds into the future. Helpful for creating the ``cached_until`` value for the module output. .. note:: from version 3.1 modules no longer need to explicitly set a ``cached_until`` in their response unless they wish to directly control it. :param seconds: specifies the number of seconds that should occur before the update is required. Passing a value of ``CACHE_FOREVER`` returns ``CACHE_FOREVER`` which can be useful for some modules. :param sync_to: causes the update to be synchronized to a time period. 1 would cause the update on the second, 60 to the nearest minute. By default we synchronize to the nearest second. 0 will disable this feature. :param offset: is used to alter the base time used. A timer that started at a certain time could set that as the offset and any synchronization would then be relative to that time. """ # if called with CACHE_FOREVER we just return this if seconds is self.CACHE_FOREVER: return self.CACHE_FOREVER if seconds is None: # If we have a sync_to then seconds can be 0 if sync_to and sync_to > 0: seconds = 0 else: try: # use py3status modules cache_timeout seconds = self._py3status_module.cache_timeout except AttributeError: # use default cache_timeout seconds = self._module.config["cache_timeout"] # Unless explicitly set we sync to the nearest second # Unless the requested update is in less than a second if sync_to is None: if seconds and seconds < 1: if 1 % seconds == 0: sync_to = seconds else: sync_to = 0 else: sync_to = 1 if seconds: seconds -= 0.1 requested = time() + seconds - offset # if sync_to then we find the sync time for the requested time if sync_to: requested = (requested + sync_to) - (requested % sync_to) return requested + offset def format_contains(self, format_string, names): """ Determines if ``format_string`` contains a placeholder string ``names`` or a list of placeholders ``names``. ``names`` is tested against placeholders using fnmatch so the following patterns can be used: .. code-block:: none * matches everything ? matches any single character [seq] matches any character in seq [!seq] matches any character not in seq This is useful because a simple test like ``'{placeholder}' in format_string`` will fail if the format string contains placeholder formatting eg ``'{placeholder:.2f}'`` """ # We cache things to prevent parsing the format_string more than needed if isinstance(names, list): key = str(names) else: key = names names = [names] try: return self._format_placeholders_cache[format_string][key] except KeyError: pass if format_string not in self._format_placeholders: placeholders = self._formatter.get_placeholders(format_string) self._format_placeholders[format_string] = placeholders else: placeholders = self._format_placeholders[format_string] if format_string not in self._format_placeholders_cache: self._format_placeholders_cache[format_string] = {} for name in names: for placeholder in placeholders: if fnmatch(placeholder, name): self._format_placeholders_cache[format_string][key] = True return True self._format_placeholders_cache[format_string][key] = False return False def get_color_names_list(self, format_strings): """ Returns a list of color names in ``format_string``. :param format_strings: Accepts a format string or a list of format strings. """ if not format_strings: return [] if not getattr(self._py3status_module, "thresholds", None): return [] if isinstance(format_strings, basestring): format_strings = [format_strings] names = set() for string in format_strings: for color in string.replace("&", " ").split("color=")[1::1]: color = color.split()[0] if "#" in color: continue if color in ["good", "bad", "degraded", "None", "threshold"]: continue if color in COLOR_NAMES: continue names.add(color) return list(names) def get_placeholders_list(self, format_string, match=None): """ Returns a list of placeholders in ``format_string``. If ``match`` is provided then it is used to filter the result using fnmatch so the following patterns can be used: .. code-block:: none * matches everything ? matches any single character [seq] matches any character in seq [!seq] matches any character not in seq This is useful because we just get simple placeholder without any formatting that may be applied to them eg ``'{placeholder:.2f}'`` will give ``['{placeholder}']`` """ if format_string not in self._format_placeholders: placeholders = self._formatter.get_placeholders(format_string) self._format_placeholders[format_string] = placeholders else: placeholders = self._format_placeholders[format_string] if not match: return list(placeholders) # filter matches found = [] for placeholder in placeholders: if fnmatch(placeholder, match): found.append(placeholder) return found def get_placeholder_formats_list(self, format_string): """ Parses the format_string and returns a list of tuples [(placeholder, format), ...]. eg ``'{placeholder:.2f}'`` will give ``[('placeholder', ':.2f')]`` """ return self._formatter.get_placeholder_formats_list(format_string) def update_placeholder_formats(self, format_string, formats): """ Update a format string adding formats if they are not already present. This is useful when for example a placeholder has a floating point value but by default we only want to show it to a certain precision. """ return self._formatter.update_placeholder_formats( format_string, formats) def safe_format(self, format_string, param_dict=None, force_composite=False, attr_getter=None): r""" Parser for advanced formatting. Unknown placeholders will be shown in the output eg ``{foo}``. Square brackets ``[]`` can be used. The content of them will be removed from the output if there is no valid placeholder contained within. They can also be nested. A pipe (vertical bar) ``|`` can be used to divide sections the first valid section only will be shown in the output. A backslash ``\`` can be used to escape a character eg ``\[`` will show ``[`` in the output. ``\?`` is special and is used to provide extra commands to the format string, example ``\?color=#FF00FF``. Multiple commands can be given using an ampersand ``&`` as a separator, example ``\?color=#FF00FF&show``. ``\?if=<placeholder>`` can be used to check if a placeholder exists. An exclamation mark ``!`` after the equals sign ``=`` can be used to negate the condition. ``\?if=<placeholder>=<value>`` can be used to determine if {<placeholder>} would be replaced with <value>. ``[]`` in <value> don't need to be escaped. ``{<placeholder>}`` will be converted, or removed if it is None or empty. Formatting can also be applied to the placeholder Eg ``{number:03.2f}``. example format_string: ``"[[{artist} - ]{title}]|{file}"`` This will show ``artist - title`` if artist is present, ``title`` if title but no artist, and ``file`` if file is present but not artist or title. param_dict is a dictionary of placeholders that will be substituted. If a placeholder is not in the dictionary then if the py3status module has an attribute with the same name then it will be used. .. note:: Added in version 3.3 Composites can be included in the param_dict. The result returned from this function can either be a string in the case of simple parsing or a Composite if more complex. If force_composite parameter is True a composite will always be returned. attr_getter is a function that will when called with an attribute name as a parameter will return a value. """ try: return self._formatter.format( format_string, self._py3status_module, param_dict, force_composite=force_composite, attr_getter=attr_getter, ) except Exception: self._report_exception( u"Invalid format `{}`".format(format_string)) return "invalid format" def build_composite(self, format_string, param_dict=None, composites=None, attr_getter=None): """ .. note:: deprecated in 3.3 use safe_format(). Build a composite output using a format string. Takes a format_string and treats it the same way as ``safe_format()`` but also takes a composites dict where each key/value is the name of the placeholder and either an output eg ``{'full_text': 'something'}`` or a list of outputs. """ if param_dict is None: param_dict = {} # merge any composites into the param_dict. # as they are no longer dealt with separately if composites: for key, value in composites.items(): param_dict[key] = Composite(value) try: return self._formatter.format( format_string, self._py3status_module, param_dict, force_composite=True, attr_getter=attr_getter, ) except Exception: self._report_exception( u"Invalid format `{}`".format(format_string)) return [{"full_text": "invalid format"}] def composite_update(self, item, update_dict, soft=False): """ Takes a Composite (item) if item is a type that can be converted into a Composite then this is done automatically. Updates all entries it the Composite with values from update_dict. Updates can be soft in which case existing values are not overwritten. A Composite object will be returned. """ return Composite.composite_update(item, update_dict, soft) def composite_join(self, separator, items): """ Join a list of items with a separator. This is used in joining strings, responses and Composites. A Composite object will be returned. """ return Composite.composite_join(separator, items) def composite_create(self, item): """ Create and return a Composite. The item may be a string, dict, list of dicts or a Composite. """ return Composite(item) def is_composite(self, item): """ Check if item is a Composite and return True if it is. """ return isinstance(item, Composite) def get_composite_string(self, format_string): """ Return a string from a Composite. """ if not isinstance(format_string, Composite): return "" return format_string.text() def check_commands(self, cmd_list): """ Checks to see if commands in list are available using ``which``. returns the first available command. If a string is passed then that command will be checked for. """ # if a string is passed then convert it to a list. This prevents an # easy mistake that could be made if isinstance(cmd_list, basestring): cmd_list = [cmd_list] for cmd in cmd_list: if self.command_run("which {}".format(cmd)) == 0: return cmd def command_run(self, command): """ Runs a command and returns the exit code. The command can either be supplied as a sequence or string. An Exception is raised if an error occurs """ # convert the command to sequence if a string if isinstance(command, basestring): command = shlex.split(command) try: return Popen(command, stdout=PIPE, stderr=PIPE, close_fds=True).wait() except Exception as e: # make a pretty command for error loggings and... if isinstance(command, basestring): pretty_cmd = command else: pretty_cmd = " ".join(command) msg = "Command `{cmd}` {error}".format(cmd=pretty_cmd, error=e.errno) raise exceptions.CommandError(msg, error_code=e.errno) def command_output(self, command, shell=False, capture_stderr=False, localized=False): """ Run a command and return its output as unicode. The command can either be supplied as a sequence or string. :param command: command to run can be a str or list :param shell: if `True` then command is run through the shell :param capture_stderr: if `True` then STDERR is piped to STDOUT :param localized: if `False` then command is forced to use its default (English) locale A CommandError is raised if an error occurs """ # make a pretty command for error loggings and... if isinstance(command, basestring): pretty_cmd = command else: pretty_cmd = " ".join(command) # convert the non-shell command to sequence if it is a string if not shell and isinstance(command, basestring): command = shlex.split(command) stderr = STDOUT if capture_stderr else PIPE env = self._english_env if not localized else None try: process = Popen( command, stdout=PIPE, stderr=stderr, close_fds=True, universal_newlines=True, shell=shell, env=env, ) except Exception as e: msg = "Command `{cmd}` {error}".format(cmd=pretty_cmd, error=e) raise exceptions.CommandError(msg, error_code=e.errno) output, error = process.communicate() if self._is_python_2 and isinstance(output, str): output = output.decode("utf-8") error = error.decode("utf-8") retcode = process.poll() if retcode: # under certain conditions a successfully run command may get a # return code of -15 even though correct output was returned see # #664. This issue seems to be related to arch linux but the # reason is not entirely clear. if retcode == -15: msg = "Command `{cmd}` returned SIGTERM (ignoring)" self.log(msg.format(cmd=pretty_cmd)) else: msg = "Command `{cmd}` returned non-zero exit status {error}" output_oneline = output.replace("\n", " ") if output_oneline: msg += " ({output})" msg = msg.format(cmd=pretty_cmd, error=retcode, output=output_oneline) raise exceptions.CommandError(msg, error_code=retcode, error=error, output=output) return output def _storage_init(self): """ Ensure that storage is initialized. """ if not self._storage.initialized: self._storage.init(self._module._py3_wrapper, self._is_python_2) def storage_set(self, key, value): """ Store a value for the module. """ if not self._module: return self._storage_init() module_name = self._module.module_full_name return self._storage.storage_set(module_name, key, value) def storage_get(self, key): """ Retrieve a value for the module. """ if not self._module: return self._storage_init() module_name = self._module.module_full_name return self._storage.storage_get(module_name, key) def storage_del(self, key=None): """ Remove the value stored with the key from storage. If key is not supplied then all values for the module are removed. """ if not self._module: return self._storage_init() module_name = self._module.module_full_name return self._storage.storage_del(module_name, key=key) def storage_keys(self): """ Return a list of the keys for values stored for the module. Keys will contain the following metadata entries: - '_ctime': storage creation timestamp - '_mtime': storage last modification timestamp """ if not self._module: return [] self._storage_init() module_name = self._module.module_full_name return self._storage.storage_keys(module_name) def storage_items(self): """ Return key, value pairs of the stored data for the module. Keys will contain the following metadata entries: - '_ctime': storage creation timestamp - '_mtime': storage last modification timestamp """ if not self._module: return {}.items() self._storage_init() items = [] module_name = self._module.module_full_name for key in self._storage.storage_keys(module_name): value = self._storage.storage_get(module_name, key) items.add((key, value)) return items def play_sound(self, sound_file): """ Plays sound_file if possible. """ self.stop_sound() cmd = self.check_commands(["ffplay", "paplay", "play"]) if cmd: if cmd == "ffplay": cmd = "ffplay -autoexit -nodisp -loglevel 0" sound_file = os.path.expanduser(sound_file) c = shlex.split("{} {}".format(cmd, sound_file)) self._audio = Popen(c) def stop_sound(self): """ Stops any currently playing sounds for this module. """ if self._audio: self._audio.kill() self._audio = None def threshold_get_color(self, value, name=None): """ Obtain color for a value using thresholds. The value will be checked against any defined thresholds. These should have been set in the i3status configuration. If more than one threshold is needed for a module then the name can also be supplied. If the user has not supplied a named threshold but has defined a general one that will be used. If the gradients config parameter is True then rather than sharp thresholds we will use a gradient between the color values. :param value: numerical value to be graded :param name: accepts a string, otherwise 'threshold' accepts 3-tuples to allow name with different values eg ('name', 'key', 'thresholds') """ # If first run then process the threshold data. if self._thresholds is None: self._thresholds_init() # allow name with different values if isinstance(name, tuple): name_used = "{}/{}".format(name[0], name[1]) if name[2]: self._thresholds[name_used] = [(x[0], self._get_color(x[1])) for x in name[2]] name = name[0] else: # if name not in thresholds info then use defaults name_used = name if name_used not in self._thresholds: name_used = None # convert value to int/float thresholds = self._thresholds.get(name_used) color = None try: value = float(value) except (TypeError, ValueError): pass # skip on empty thresholds/values if not thresholds or value in [None, ""]: pass elif isinstance(value, basestring): # string for threshold in thresholds: if value == threshold[0]: color = threshold[1] break else: # int/float try: if self._get_config_setting("gradients"): try: colors, minimum, maximum = self._threshold_gradients[ name_used] except KeyError: colors = self._gradients.make_threshold_gradient( self, thresholds) minimum = min(thresholds)[0] maximum = max(thresholds)[0] self._threshold_gradients[name_used] = ( colors, minimum, maximum, ) if value < minimum: color = colors[0] elif value > maximum: color = colors[-1] else: value -= minimum col_index = int( ((len(colors) - 1) / (maximum - minimum)) * value) color = colors[col_index] else: color = thresholds[0][1] for threshold in thresholds: if value >= threshold[0]: color = threshold[1] else: break except TypeError: color = None # save color so it can be accessed via safe_format() if name: color_name = "color_threshold_%s" % name else: color_name = "color_threshold" setattr(self._py3status_module, color_name, color) return color def request( self, url, params=None, data=None, headers=None, timeout=None, auth=None, cookiejar=None, ): """ Make a request to a url and retrieve the results. If the headers parameter does not provide an 'User-Agent' key, one will be added automatically following the convention: py3status/<version> <per session random uuid> :param url: url to request eg `http://example.com` :param params: extra query string parameters as a dict :param data: POST data as a dict. If this is not supplied the GET method will be used :param headers: http headers to be added to the request as a dict :param timeout: timeout for the request in seconds :param auth: authentication info as tuple `(username, password)` :param cookiejar: an object of a CookieJar subclass :returns: HttpResponse """ # The aim of this function is to be a limited lightweight replacement # for the requests library but using only pythons standard libs. # IMPORTANT NOTICE # This function is excluded from private variable hiding as it is # likely to need api keys etc which people may have obfuscated. # Therefore it is important that no logging is done in this function # that might reveal this information. if headers is None: headers = {} if timeout is None: timeout = getattr(self._py3status_module, "request_timeout", 10) if "User-Agent" not in headers: headers["User-Agent"] = "py3status/{} {}".format( version, self._uid) return HttpResponse( url, params=params, data=data, headers=headers, timeout=timeout, auth=auth, cookiejar=cookiejar, )
class BaseTestServer(object): def __init__(self, host='localhost', port=None, cwd=None, shell=False, stdin=DEVNULL, stdout=DEVNULL, stderr=DEVNULL): self.host = host self.port = port or port_for.select_random() self.proc = None self.shell = shell self.cwd = cwd self.stdin = stdin self.stdout = stdout self.stderr = stderr self.arguments = [ sys.executable, '-u', '-m', 'SimpleHTTPServer', str(self.port) ] def start(self): self.proc = Popen(self.arguments, stdin=self.stdin, stdout=self.stdout, stderr=self.stderr, shell=self.shell, cwd=self.cwd, env=get_testenv()) self.proc.poll() if self.proc.returncode: msg = ( "unable to start server. error code: %d - stderr follows: \n%s" ) % (self.proc.returncode, self.proc.stderr.read()) raise RuntimeError(msg) try: self._wait_for_port() finally: print(self._non_block_read(self.proc.stderr)) pass def stop(self): if self.proc is None: raise RuntimeError("Server wasn't started") self.proc.kill() self.proc.wait() self.proc = None def __enter__(self): self.start() return self def __exit__(self, exc_type, exc_val, exc_tb): self.stop() def url(self, path=''): return urljoin('http://{}:{}'.format(self.host, self.port), path) def _wait_for_port(self, delay=0.1, attempts=20): """Imports could take some time, server opens port with some delay.""" while attempts > 0: s = socket.socket() try: s.connect((self.host, self.port)) except Exception: time.sleep(delay) attempts -= 1 else: return finally: s.close() raise RuntimeError("Port %d is not open" % self.port) @staticmethod def _non_block_read(output): if output is None: return '' fd = output.fileno() fl = fcntl.fcntl(fd, fcntl.F_GETFL) fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK) try: return output.read() except Exception: return ''
class P2P: """ Forks a child process and executes p2pd command with given arguments. Can be used for peer to peer communication and procedure calls. Sends SIGKILL to the child in destructor. """ HEADER_LEN = 8 BYTEORDER = 'big' PB_HEADER_LEN = 1 RESULT_MESSAGE = b'\x00' ERROR_MESSAGE = b'\x01' DHT_MODE_MAPPING = { 'dht': { 'dht': 1 }, 'dht_server': { 'dhtServer': 1 }, 'dht_client': { 'dhtClient': 1 }, } FORCE_REACHABILITY_MAPPING = { 'public': { 'forceReachabilityPublic': 1 }, 'private': { 'forceReachabilityPrivate': 1 }, } def __init__(self): self._child = None self._alive = False self._listen_task = None self._server_stopped = asyncio.Event() @classmethod async def create(cls, *args, quic: bool = True, tls: bool = True, conn_manager: bool = True, dht_mode: str = 'dht_server', force_reachability: Optional[str] = None, nat_port_map: bool = True, auto_nat: bool = True, bootstrap: bool = False, bootstrap_peers: Optional[List[str]] = None, use_global_ipfs: bool = False, host_port: int = None, daemon_listen_port: int = None, **kwargs): """ Start a new p2pd process and connect to it. :param args: :param quic: Enables the QUIC transport :param tls: Enables TLS1.3 channel security protocol :param conn_manager: Enables the Connection Manager :param dht_mode: DHT mode (dht_client/dht_server/dht) :param force_reachability: Force reachability mode (public/private) :param nat_port_map: Enables NAT port mapping :param auto_nat: Enables the AutoNAT service :param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled :param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers :param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None) :param host_port: port for p2p network :param daemon_listen_port: port for connection daemon and client binding :param kwargs: :return: new wrapper for p2p daemon """ assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \ 'Trying to create with bootstrap node without bootstrap nodes list. ' \ 'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \ 'If you really want this, pass use_global_ipfs=True' assert not (bootstrap_peers is not None and use_global_ipfs), \ 'Non empty bootstrap_nodes and use_global_ipfs=True are incompatible.' \ 'Choose one option: your nodes list (preferable) or global ipfs (very unstable)' self = cls() with path(cli, P2PD_FILENAME) as p: p2pd_path = p bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers) dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0}) force_reachability = cls.FORCE_REACHABILITY_MAPPING.get( force_reachability, {}) proc_args = self._make_process_args(str(p2pd_path), *args, quic=quic, tls=tls, connManager=conn_manager, natPortMap=nat_port_map, autonat=auto_nat, b=bootstrap, **{ **bootstrap_peers, **dht, **force_reachability, **kwargs }) self._assign_daemon_ports(host_port, daemon_listen_port) for try_count in range(NUM_RETRIES): try: self._initialize(proc_args) await self._wait_for_client(RETRY_DELAY * (2**try_count)) break except Exception as e: logger.debug(f"Failed to initialize p2p daemon: {e}") self._terminate() if try_count == NUM_RETRIES - 1: raise self._assign_daemon_ports() return self @classmethod async def replicate(cls, daemon_listen_port: int, host_port: int): """ Connect to existing p2p daemon :param daemon_listen_port: port for connection daemon and client binding :param host_port: port for p2p network :return: new wrapper for existing p2p daemon """ self = cls() # There is no child under control # Use external already running p2pd self._child = None self._alive = True self._assign_daemon_ports(host_port, daemon_listen_port) self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) await self._wait_for_client() return self async def wait_for_at_least_n_peers(self, n_peers, attempts=3, delay=1): for _ in range(attempts): peers = await self._client.list_peers() if len(peers) >= n_peers: return await asyncio.sleep(delay) raise RuntimeError('Not enough peers') def _initialize(self, proc_args: List[str]) -> None: proc_args = deepcopy(proc_args) proc_args.extend( self._make_process_args( hostAddrs= f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic', listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}')) self._child = Popen(args=proc_args, encoding="utf8") self._alive = True self._client_listen_port = find_open_port() self._client = p2pclient.Client( Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'), Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}')) async def _wait_for_client(self, delay=0): await asyncio.sleep(delay) encoded = await self._client.identify() self.id = encoded[0].to_base58() def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None): if host_port is None: host_port = find_open_port() if daemon_listen_port is None: daemon_listen_port = find_open_port() while daemon_listen_port == host_port: daemon_listen_port = find_open_port() self._host_port, self._daemon_listen_port = host_port, daemon_listen_port @staticmethod async def send_raw_data(byte_str, writer): request = len(byte_str).to_bytes(P2P.HEADER_LEN, P2P.BYTEORDER) + byte_str writer.write(request) @staticmethod async def send_msgpack(data, writer): raw_data = MSGPackSerializer.dumps(data) await P2P.send_raw_data(raw_data, writer) @staticmethod async def send_protobuf(protobuf, out_proto_type, writer): if type(protobuf) != out_proto_type: raise TypeError('Unary handler returned protobuf of wrong type.') if out_proto_type == p2pd_pb2.RPCError: await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer) else: await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer) await P2P.send_raw_data(protobuf.SerializeToString(), writer) @staticmethod async def receive_raw_data(reader: asyncio.StreamReader, header_len=HEADER_LEN): header = await reader.readexactly(header_len) content_length = int.from_bytes(header, P2P.BYTEORDER) data = await reader.readexactly(content_length) return data @staticmethod async def receive_msgpack(reader): return MSGPackSerializer.loads(await P2P.receive_raw_data(reader)) @staticmethod async def receive_protobuf(in_proto_type, reader): msg_type = await P2P.receive_raw_data(reader) if msg_type == P2P.RESULT_MESSAGE: protobuf = in_proto_type() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return protobuf, None elif msg_type == P2P.ERROR_MESSAGE: protobuf = p2pd_pb2.RPCError() protobuf.ParseFromString(await P2P.receive_raw_data(reader)) return None, protobuf else: raise TypeError('Invalid Protobuf message type') @staticmethod def _handle_stream(handle): async def do_handle_stream(stream_info, reader, writer): try: request = await P2P.receive_raw_data(reader) except asyncio.IncompleteReadError: logger.debug( "Incomplete read while receiving request from peer") writer.close() return try: result = handle(request) await P2P.send_raw_data(result, writer) finally: writer.close() return do_handle_stream @staticmethod def _handle_unary_stream(handle, context, in_proto_type, out_proto_type): async def watchdog(reader: asyncio.StreamReader): await reader.read(n=1) raise P2PInterruptedError() async def do_handle_unary_stream(stream_info: StreamInfo, reader: asyncio.StreamReader, writer: asyncio.StreamWriter) -> None: try: try: request = await P2P.receive_protobuf(in_proto_type, reader) except asyncio.IncompleteReadError: logger.debug( "Incomplete read while receiving request from peer") return except google.protobuf.message.DecodeError as error: logger.exception(error) return context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr done, pending = await asyncio.wait( [watchdog(reader), handle(request, context)], return_when=asyncio.FIRST_COMPLETED) try: result = done.pop().result() await P2P.send_protobuf(result, out_proto_type, writer) except P2PInterruptedError: pass except Exception as exc: error = p2pd_pb2.RPCError(message=str(exc)) await P2P.send_protobuf(error, p2pd_pb2.RPCError, writer) finally: pending_task = pending.pop() pending_task.cancel() try: await pending_task except asyncio.CancelledError: pass finally: writer.close() return do_handle_unary_stream def start_listening(self): async def listen(): async with self._client.listen(): await self._server_stopped.wait() self._listen_task = asyncio.create_task(listen()) async def stop_listening(self): if self._listen_task is not None: self._server_stopped.set() self._listen_task.cancel() try: await self._listen_task except asyncio.CancelledError: self._listen_task = None self._server_stopped.clear() async def add_stream_handler(self, name, handle): if self._listen_task is None: self.start_listening() await self._client.stream_handler(name, self._handle_stream(handle)) async def add_unary_handler(self, name, handle, in_proto_type, out_proto_type): if self._listen_task is None: self.start_listening() context = P2PContext(id=self.id, port=self._host_port, handle_name=name) await self._client.stream_handler( name, P2P._handle_unary_stream(handle, context, in_proto_type, out_proto_type)) async def call_peer_handler(self, peer_id, handler_name, input_data): libp2p_peer_id = PeerID.from_base58(peer_id) stream_info, reader, writer = await self._client.stream_open( libp2p_peer_id, (handler_name, )) try: await P2P.send_raw_data(input_data, writer) return await P2P.receive_raw_data(reader) finally: writer.close() def __del__(self): self._terminate() @property def is_alive(self): return self._alive async def shutdown(self): await asyncio.get_event_loop().run_in_executor(None, self._terminate) def _terminate(self): self._alive = False if self._child is not None and self._child.poll() is None: self._child.kill() self._child.wait() @staticmethod def _make_process_args(*args, **kwargs) -> List[str]: proc_args = [] proc_args.extend(str(entry) for entry in args) proc_args.extend(f'-{key}={P2P._convert_process_arg_type(value)}' if value is not None else f'-{key}' for key, value in kwargs.items()) return proc_args @staticmethod def _convert_process_arg_type(val): if isinstance(val, bool): return 1 if val else 0 return val @staticmethod def _make_bootstrap_peers(nodes): if nodes is None: return {} return {'bootstrapPeers': ','.join(nodes)}
from subprocess import Popen from time import sleep p = Popen("gedit") sleep(15) p.kill() # p.terminate()
class Airodump(Air): """ TODO This accepts the following parameters from airodump-ng's help. * ivs * gpsd * beacons * manufacturer * uptime * ignore_negative_one * a * showack * h * f * update * berlin * r * x * encrypt * netmask * bssid * essid * output_format * write * essid_regex """ _aps = [] _clients = [] _stop = False _allowed_arguments = ( ('ivs', False), ('gpsd', False), ('beacons', False), ('manufacturer', False), ('uptime', False), ('ignore_negative_one', False), ('a', False), ('showack', False), ('h', False), ('f', False), ('update', False), ('berlin', False), ('r', False), ('x', False), ('encrypt', False), ('netmask', False), ('bssid', False), ('essid', False), ('output_format', False), ('write', False), ('essid_regex', False), ('cswitch', False), ('channel', False), ) def __init__(self, interface=False, **kwargs): self.interface = interface super(self.__class__, self).__init__(**kwargs) @property def tree(self): """ Returns currently reported aps """ keys = [ 'FirstTimeSeen', 'LastTimeSeen', 'channel', 'Speed', 'Privacy', 'Cipher', 'Authentication', 'Power', 'beacons', 'IV', 'LANIP', 'IDlength', 'ESSID', 'Key' ] c_keys = [ 'Station MAC', 'FirstTimeSeen', 'LastTimeSeen', 'Power', 'Packets', 'BSSID', 'ProbedESSIDs' ] self.update_results() aps = {} for ap_ in self._aps: bssid = ap_.pop(0) aps[bssid] = dict(zip(keys, ap_)) aps[bssid]['clients'] = [] for client in self.clients: if client[0] == bssid: aps[bssid]['clients'].append(dict(zip(c_keys, client))) return aps @property def clients(self): """ Returns currently reported clients """ self.update_results() return self._clients def scan(self): """ Get next result: implement in childrens Both this and previous one must be responsible for duplicates """ self.start() while not os.path.exists(self.curr_csv): time.sleep(5) def watch_process(self): """ Watcher thread. This one relaunches airodump eatch time it dies until we call stop() """ psutil.wait_procs([psutil.Process(self._proc.pid)], callback=self.start) def start(self, _=False): """ Start process. psutil sends an argument (that we don't actually need...) interface defaults to monitor interface 0 as started by Airmon """ if not self._stop: self._current_execution += 1 flags = self.flags if '--write' not in flags: flags.extend(['--write', self.writepath]) if '--output-format' not in flags: flags.extend(['--output-format', 'csv']) line = ["airodump-ng"] + flags + self.arguments + [self.interface] self._proc = Popen(line, bufsize=0, env={'PATH': os.environ['PATH']}, stderr=DEVNULL, stdin=DEVNULL, stdout=DEVNULL) os.system('stty sane') time.sleep(5) watcher = threading.Thread(target=self.watch_process) watcher.start() def stop(self): """ Stop proc. """ self._stop = True return self._proc.kill() def update_results(self): """ Updates self.clients and self.aps """ clis = [] aps = [] with open(self.curr_csv) as fileo: file_ = fileo.readlines() file_enum = enumerate(file_) num = 0 for num, line in file_enum: line = line.replace('\0', '') if not line: continue if line.startswith('BSSID'): continue if line.startswith('Station'): num += 1 break aps.append(line) for line in file_[num:]: clis.append(line) def clean_rows(reader): """ Airodump-ng's csv info comes a bit unclean. Strip each line of its extra blank spaces """ return [[a.strip() for a in row] for row in reader if row] self._aps = clean_rows(csv.reader(StringIO('\n'.join(aps)))) self._clients = clean_rows(csv.reader(StringIO('\n'.join(clis))))
def gradeGroup(group_dir, question_ID): #student_question_ID = student_ID + question_ID os.chdir("./Groups/"+group_dir) #----------------------------------------------------- # Edit the .bochsrc-no-gui, replace the partport1 line to this question timestamped output file # We do this to make sure that Autograde doesn't mistakenly take the same file for the next question #----------------------------------------------------- # fraction of a second time stamp QTimestamp = str(time.time()).replace(".","") # This question specific student_stdout file name student_stdout_default = "student_stdout" student_stdout_Q_specific = student_stdout_default+"_"+QTimestamp bochsrcDefault = open(os.getcwd()+"/.bochsrc-no-gui", 'r') bochsrcAutograde = open(os.getcwd()+"/.bochsrc_autograde_"+QTimestamp, 'w') for line in bochsrcDefault: bochsrcAutograde.write(line.replace(student_stdout_default, student_stdout_Q_specific)) bochsrcDefault.close() bochsrcAutograde.close() #----------------------------------------------------- # TODO: Improvment: If a compilation error occurs to a student folder, let the script save the output # of "make" into a disk file (just like it does for Bochs output) # According to the number of cores in your machine, you can change the -j number to compile faster # example: I use -j4 on my quad core machine makeProc = subprocess.Popen("make -j2", shell=True, stdout=PIPE, stderr=PIPE) #makeProc.wait() makeProc.communicate() #wait() caused the deadlock with stdout, err = PIPE, online documentation warns against that, and suggested the use of communicate() question_grade = -100 if (os.path.isfile(os.getcwd()+"/obj/kern/bochs.img") == True): myCommand = 'bochs-no-gui -q -f .bochsrc_autograde_' + QTimestamp #proc = Popen(myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, preexec_fn=os.setsid) #si = subprocess.STARTUPINFO() #si.dwFlags |= subprocess.STARTF_USESTDHANDLES #proc = Popen(myCommand, shell=True, startupinfo=si, stdout=PIPE, stderr=STDOUT) #shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE) proc = Popen("exec " + myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE) #===== WARNING: Unix only work-around: non-blocking streams # default read() and readline() in Python block until a # valid output is available. # This makes it difficult to stop the bochs process # when we reach the FOS>, because we will not know when to check # as long as read() is working and doesn't return # # The workaround using fcntl non-blocking flag is Unix only solution # we should find an equivalent library to other platforms, in case # we migrate the course to windows or Mac #import fcntl #file_flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL) #fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, file_flags | os.O_NONBLOCK) # UPDATE: We didn't use this non-blocking work-around, and we returned # to using the default blocking read(1), but we made sure # to always check for the FOS> after every character read # from the stream. if for some unusual reason the FOS> # prompt isn't printed, we will have to manually close the # bochs window to allow the script to continue working. # - JUST MAKE SURE to run the script with python - u (unbuffered) #============= stream_data = "" question_grade = 0 #A closure (inner function) that I will use later to terminate any stucking threads or processes # and finialize remainig resources def finializeAndCleanUp(): try: if(proc.poll() is None): #kill the Bochs process if not already killed proc.kill() #os.kill(proc.pid, signal.SIGKILL) thefile.close() except Exception, msg: print msg try: startTimeOfQuestion = datetime.now() while os.path.isfile(os.getcwd() + "/" + student_stdout_Q_specific) == False: #keep waiting till we find the new student_stdout file generated by Bochs if ((datetime.now() - startTimeOfQuestion).seconds > 40): print "Timeout, no console output from Bochs" shutil.rmtree('./obj', ignore_errors=True) os.chdir("../../") if(proc.poll() is None): #kill the Bochs process if not already killed proc.kill() return -100, (datetime.now() - startTimeOfQuestion).seconds time.sleep(0.001) #TODO: Consider removing the try catch statements as a running time optimization thefile = open(os.getcwd() + "/"+student_stdout_Q_specific,"r") #+ "/Students"+"/"+student_dir+ while True: timeNow = datetime.now() if((timeNow - startTimeOfQuestion).seconds > 60): question_grade = -6 finializeAndCleanUp() print "Timeout" break try: #TODO: Revise this part to enable time out for project correction #def signal_handler(signum, frame): # raise Exception("Timed out!") #signal.signal(signal.SIGALRM, signal_handler) #signal.alarm(60*1) # 1 Min #try: # chunk = thefile.read(1) #except Exception, msg: # question_grade = -6 # finializeAndCleanUp() # print msg # break #time.sleep(0.00005) #this slowed down the loop consideralby, taking about 5 seconds chunk = thefile.read(1) if not chunk: continue #else: # sys.stdout.write(chunk) # sys.stdout.flush() stream_data += chunk #print stream_data if "InitialWSError1" in stream_data: question_grade = -4 finializeAndCleanUp() break elif "InitialWSError2" in stream_data: question_grade = -5 finializeAndCleanUp() break elif "[EVAL_FINAL]panic" in stream_data: if tests[question_ID] == 'tia' and "kernel [EVAL_FINAL]panic" in stream_data: question_grade = 1 finializeAndCleanUp() break else: question_grade = -7 #Panic Case finializeAndCleanUp() break elif stream_data.count('!! FCIS says HELLO !!') > 1: question_grade = -3 #Restart Case finializeAndCleanUp() break elif outputs[question_ID] == "TESTING MULTIPLE TIMES" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 3: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] == "TESTING MULTIPLE TIMES S2" and "[AUTO_GR@DING]Fibonacci #30 = 1346269" in stream_data and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 4: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] == "TESTING MULTIPLE TIMES S3" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 2: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] == "TESTING MULTIPLE TIMES S4" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 1: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] == "TESTING ENVFREE SC 2" and stream_data.count('[AUTO_GR@DING]Congratulations!! test Scenario 2 completed successfully.') == 1 and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 2: question_grade = 1 finializeAndCleanUp() break elif outputs[question_ID] in stream_data: question_grade = 1 finializeAndCleanUp() break elif "FOS>" in stream_data: finializeAndCleanUp() break elif proc.poll() is not None: #if process is finished, break the loop finializeAndCleanUp() break #elif chunk is None: #break #elif chunk == "": #break except IOError: if proc.poll() is not None: #print "exiting loop, io error" finializeAndCleanUp() break except Exception, msg: #print msg if proc.poll() is not None: #print "exiting loop" finializeAndCleanUp() break except IOError, msg: #print "exiting, io error opening student_stdout_..." print msg finializeAndCleanUp()
def inner_run(self, *args, **options): import sys shutdown_message = options.get('shutdown_message', '') do_reload = options.get('use_reloader', True) #We use the old dev appserver if threading is disabled or --old was passed use_old_dev_appserver = options.get( 'use_old_dev_appserver') or not options.get("use_threading") quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C' from djangae.utils import find_project_root, data_root from djangae.sandbox import _find_sdk_from_python_path from django.conf import settings from django.utils import translation #Check for app.yaml expected_path = os.path.join(find_project_root(), "app.yaml") if not os.path.exists(expected_path): sys.stderr.write("Unable to find app.yaml at '%s'\n" % expected_path) sys.exit(1) self.stdout.write("Validating models...\n\n") self.validate(display_num_errors=True) self.stdout.write( ("%(started_at)s\n" "Django version %(version)s, using settings %(settings)r\n" "Starting development server at http://%(addr)s:%(port)s/\n" "Quit the server with %(quit_command)s.\n") % { "started_at": datetime.now().strftime('%B %d, %Y - %X'), "version": self.get_version(), "settings": settings.SETTINGS_MODULE, "addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr, "port": self.port, "quit_command": quit_command, }) sys.stdout.write("\n") sys.stdout.flush() # django.core.management.base forces the locale to en-us. We should # set it up correctly for the first request (particularly important # in the "--noreload" case). translation.activate(settings.LANGUAGE_CODE) #Will have been set by setup_paths sdk_path = _find_sdk_from_python_path() if use_old_dev_appserver: dev_appserver = os.path.join(sdk_path, "old_dev_appserver.py") command = [ dev_appserver, find_project_root(), "-p", self.port, "-h", self.addr, "--use_sqlite", "--high_replication", "--allow_skipped_files", ] else: dev_appserver = os.path.join(sdk_path, "dev_appserver.py") command = [ dev_appserver, find_project_root(), "--port", self.port, "--host", self.addr, "--admin_port", str(int(self.port) + 1), "--automatic_restart", "True" if do_reload else "False", "--allow_skipped_files" ] process = Popen(command, stdout=sys.__stdout__, stderr=sys.__stderr__, cwd=find_project_root()) #This makes sure that dev_appserver gets killed on reload import atexit atexit.register(process.kill) try: process.wait() except KeyboardInterrupt: #Tell the dev appserver to shutdown and forcibly kill #if it takes too long process.send_signal(signal.SIGTERM) time.sleep(2) process.kill() if shutdown_message: sys.stdout.write(shutdown_message) #Some weird race condition crazy sometimes makes this None... if sys: sys.exit(process.returncode)
class DockerizedJobServer(object): """ Spins up the JobServer in a docker container for local execution """ def __init__(self, job_host="localhost", job_port=None, artifact_port=None, expansion_port=None, harness_port_range=(8100, 8200), max_connection_retries=5): self.job_host = job_host self.job_port = job_port self.expansion_port = expansion_port self.artifact_port = artifact_port self.harness_port_range = harness_port_range self.max_connection_retries = max_connection_retries self.docker_process = None self.process_lock = Lock() def start(self): # TODO This is hardcoded to Flink at the moment but should be changed job_server_image_name = os.environ['USER'] + \ "-docker-apache.bintray.io/beam/flink-job-server:latest" docker_path = check_output(['which', 'docker']).strip().decode('utf-8') cmd = [ "docker", "run", # We mount the docker binary and socket to be able to spin up # "sibling" containers for the SDK harness. "-v", ':'.join([docker_path, "/bin/docker"]), "-v", "/var/run/docker.sock:/var/run/docker.sock" ] self.job_port, self.artifact_port, self.expansion_port = \ DockerizedJobServer._pick_port(self.job_port, self.artifact_port, self.expansion_port) args = [ '--job-host', self.job_host, '--job-port', str(self.job_port), '--artifact-port', str(self.artifact_port), '--expansion-port', str(self.expansion_port) ] if sys.platform == "darwin": # Docker-for-Mac doesn't support host networking, so we need to explictly # publish ports from the Docker container to be able to connect to it. # Also, all other containers need to be aware that they run Docker-on-Mac # to connect against the internal Docker-for-Mac address. cmd += ["-e", "DOCKER_MAC_CONTAINER=1"] cmd += ["-p", "{}:{}".format(self.job_port, self.job_port)] cmd += [ "-p", "{}:{}".format(self.artifact_port, self.artifact_port) ] cmd += [ "-p", "{}:{}".format(self.expansion_port, self.expansion_port) ] cmd += [ "-p", "{0}-{1}:{0}-{1}".format(self.harness_port_range[0], self.harness_port_range[1]) ] else: # This shouldn't be set for MacOS because it detroys port forwardings, # even though host networking is not supported on MacOS. cmd.append("--network=host") cmd.append(job_server_image_name) cmd += args logging.debug("Starting container with %s", cmd) try: self.docker_process = Popen(cmd) atexit.register(self.stop) signal.signal(signal.SIGINT, self.stop) except: # pylint:disable=bare-except logging.exception("Error bringing up container") self.stop() return "{}:{}".format(self.job_host, self.job_port) def stop(self): with self.process_lock: if not self.docker_process: return num_retries = 0 while self.docker_process.poll() is None and \ num_retries < self.max_connection_retries: logging.debug("Sending SIGINT to job_server container") self.docker_process.send_signal(signal.SIGINT) num_retries += 1 time.sleep(1) if self.docker_process.poll is None: self.docker_process.kill() @staticmethod def _pick_port(*ports): """ Returns a list of ports, same length as input ports list, but replaces all None or 0 ports with a random free port. """ sockets = [] def find_free_port(port): if port: return port else: s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sockets.append(s) s.bind(('localhost', 0)) _, free_port = s.getsockname() return free_port ports = list(map(find_free_port, ports)) # Close sockets only now to avoid the same port to be chosen twice for s in sockets: s.close() return ports
def run_game(client1, client2, name): c1_status = update(client1) if c1_status != "good!": return ("Error", c1_status) c2_status = update(client2) if c2_status != "good!": return ("Error", c2_status) # if not update('server'): # return -13 startTime = time.time() #now start the server... servergood = False port = startport #slog = open(rootdir+"server.log","w") while not servergood: serverp = pexpect.spawn('python main.py ' + str(port), cwd=rootdir + 'server/', timeout=600) i = serverp.expect([ 'Unable to open socket!', 'Server Started', pexpect.EOF, pexpect.TIMEOUT ], timeout=10) if i == 0 or i == 3 or i == 2: #print "bad socket" port += 1 serverp.close(True) elif i == 1: #print tentacle+" server should be started on port "+str(port) servergood = True else: print 'THE WORLD IS AT AN END' return ("Error", "fail") #serverp.logfile = slog time.sleep(2) #now client 1.. client1p = Popen('/bin/bash ./run localhost:' + str(port), cwd=rootdir + repositories[client1], shell=True, stdout=file('/dev/null', 'w'), stderr=file('/dev/null', 'w')) #client1p = pexpect.spawn('/bin/bash ./run localhost:'+str(port)+'> /dev/null 2>&1', cwd = rootdir+repositories[client1], timeout = 600) #i = client1p.expect(['Creating game 1',pexpect.EOF,pexpect.TIMEOUT], timeout = 10) i = serverp.expect(['Creating game 1', pexpect.EOF, pexpect.TIMEOUT], timeout=10) if i == 0: pass #print tentacle+" game created!" else: #print client1p.returncode #print tentacle+" game failed to create:"+str(i) return ("Error", "Game failed to create " + client1) #and client 2... client2p = Popen(['/bin/bash ./run localhost:' + str(port) + ' 0'], cwd=rootdir + repositories[client2], shell=True, stdout=file('/dev/null', 'w'), stderr=file('/dev/null', 'w')) #client2p = pexpect.spawn('/bin/bash ./run localhost:'+str(port)+' 0 > /dev/null 2>&1', cwd = rootdir+repositories[client2], timeout = 600) #i = client2p.expect([pexpect.EOF,pexpect.TIMEOUT],timeout=5) i = serverp.expect(['Starting', pexpect.EOF, pexpect.TIMEOUT], timeout=5) if i != 0: #print tentacle+' '+client2+" couldn't connect" return ("Error", client2 + " couldn't connect to server") print tentacle + " game started! (" + client1 + " vs. " + client2 + " on port " + str( port) + ")" result = '' try: while time.time() < startTime + 600 and len(result) < 5: #result = serverp.expect(["Tie game!", "1 Wins!", "2 Wins", pexpect.TIMEOUT], timeout = 5) #try: # client1p.read_nonblocking(1024, timeout = 0) #except: # pass #try: # client2p.read_nonblocking(1024, timeout = 0) #except: # pass try: result = serverp.readline() if "win" not in result.lower() and "tie" not in result.lower(): result = '' except pexpect.TIMEOUT: pass except SoftTimeLimitExceeded: print tentacle + " TIMEOUT!" serverp.close(True) client1p.kill() client2p.kill() return ("Error", "Timeout!") #result = serverp.readlines() #print result #print tentacle+" game completed!" c1_score = 0 c2_score = 0 if "1" in result: # == "1 Wins!": c1_score = 1 winner = client1 #print client1, "wins" elif "2" in result or result == '': # == "2 Wins!": c2_score = 1 winner = client2 #print client2, "wins" elif "tie" in result.lower(): # == "Tie game!": #print "tie" winner = client2 c2_score = 1 elif result == '': pass #print "someone crashed!" else: print "wat?", result return ("Error", "Unknown results from server:" + result) serverp.close(True) client1p.kill() client2p.kill() #slog.close() #client1p.close(True) #client2p.close(True) #Popen("cp "+rootdir+"server.log ~/"+tentacle+str(time.time())+".server.log", shell = True).wait() #logfile = open(rootdir+'server/logs/1.gamelog.bz2','rb') #log = logfile.read() s3conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY) logbucket = s3conn.get_bucket("megaminer7") newkey = Key(logbucket) logname = 'logs/arena/' + str(tentacle) + '-' + str( time.time()) + '.gamelog.bz2' newkey.key = logname newkey.set_contents_from_filename(rootdir + "server/logs/1.gamelog.bz2") webserver.set_game_stat(client1, client2, c1_score, c2_score, working_copies[client1], working_copies[client2], logname) remove(rootdir + "server/logs/1.gamelog") remove(rootdir + "server/logs/1.gamelog.bz2") return (winner, logname)
class Client(object): def __init__(self, python_path, max_pickle_version, config_dir): # Make sure to init these variables (used in __del__) early on in case we # have an exception self._poller = None self._server_process = None self._socket_path = None data_transferer.defaultProtocol = max_pickle_version self._config_dir = config_dir # The client launches the server when created; we use # Unix sockets for now server_module = ".".join([__package__, "server"]) self._socket_path = "/tmp/%s_%d" % (os.path.basename(config_dir), os.getpid()) if os.path.exists(self._socket_path): raise RuntimeError("Existing socket: %s" % self._socket_path) env = os.environ.copy() #env["PYTHONPATH"] = ":".join(sys.path) self._server_process = Popen( [ python_path, "-u", "-m", server_module, str(max_pickle_version), config_dir, self._socket_path ], env=env, stdout=PIPE, stderr=PIPE, bufsize=1, universal_newlines=True, # Forces text as well ) # Read override configuration sys.path.insert(0, config_dir) override_module = importlib.import_module("overrides") sys.path = sys.path[1:] # Determine all overrides self._overrides = {} self._getattr_overrides = {} self._setattr_overrides = {} for override in override_module.__dict__.values(): if isinstance(override, (LocalOverride, LocalAttrOverride)): for obj_name, obj_funcs in override.obj_mapping.items(): if isinstance(override, LocalOverride): override_dict = self._overrides.setdefault( obj_name, {}) elif override.is_setattr: override_dict = self._setattr_overrides.setdefault( obj_name, {}) else: override_dict = self._getattr_overrides.setdefault( obj_name, {}) if isinstance(obj_funcs, str): obj_funcs = (obj_funcs, ) for name in obj_funcs: if name in override_dict: raise ValueError( "%s was already overriden for %s" % (name, obj_name)) override_dict[name] = override.func self._proxied_objects = {} # Wait for the socket to be up on the other side; we also check if the # server had issues starting up in which case we report that and crash out while not os.path.exists(self._socket_path): returncode = self._server_process.poll() if returncode is not None: raise RuntimeError( "Server did not properly start: %s" % self._server_process.stderr.read(), ) time.sleep(1) # Open up the channel and setup the datastransfer pipeline self._channel = Channel(SocketByteStream.unixconnect( self._socket_path)) self._datatransferer = DataTransferer(self) # Make PIPEs non-blocking; this is helpful to be able to # order the messages properly for f in (self._server_process.stdout, self._server_process.stderr): fl = fcntl.fcntl(f, fcntl.F_GETFL) fcntl.fcntl(f, fcntl.F_SETFL, fl | os.O_NONBLOCK) # Set up poller self._poller = select.poll() self._poller.register(self._server_process.stdout, select.POLLIN) self._poller.register(self._server_process.stderr, select.POLLIN) self._poller.register(self._channel, select.POLLIN | select.POLLHUP) # Get all exports that we are proxying response = self._communicate({ FIELD_MSGTYPE: MSG_CONTROL, FIELD_OPTYPE: CONTROL_GETEXPORTS }) self._proxied_classes = { k: None for k in itertools.chain(response[FIELD_CONTENT]["classes"], response[FIELD_CONTENT]["proxied"]) } # Proxied standalone functions are functions that are proxied # as part of other objects like defaultdict for which we create a # on-the-fly simple class that is just a callable. This is therefore # a special type of self._proxied_classes self._proxied_standalone_functions = {} self._export_info = { "classes": response[FIELD_CONTENT]["classes"], "functions": response[FIELD_CONTENT]["functions"], "values": response[FIELD_CONTENT]["values"], "exceptions": response[FIELD_CONTENT]["exceptions"], "aliases": response[FIELD_CONTENT]["aliases"] } self._aliases = response[FIELD_CONTENT]["aliases"] def __del__(self): # Clean up the server; we drain all messages if any if self._poller is not None: # If we have self._poller, we have self._server_process self._poller.unregister(self._channel) last_evts = self._poller.poll(5) for fd, _ in last_evts: # Readlines will never block here because bufsize is set to # 1 (line buffering) if fd == self._server_process.stdout.fileno(): sys.stdout.write(self._server_process.stdout.readline()) elif fd == self._server_process.stderr.fileno(): sys.stderr.write(self._server_process.stderr.readline()) sys.stdout.flush() sys.stderr.flush() if self._server_process is not None: # Attempt to send it a terminate signal and then wait and kill try: self._channel.send({ FIELD_MSGTYPE: MSG_CONTROL, FIELD_OPTYPE: CONTROL_SHUTDOWN }) self._channel.recv(timeout=10) # If we receive, we are sure we # are good except: # noqa E722 pass # If there is any issue sending this message, just ignore it self._server_process.kill() if self._socket_path is not None and os.path.exists(self._socket_path): os.unlink(self._socket_path) @property def name(self): return self._config_dir def get_exports(self): return self._export_info def stub_request(self, stub, request_type, *args, **kwargs): # Encode the operation to send over the wire and wait for the response target = self.encode(stub) encoded_args = [] for arg in args: encoded_args.append(self.encode(arg)) encoded_kwargs = [] for k, v in kwargs.items(): encoded_kwargs.append((self.encode(k), self.encode(v))) response = self._communicate({ FIELD_MSGTYPE: MSG_OP, FIELD_OPTYPE: request_type, FIELD_TARGET: target, FIELD_ARGS: self.encode(args), FIELD_KWARGS: self.encode([(k, v) for k, v in kwargs.items()]), }) response_type = response[FIELD_MSGTYPE] if response_type == MSG_REPLY: return self.decode(response[FIELD_CONTENT]) elif response_type == MSG_EXCEPTION: raise load_exception(self._datatransferer, response[FIELD_CONTENT]) elif response_type == MSG_INTERNAL_ERROR: raise RuntimeError( "Error in the server runtime:\n\n===== SERVER TRACEBACK =====\n%s" % response[FIELD_CONTENT]) def encode(self, obj): # This encodes an object to transfer back out # Basic data types will be sent over directly. # In this direction (client -> server), we error on non-basic # types. This could be changed by modifying the pickle_object method # here. return self._datatransferer.dump(obj) def decode(self, json_obj): # This decodes an object that was transferred in. This will call # unpickle_object where needed. Any remote object that is handled by # this connection will be converted to a local stub. return self._datatransferer.load(json_obj) def get_local_class(self, name, obj_id=None): # Gets (and creates if needed), the class mapping to the remote # class of name 'name'. name = self._get_canonical_name(name) if name == 'function': # Special handling of pickled functions. We create a new class that # simply has a __call__ method that will forward things back to # the server side. if obj_id is None: raise RuntimeError( "Local function unpickling without an object ID") if obj_id not in self._proxied_standalone_functions: self._proxied_standalone_functions[obj_id] = create_class( self, '__function_%s' % obj_id, {}, {}, {}, {'__call__': ''}) return self._proxied_standalone_functions[obj_id] if name not in self._proxied_classes: raise ValueError("Class '%s' is not known" % name) local_class = self._proxied_classes[name] if local_class is None: # We need to build up this class. To do so, we take everything that the # remote class has and remove UNSUPPORTED things and overriden things remote_methods = self.stub_request(None, OP_GETMETHODS, name) local_class = create_class(self, name, self._overrides.get(name, {}), self._getattr_overrides.get(name, {}), self._setattr_overrides.get(name, {}), remote_methods) self._proxied_classes[name] = local_class return local_class def can_pickle(self, obj): return getattr(obj, "___connection___", None) == self def pickle_object(self, obj): # This function is called to pickle obj to be transferred back to the # server. In this direction, we only allow objects that already exist # on the remote side so if this is not a stub, we do not allow it to be # transferred if getattr(obj, "___connection___", None) == self: # This is something we can transfer over return ObjReference(VALUE_LOCAL, obj.___remote_class_name___, obj.___identifier___) raise ValueError( "Cannot send object of type %s from client to server" % type(obj)) def unpickle_object(self, obj): # This function is called when the server sends a remote reference. # We create a local stub for it locally if (not isinstance(obj, ObjReference)) or obj.value_type != VALUE_REMOTE: raise ValueError("Invalid transferred object: %s" % str(obj)) remote_class_name = obj.class_name obj_id = obj.identifier local_instance = self._proxied_objects.get(obj_id) if not local_instance: local_class = self.get_local_class(remote_class_name, obj_id) local_instance = local_class(self, remote_class_name, obj_id) return local_instance def _get_canonical_name(self, name): # We look at the aliases looking for the most specific match first base_name = self._aliases.get(name) if base_name is not None: return base_name for idx in reversed( [pos for pos, char in enumerate(name) if char == '.']): base_name = self._aliases.get(name[:idx]) if base_name is not None: return ".".join([base_name, name[idx + 1:]]) return name def _communicate(self, msg): self._channel.send(msg) response_ready = False while not response_ready: evt_list = self._poller.poll() for fd, _ in evt_list: if fd == self._channel.fileno(): # We deal with this last as this basically gives us the # response so we stop looking at things on stdout/stderr response_ready = True # Readlines will never block here because bufsize is set to 1 # (line buffering) elif fd == self._server_process.stdout.fileno(): sys.stdout.write(self._server_process.stdout.readline()) elif fd == self._server_process.stderr.fileno(): sys.stderr.write(self._server_process.stderr.readline()) # We make sure there is nothing left to read. On the server side, a # flush happens before we respond so we read until we get an exception; # this is non blocking while True: try: line = self._server_process.stdout.readline() if not line: break sys.stdout.write(line) except (OSError, TypeError): break while True: try: line = self._server_process.stderr.readline() if not line: break sys.stderr.write(line) except (OSError, TypeError): break sys.stdout.flush() sys.stderr.flush() return self._channel.recv()
class VLC(object): def __init__(self, host="127.0.0.1", port=8080): # connection attrs self.host = host self.port = port # private playlist var, stores list of file paths # mirrors the list in the player at all times self._playlist = [] # used to determine if a self.supported = ["mp4", "avi", "mkv", "flv", ".aac", "3gp"] # add more later # creating an instance of the vlc window # local socket connection to the vlc player @property def playlist(self): '''returns list of file paths''' return self._playlist @property def connection_open(self): try: resp = requests.get("http://%s:%s/requests/status.json" % (self.host, self.port), auth=("", "1234")) if resp.status_code == 200: return True else: return False except Exception as e: return False @playlist.setter def playlist(self, arg): """Takes a string, tuple or a list as an argument and updates the player's playlist and the local_playlist variable enqueues the vlc object with a playlist of all the files stored in it can only add files to the playlist""" if isinstance(arg, (list, tuple)): for path in arg: self.check_path(path) if not path in self._playlist: data = self._enqueue(path) elif isinstance(arg, str): self.check_path(arg) if not arg in self._playlist: data = self._enqueue(arg) @playlist.deleter def playlist(self): '''clears the local playlist var and the remote one''' self._playlist = [] self.clear() @property def current(self): resp = requests.get("http://%s:%s/requests/status.json" % (self.host, self.port), auth=("", "1234")) js = json.loads(resp.content) return js["information"]["category"]["meta"]["filename"] @property def player_status(self): resp = requests.get("http://%s:%s/requests/status.json" % (self.host, self.port), auth=("", "1234")) js = json.loads(resp.content) return js["state"] def create_player(self): if self.connection_open: return self.conn = Popen(["vlc", "-I", "http"], stdout=PIPE, stdin=PIPE, stderr=PIPE) def check_path(self, path): '''Ensures all files added to the application are valid paths.''' if not os.path.isfile(path): print "file exception %s" % path raise FileNotFoundException() path, file = os.path.split(path) name, ext = file.split(".") if ext not in self.supported: print " supported exception %s" % path raise UnsupportedFileTypeException() def execute(self, cmd): '''Prepare a command and send it to VLC''' request_string = \ "http://%s:%s/requests/status.xml?command=%s" % ( self.host, self.port, cmd) resp = requests.get(request_string, auth=("", "1234")) def toggle_fullscreen(self): '''puts the windows in full screen''' resp = requests.get("http://%s:%s/requests/status.json" % (self.host, self.port), auth=("", "1234")) js = json.loads(resp.content) if js["fullscreen"] == 0: print "executed fullscreen" self.execute('fullscreen') def toggle_loop(self): '''makes the vlc player loop the current playlist''' resp = requests.get("http://%s:%s/requests/status.json" % (self.host, self.port), auth=("", "1234")) js = json.loads(resp.content) if not js["loop"]: self.execute("pl_loop") def pause(self): """Checks the current state to make sure the player is playing something""" self.execute('pl_pause') def play(self): """First checks if a valid file is currently loaded.""" self.execute('pl_play') time.sleep(5) self.toggle_fullscreen() self.toggle_loop() def stop(self): """checks first if there is something to stop""" self.execute('pl_stop') def _enqueue(self, path): '''adds a file to the playlist''' data = self.execute('in_enqueue&input=%s' % (os.path.abspath(path), )) self._playlist.append(path) def clear(self): '''clears all files from the playlist''' self.execute('pl_empty') self._playlist = [] def shutdown(self): self.stop() self.conn.kill()
class Proxy(object): """Create a proxy object as intermediary between client code and remote functionality. This class is a context manager, so when used in a ``with`` statement, it ensures the remote proxy server is stopped and disposed correctly. However, if the proxy server is left open, it can be re-used for a follow-up connection, saving start up time. Parameters ---------- package : string, optional The base package for the requested functionality. Default is `None`, in which case a full path to function calls should be provided. python : string, optional The python executable that should be used to execute the code. Default is ``'pythonw'``. url : string, optional The server address. Default is ``'http://127.0.0.1'``. port : int, optional The port number on the remote server. Default is ``1753``. Notes ----- If the server is your *localhost*, which will often be the case, it is better to specify the address explicitly (``'http://127.0.0.1'``) because resolving *localhost* takes a surprisingly significant amount of time. The service will make the correct (version of the requested) functionality available even if that functionality is part of a virtual environment. This is because it will use the specific python interpreter for which the functionality is installed to start the server. If possible, the proxy will try to reconnect to an already existing service Examples -------- Minimal example showing connection to the proxy server, and ensuring the server is disposed after using it: .. code-block:: python from compas.rpc import Proxy with Proxy('compas.numerical') as numerical: pass """ def __init__(self, package=None, python=None, url='http://127.0.0.1', port=1753, service=None): self._package = None self._python = compas._os.select_python(python) self._url = url self._port = port self._service = None self._process = None self._function = None self._profile = None self.service = service self.package = package self._implicitely_started_server = False self._server = self._try_reconnect() if self._server is None: self._server = self.start_server() self._implicitely_started_server = True def __enter__(self): return self def __exit__(self, *args): # If we started the RPC server, we will try to clean up and stop it # otherwise we just disconnect from it if self._implicitely_started_server: self.stop_server() else: self._server.__close() @property def address(self): return "{}:{}".format(self._url, self._port) @property def profile(self): """A profile of the executed code.""" return self._profile @profile.setter def profile(self, profile): self._profile = profile @property def package(self): """The base package from which functionality will be called.""" return self._package @package.setter def package(self, package): self._package = package @property def service(self): return self._service @service.setter def service(self, service): if not service: self._service = 'compas.rpc.services.default' else: self._service = service @property def python(self): return self._python @python.setter def python(self, python): self._python = python def _try_reconnect(self): """Try and reconnect to an existing proxy server. Returns ------- ServerProxy Instance of the proxy if reconnection succeeded, otherwise ``None``. """ server = ServerProxy(self.address) try: server.ping() except Exception: return None else: print("Reconnecting to an existing server proxy.") return server def start_server(self): """Start the remote server. Returns ------- ServerProxy Instance of the proxy, if the connection was successful. Raises ------ RPCServerError If the server providing the requested service cannot be reached after 100 contact attempts (*pings*). Examples -------- >>> p = Proxy() >>> p.stop_server() >>> p.start_server() """ env = compas._os.prepare_environment() # this part starts the server side of the RPC setup # it basically launches a subprocess # to start the default service # the default service creates a server # and registers a dispatcher for custom functionality try: Popen except NameError: self._process = Process() for name in env: if self._process.StartInfo.EnvironmentVariables.ContainsKey( name): self._process.StartInfo.EnvironmentVariables[name] = env[ name] else: self._process.StartInfo.EnvironmentVariables.Add( name, env[name]) self._process.StartInfo.UseShellExecute = False self._process.StartInfo.RedirectStandardOutput = True self._process.StartInfo.RedirectStandardError = True self._process.StartInfo.FileName = self.python self._process.StartInfo.Arguments = '-m {0} {1}'.format( self.service, str(self._port)) self._process.Start() else: args = [self.python, '-m', self.service, str(self._port)] self._process = Popen(args, stdout=PIPE, stderr=PIPE, env=env) # this starts the client side # it creates a proxy for the server # and tries to connect the proxy to the actual server server = ServerProxy(self.address) print("Starting a new proxy server...") success = False count = 100 while count: try: server.ping() except Exception: time.sleep(0.1) count -= 1 print(" {} attempts left.".format(count)) else: success = True break if not success: raise RPCServerError("The server is not available.") else: print("New proxy server started.") return server def stop_server(self): """Stop the remote server and terminate/kill the python process that was used to start it. Examples -------- >>> p = Proxy() >>> p.stop_server() >>> p.start_server() """ print("Stopping the server proxy.") try: self._server.remote_shutdown() except Exception: pass self._terminate_process() def _terminate_process(self): """Attempts to terminate the python process hosting the proxy server. The process reference might not be present, e.g. in the case of reusing an existing connection. In that case, this is a no-op. """ if not self._process: return try: self._process.terminate() except Exception: pass try: self._process.kill() except Exception: pass def __getattr__(self, name): if self.package: name = "{}.{}".format(self.package, name) try: self._function = getattr(self._server, name) except Exception: raise RPCServerError() return self._proxy def _proxy(self, *args, **kwargs): """Callable replacement for the requested functionality. Parameters ---------- args : list Positional arguments to be passed to the remote function. kwargs : dict Named arguments to be passed to the remote function. Returns ------- object The result returned by the remote function. Warning ------- The `args` and `kwargs` have to be JSON-serialisable. This means that, currently, only native Python objects are supported. The returned results will also always be in the form of built-in Python objects. """ idict = {'args': args, 'kwargs': kwargs} istring = json.dumps(idict, cls=DataEncoder) # it makes sense that there is a broken pipe error # because the process is not the one receiving the feedback # when there is a print statement on the server side # this counts as output # it should be sent as part of RPC communication try: ostring = self._function(istring) except Exception: # not clear what the point of this is # self.stop_server() # if this goes wrong, it means a Fault error was generated by the server # no need to stop the server for this raise if not ostring: raise RPCServerError("No output was generated.") result = json.loads(ostring) if result['error']: raise RPCServerError(result['error']) self.profile = result['profile'] return result['data']