class AirpnpProcess(object):
def __init__(self, config={}):
self.config = config
def create_config(self, config):
f = tempfile.NamedTemporaryFile(delete=True)
f.write("[airpnp]\n")
for k, v in config.items():
f.write("%s=%s\n" % (k, v))
f.flush()
return f.name
def __enter__(self):
configfn = self.create_config(self.config)
args = ["twistd", "-n", "airpnp", "-c", configfn]
cwd = os.path.join(os.path.dirname(__file__), "..")
self.proc = Popen(args, stdout=PIPE, stderr=STDOUT, cwd=cwd, close_fds=ON_POSIX, bufsize=1)
q = Queue()
t = Thread(target=enqueue_output, args=(self.proc.stdout, q))
t.daemon = True
t.start()
return q
def __exit__(self, type, value, tb):
self.proc.kill()
class DownloadThread(threading.Thread):
def __init__(self, title, url):
threading.Thread.__init__(self)
self.progress = 'Preparing download...'
self.updated = True
self.killed = False
# create the youtube-dl subprocess
file = title + '.%(ext)s'
output = '--output=' + file
max_quality = '--max-quality=35'
command = ['youtube-dl', '--no-part', '--continue', max_quality, output, url]
self.download_process = Popen(command, stdout=PIPE, universal_newlines=True)
def kill(self):
self.download_process.kill()
self.killed = True
def run(self):
while self.progress != '':
self.progress = self.download_process.stdout.readline()
self.updated = True
if self.killed:
self.progress = 'Aborted downloading'
else:
self.progress = 'Finished downloading'
def test_nanny_worker_ports(loop):
try:
worker = Popen(['dworker', '127.0.0.1:8989', '--host', '127.0.0.1',
'--worker-port', '8788', '--nanny-port', '8789'],
stdout=PIPE, stderr=PIPE)
sched = Popen(['dscheduler', '--port', '8989'], stdout=PIPE, stderr=PIPE)
with Executor('127.0.0.1:8989', loop=loop) as e:
start = time()
while True:
d = sync(e.loop, e.scheduler.identity)
if d['workers']:
break
else:
assert time() - start < 5
sleep(0.1)
assert d['workers']['127.0.0.1:8788']['services']['nanny'] == 8789
finally:
with ignoring(Exception):
w = rpc('127.0.0.1:8789')
sync(loop, w.terminate)
with ignoring(Exception):
os.kill(sched.pid, signal.SIGINT)
with ignoring(Exception):
worker.kill()
def process(self, request, response, report):
if response.is_html:
referrer = str(request)
pth = os.path.dirname(os.path.realpath(__file__))
proc = Popen([self.phantomJsPath, '{0}{1}scrape.js'.format(pth, os.sep), referrer], stdin=PIPE, stdout=PIPE, stderr=PIPE)
try:
out, err = proc.communicate(response.content.encode(), timeout=10)
rc = proc.returncode
except TimeoutExpired:
proc.kill()
out, err = proc.communicate()
if len(err) > 0:
report.add_error(err.decode('utf-8'))
else:
report.add_error(out.decode('utf-8'))
else:
if rc == 0:
result = out.decode('utf-8').strip()
urls = set()
for url in set(result.splitlines()):
if self._log_url(url):
urls.add(url)
self._add_request(url, referrer)
self._report(report, urls)
else:
if len(err) > 0:
report.add_error(err.decode('utf-8'))
else:
report.add_error(out.decode('utf-8'))
class OpenPostgreSQLsshTunnel(object):
""" Class to let us open an ssh tunnel, then close it when done """
def __init__(self, port=5432):
self.tunnel_process = 0
self.postgre_port = 5432
self.remote_port = port
def __enter__(self):
if HOSTNAME != 'dilepton-tower':
import time
import shlex
self.postgre_port = self.remote_port
_cmd = 'ssh -N -L localhost:%d' % self.remote_port + \
':localhost:5432 [email protected]'
args = shlex.split(_cmd)
self.tunnel_process = Popen(args, shell=False)
time.sleep(5)
return self.postgre_port
def __exit__(self, exc_type, exc_value, traceback):
if self.tunnel_process:
self.tunnel_process.kill()
if exc_type or exc_value or traceback:
return False
else:
return True
class MongodbPlugin(object):
def __init__(self):
self.mongo = None
self.tmpdir = tempfile.mkdtemp()
def pytest_sessionstart(self, session):
port = session.config.getvalue('mongodb_port')
self.mongo = Popen(["mongod", "--dbpath", self.tmpdir,
"--port", str(port)],
stdin=PIPE, stdout=PIPE, stderr=PIPE)
for each in range(10):
if 'waiting for connections' in self.mongo.stdout.readline():
break
else:
raise OSError('Mongodb start timeout.')
def pytest_sessionfinish(self, session):
if self.mongo is not None:
try:
self.mongo.kill()
self.mongo.communicate()
self.mongo.wait()
finally:
shutil.rmtree(self.tmpdir)
def load_shell(self, chunk):
"""Run shell commands from code chunks"""
if chunk['evaluate']:
lines = chunk['content'].lstrip().splitlines()
result = "\n"
for line in lines:
command = line.split()
major, minor = sys.version_info[:2]
cmd = Popen(command, stdout=PIPE)
if major == 2 or minor < 3: # Python 2 doesn't have timeout for subprocess
try:
content = cmd.communicate()[0].decode('utf-8').replace("\r", "") + "\n"
except Exception as e:
content = "Pweave ERROR can't execute shell command:\n %s\n" % command
content += str(e)
sys.stdout.write(" Pweave ERROR can't execute shell command:\n %s\n" % line)
print(str(e))
else:
try:
content = cmd.communicate(timeout=20)[0].decode('utf-8').replace("\r", "") + "\n"
except TimeoutExpired:
cmd.kill()
content, errs = cmd.communicate()
sys.stdout.write("Shell command timeout:\n %s\n" % line)
content = content.decode('utf-8').replace("\r", "") + "\n"
if chunk['term']:
result += "$ %s\n" % line
result += content
else:
result = ""
return result
def newjob():
choiceddevices = dict(request.form).get('choicedDevice')
jobname = request.form.get('jobName')
testtype = request.form.get('testType')
monkeyconfig = {}
monkeyconfig["actioncount"] = request.form.get("actioncount")
monkeyconfig["actiondelay"] = request.form.get("actiondelay")
monkeyconfig["touchpercent"] = request.form.get("touchpercent")
monkeyconfig["motionpercent"] = request.form.get("motionpercent")
monkeyconfig["pinchzoompercent"] = request.form.get("pinchzoompercent")
monkeyconfig["majornavpercent"] = request.form.get("majornavpercent")
monkeyconfig["syskeyspercent"] = request.form.get("syskeyspercent")
monkeyconfig["appswitchpercent"] = request.form.get("appswitchpercent")
order = request.args.get("order").split("|")
f = request.files['file']
fname = secure_filename(f.filename)
apk = os.path.join(Config.UPLOAD_FOLDER,fname)
if not os.path.isdir(Config.UPLOAD_FOLDER):
os.makedirs(Config.UPLOAD_FOLDER)
f.save(apk)
cmd_activity = "aapt d badging %s|%s launchable-activity" %(apk,"findstr" if system == "Windows" else "grep")
cmd_package = "aapt d badging %s|%s package" %(apk,"findstr" if system == "Windows" else "grep")
activity = Popen(cmd_activity,stdout=PIPE,shell=True)
package = Popen(cmd_package,stdout=PIPE,shell=True)
main_activity = activity.stdout.read().decode().split("name='")[1].split("'")[0]
packageName = package.stdout.read().decode().split("name='")[1].split("'")[0]
activity.kill()
package.kill()
testjob = Testjob(jobname,testtype,choiceddevices,apk,packageName,main_activity,order,monkeyconfig)
db.session.add(testjob)
db.session.commit()
return redirect(url_for(".jobs"))
class Player:
def __init__(self, name, cmd):
self.name = name
self.forts = set()
self.marches = set()
args = cmd.split(' ')
self.process = Popen(list(args) + [name], stdin=PIPE, stdout=PIPE,
universal_newlines=True)
def capture(self, fort):
if fort.owner:
fort.owner.forts.remove(fort)
self.forts.add(fort)
fort.owner = self
def is_defeated(self):
return (not self.forts) and (not self.marches)
def send_state(self):
if not self.process.poll():
self.process.stdin.write(show_visible(self.forts))
self.process.stdin.write('\n')
self.process.stdin.flush()
def read_commands(self, game):
if not self.process.poll():
try:
read_commands(game, self, self.process.stdout)
except StopIteration:
self.process.kill()
def run(self):
print "Starting asynchrone Thread with Command " + self.cmd
p = Popen([self.cmd], stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=False)
if bDebug:
print getTime() + ": PID of Thread = " + str(p.pid)
# wrap p.stdout with a NonBlockingStreamReader object:
nbsr = NonBlockingStreamReader(p.stdout)
# get the output
strTemp = ''
while self.isrunning:
output = nbsr.readline(0.1)
# 0.1 secs to let the shell output the result
if not output:
if strTemp != '':
# print strTemp
try:
self.connstream.write(strTemp)
except:
if bDebug:
print getTime() + ": Connection was destroyed by Client!"
print getTime() + ": PID = " + str(os.getpid())
print getTime() + ": Group PID = " + str(os.getpgid(os.getpid()))
# p.kill() stops subprocess
p.kill()
self.isrunning = False
break
strTemp = ''
continue
else:
strTemp = strTemp + output
nbsr.stop()
return
class MockServer(object):
def __init__(self, http_port=None, https_port=None, proxy_port=None):
self.http_port = http_port if http_port is not None else get_ephemeral_port()
self.https_port = https_port if https_port is not None else get_ephemeral_port()
self.proxy_port = proxy_port if proxy_port is not None else get_ephemeral_port()
def __enter__(self):
self.proc = Popen([
sys.executable,
'-u', '-m', 'splash.tests.mockserver',
'--http-port', str(self.http_port),
'--https-port', str(self.https_port),
'--proxy-port', str(self.proxy_port),
],
env=get_testenv()
)
for port in (self.http_port, self.https_port, self.proxy_port):
_wait_for_port(port)
return self
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()
self.proc.wait()
def url(self, path, gzip=True, host='localhost'):
gzip_path = '' if not gzip else '/gzip'
return "http://%s:%s%s/%s" % (
host, self.http_port, gzip_path, path.lstrip('/')
)
def https_url(self, path):
return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))
def test_reload_wrapper_preservation(self):
# This test verifies that when `python -m tornado.autoreload`
# is used on an application that also has an internal
# autoreload, the reload wrapper is preserved on restart.
main = """\
import os
import sys
# This import will fail if path is not set up correctly
import testapp
if 'tornado.autoreload' not in sys.modules:
raise Exception('started without autoreload wrapper')
import tornado.autoreload
print('Starting')
sys.stdout.flush()
if 'TESTAPP_STARTED' not in os.environ:
os.environ['TESTAPP_STARTED'] = '1'
# Simulate an internal autoreload (one not caused
# by the wrapper).
tornado.autoreload._reload()
else:
# Exit directly so autoreload doesn't catch it.
os._exit(0)
"""
# Create temporary test application
path = mkdtemp()
os.mkdir(os.path.join(path, 'testapp'))
self.addCleanup(shutil.rmtree, path)
init_file = os.path.join(path, 'testapp', '__init__.py')
open(init_file, 'w').close()
main_file = os.path.join(path, 'testapp', '__main__.py')
with open(main_file, 'w') as f:
f.write(main)
# Make sure the tornado module under test is available to the test
# application
pythonpath = os.getcwd()
if 'PYTHONPATH' in os.environ:
pythonpath += os.pathsep + os.environ['PYTHONPATH']
autoreload_proc = Popen(
[sys.executable, '-m', 'tornado.autoreload', '-m', 'testapp'],
stdout=subprocess.PIPE, cwd=path,
env=dict(os.environ, PYTHONPATH=pythonpath),
universal_newlines=True)
for i in range(20):
if autoreload_proc.poll() is not None:
break
time.sleep(0.1)
else:
autoreload_proc.kill()
raise Exception("subprocess failed to terminate")
out = autoreload_proc.communicate()[0]
self.assertEqual(out, 'Starting\n' * 2)
def err_check(alpha,coord_vector,slope,space_type): #Runs OPIUM, checks test config errors and returns their sum
if space_type == 'coeff':
test_vector = coord_vector.copy()
test_vector = test_vector - alpha*slope
grid = fit_to_func(test_vector,'fourier_cos')
setKB(grid)
elif space_type == 'valley':
test_vector = VtoC(coord_vector - alpha*slope)
grid = fit_to_func(test_vector,'fourier_cos')
setKB(grid)
else:
raise ValueError('space_type must be "coeff" or "valley"')
#Use threading to cancel OPIUM if it takes too long
proc = Popen('./opium ' + filename + ' ' + filename + '.log all rpt', shell=True)
proc_thread = threading.Thread(target=proc.communicate)
proc_thread.start()
proc_thread.join(timeout_sec)
if proc_thread.is_alive():
# Process still running - kill it and raise timeout error
try:
proc.kill()
except OSError, e:
# The process finished between the `is_alive()` and `kill()`
return proc.returncode
def _main():
c = synth.Context()
PrecomputeWaveTables(c).start()
midi_name = sys.argv[1]
file_name = sys.argv[2]
command = './identifykey.sh -f %s' % file_name
print(command)
p = Popen(command, shell=True, stdout=PIPE)
key = p.stdout.readline().strip()
print('Detected key: [%s]' % key)
t = None
if midi_name:
a = c.add(keep_alive = True)
c.start(a)
def on_note(note):
shifted = note.key_shift('C', key)
print('%s -> %s' % (note, shifted))
if shifted:
a.add_module(copy(play.beep(c, shifted.shift_octave(-1))))
t = play.MidiListener(on_note, name=midi_name)
t.start()
else:
c.start(scale(c, key))
m = None
if file_name:
command = 'aplay %s' % (file_name,)
print(command)
m = Popen(command.split(), stdin=PIPE)
sys.stdin.readline()
if t: t.stop()
c.stop()
if m: m.kill()
def execute_test():
retvalue = -1
envvars = os.environ.copy()
if platform.system() == "Linux":
libdir = envvars.get("CMAKE_LIBRARY_DIRECTORIES")
if libdir != None:
envvars["LD_LIBRARY_PATH"] = envvars["NDDSHOME"] + "/lib/" + example + ":" + libdir.replace(";", ":")
else:
envvars["LD_LIBRARY_PATH"] = envvars["NDDSHOME"] + "/lib/" + example + ":" + project_source_dir + "/lib"
# Run server
server = Popen([output_dir + "/bin/" + example + "/" + test_name + "ServerExample"], env=envvars)
# Wait 5 seconds before run client
sleep(5)
client = Popen([output_dir + "/bin/" + example + "/" + test_name + "ClientExample"], env=envvars)
client.wait()
client.communicate()
retvalue = client.returncode
server.kill()
return retvalue
class MockServer(object):
def __init__(self, http_port=None, https_port=None, proxy_port=8990):
self.http_port = http_port if http_port is not None else _ephemeral_port()
self.https_port = https_port if https_port is not None else _ephemeral_port()
self.proxy_port = proxy_port if proxy_port is not None else _ephemeral_port()
def __enter__(self):
self.proc = Popen([
sys.executable,
'-u', '-m', 'splash.tests.mockserver',
'--http-port', str(self.http_port),
'--https-port', str(self.https_port),
'--proxy-port', str(self.proxy_port),
],
stdout=PIPE,
env=get_testenv()
)
for port in (self.http_port, self.https_port, self.proxy_port):
_wait_for_port(port)
print(_non_block_read(self.proc.stdout))
def __exit__(self, exc_type, exc_value, traceback):
self.proc.kill()
self.proc.wait()
time.sleep(0.2)
def url(self, path):
return "http://localhost:%s/%s" % (self.http_port, path.lstrip('/'))
def https_url(self, path):
return "https://localhost:%s/%s" % (self.https_port, path.lstrip('/'))
def call_command(*args):
try:
formatted_command = command.format(*args)
except IndexError:
raise WrongArgumentError(
"'{0}' not enough arguments. Called with {1}".format(command,
args))
process = Popen(formatted_command, stdout=PIPE, stderr=PIPE, shell=True,
universal_newlines=True)
if timeout:
wait_time_remaining = timeout
while process.poll() is None and wait_time_remaining > 0:
time.sleep(wait_step)
wait_time_remaining -= wait_step
if wait_time_remaining <= 0:
process.kill()
raise OperatingSystemError(
"{0} have not completed in {1} seconds".format(
formatted_command, timeout))
(out, err) = process.communicate()
if err != '':
log = logging.getLogger('sh')
log.warn(
"'{0}' has written to stderr: {1}".format(formatted_command,
err))
return out.rstrip()
def async_read(self, process):
output = []
characters = []
pattern = '###################- 100.0% 0.0 kBps'
pattern_repeat = 0
# set non-blocking flag while preserving old flags
fl = fcntl(process.stdout, F_GETFL)
fcntl(process.stdout, F_SETFL, fl | os.O_NONBLOCK)
# read char until EOF hit
while True:
try:
ch = os.read(process.stdout.fileno(), 1)
# EOF
if not ch:
break
ch = ch.decode('UTF-8')
characters.append(ch)
# New line - check the pattern and add to output string
if ch == '\n':
string = "".join(characters)
if (string.find(pattern) != -1):
pattern_repeat += 1
if (pattern_repeat == 2):
# zsync is in loop (bug)
Popen.kill(process)
break
output.append(string)
characters = []
except OSError:
# waiting for data be available on process.stdout
pass
return "".join(output)
class CrawlTestCase(TestCase):
def setUp(self):
self.proc = Popen([sys.executable, '-u', '-m', 'scrapy.tests.mockserver'], stdout=PIPE)
self.proc.stdout.readline()
def tearDown(self):
self.proc.kill()
self.proc.wait()
time.sleep(0.2)
@defer.inlineCallbacks
def test_follow_all(self):
spider = FollowAllSpider()
yield docrawl(spider)
self.assertEqual(len(spider.urls_visited), 11) # 10 + start_url
@defer.inlineCallbacks
def test_delay(self):
spider = FollowAllSpider()
yield docrawl(spider, {"DOWNLOAD_DELAY": 0.3})
t = spider.times[0]
for t2 in spider.times[1:]:
self.assertTrue(t2-t > 0.15, "download delay too small: %s" % (t2-t))
t = t2
def sendSMS(self,mobileno,text):
if(self.notLogged):
print("you are not logged in - call logIn()")
return
if(self.captchaNeeded):
with open(self.captchaPath,"wb") as f:
f.write(self.opener.open(self.captchaUrl).read())
p = Popen(["display",self.captchaPath])
self.dataDict['textcode'] = input("Captcha ? ")
p.kill()
if(len(text) <= 140):
self.dataDict['mobNo'] = mobileno
self.dataDict['text'] = text
# print(self.postDataStr.format(**self.dataDict))
try:
h = self.opener.open(self.sendSMSUrl,self.postDataStr.format(**self.dataDict).encode())
resp = h.read()
try:
msg = BS(resp).find("div",attrs={"id":"quicksms"}).find("div",attrs={"class":"quickname"}).text.strip()
if msg.endswith("submitted successfully"): pass
else: print("N : "+msg)
except:
print("N");self.captchaNeeded = True
with open("successResp.html","wb") as f: f.write(resp)
except urllib.error.HTTPError as error:
pass
def run(self):
try:
if osflag:
proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\
stderr=STDOUT,bufsize=0)
else:
from subprocess import STARTUPINFO
si=STARTUPINFO()
si.dwFlags|=1
si.wShowWindow=0
proc=Popen(self.cmd,shell=False,stdin=None,stdout=PIPE,\
stderr=STDOUT,bufsize=0,startupinfo=si)
while 1:
if self.stop_flag:
if osflag: proc.send_signal(signal.SIGKILL)
else: proc.kill()
break
if osflag:
if proc.stdout in select.select([proc.stdout],[],[],1)[0]:
line=proc.stdout.readline()
else: line=' \n'
else: line=proc.stdout.readline()
if not len(line): break
else:
if count(line,'ttl') or count(line,'TTL'): self.retries=0
else: self.retries=self.retries+1
line=' '
sleep(0.5)
proc.poll()
except: pass
class Engine:
def __init__(self):
self._proc = Popen(['stockfish'], stdin=PIPE, stdout=PIPE, stderr=PIPE)
self._proc.stdin.write('setoption name Hash value 128\n')
self._proc.stdin.write('setoption name Threads value 4\n')
self._proc.stdin.write('setoption name Best Book Move value true\n')
self._proc.stdin.write('setoption name Aggressiveness value 200\n')
self._proc.stdin.write('setoption name Cowardice value 0\n')
self._proc.stdin.write('setoption name Contempt Factor value 50\n')
def update_pos(self, fen):
self._proc.stdin.write('position fen %s\n' % fen)
def get_next_move(self, fen=None, time_limit=50):
if fen is not None:
self.update_pos(fen)
self._proc.stdin.write('go movetime %d\n' % time_limit)
line = self._proc.stdout.readline()
while not line.startswith('bestmove'):
line = self._proc.stdout.readline()
print line
return line.split(' ')[1]
def close(self):
self._proc.kill()
def launch_spl(self):
need_launch_in_shell = (os.name == "nt")
# launch sploit proccess with team_ip as arg
spl = Popen([self.sploit_name, self.team_ip],
stdout=PIPE, stderr=STDOUT, bufsize=1,
shell=need_launch_in_shell)
q = Queue()
# we are processing output in other thread to prevent blocking
def enqueue_output(queue, out):
while True:
line = out.readline()
queue.put(line)
if not line:
break
t = Thread(target=enqueue_output, args=(q, spl.stdout))
t.daemon = True
t.start()
# get output by lines until EOF
while True:
try:
remaining_time = MAX_RUNTIME - (time() - self.last_launch_time)
line = q.get(timeout=remaining_time)
except (Empty, ValueError):
log("Killing %s sploit(tried to run for more than %d secs)" % (
self.team_name, MAX_RUNTIME))
break
if not line:
break
line = line.strip()
if not line:
continue
if self.cycle_num == 1:
print("%s: %s" % (self.team_name, line))
flags = re.findall(FLAG_FORMAT, line)
OWNER_LOCK.acquire()
for flag in flags:
if flag not in self.flags:
if self.cycle_num == 1:
log("Flag from %s: %s" % (self.team_name, flag))
with open(self.flag_filename, "ab", 0) as f:
f.write(flag + b"\n")
self.flags.add(flag)
else:
if self.cycle_num == 1:
log("Flag from %s: %s (dup)" % (self.team_name, flag))
OWNER_LOCK.release()
if os.name != "nt":
spl.kill()
else:
spl.communicate()
def getLines(self):
lines = [str(line) for line in vim.current.buffer]
(row, col) = vim.current.window.cursor
cur_line = lines[row-1]
cur_line = cur_line[:col] + self.base + "AUTO332" + cur_line[col:]
lines[row-1] = cur_line
input_buffer = "\n".join(lines)
# Send the buffer to hh_client and return the lines
args = [
Const.CLIENT,
r'--auto-complete'
]
proc = Popen(args, stdin=PIPE, stdout=PIPE, stderr=PIPE)
proc.stdin.write(input_buffer);
out = proc.communicate()[0]
base = self.base
try:
proc.kill();
except:
pass
proc = args = None
#Util.debugf('getLines.txt', locals())
return [x for x in out.split("\n") if len(x) > 0]
def tesseract_ocr_and_render_pdf(
input_files,
output_file,
log,
pdfinfo,
pdfinfo_lock):
input_image = next((ii for ii in input_files if ii.endswith('.png')), '')
input_pdf = next((ii for ii in input_files if ii.endswith('.pdf')))
if not input_image:
# Skipping this page
re_symlink(input_pdf, output_file)
return
args_tesseract = [
'tesseract',
'-l', '+'.join(options.language),
input_image,
os.path.splitext(output_file)[0], # Tesseract appends suffix
'pdf'
] + options.tesseract_config
p = Popen(args_tesseract, close_fds=True, stdout=PIPE, stderr=PIPE,
universal_newlines=True)
try:
stdout, stderr = p.communicate(timeout=options.tesseract_timeout)
if stdout:
log.info(stdout)
if stderr:
log.error(stderr)
except TimeoutExpired:
p.kill()
log.info("Tesseract - page timed out")
re_symlink(input_pdf, output_file)
class DisqueNode:
def __init__(self, port, dir):
self.port = port
self.dir = dir
self.proc = None
self.socket = os.path.join(dir, 'disque.sock')
def start(self):
if not self.proc:
cmd = ["disque-server",
"--port", str(self.port),
"--dir", self.dir,
"--unixsocket", self.socket,
"--unixsocketperm", "755"]
self.proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
cmd = ['disque', '-p', str(self.port), 'info']
while True:
sleep(.01)
if self.proc.poll():
raise Exception('already stopped!', self.proc.stderr)
resp = run(cmd, stdout=PIPE, stderr=PIPE)
if not resp.returncode:
break
def stop(self):
self.proc.kill()
self.proc = None
@property
def configuration(self):
return Configuration(port=self.port, dir=self.dir, socket=self.socket)
def wpa_run(self):
dump_cmd = ['airodump-ng', '-c', self.channel, '--bssid', self.bssid, '-w', './log/' + self.bssid, self.iface]
airodump_proc = Popen(dump_cmd, stdout=DN, stderr=DN)
self.proc_list.append(airodump_proc)
self.send_deauths()
while self.key == '':
output = Popen('tshark -r ./log/' + self.bssid + '-01.cap 2>/dev/null| grep "Message 4 of 4"',shell=True, stdout=PIPE).communicate()[0]
if output.find('Message 4 of 4') != -1:
execute('rm ./log/'+self.bssid+'.key')
airodump_proc.kill()
airodump_proc.communicate()
crack_cmd = ['aircrack-ng', '-w', self.password_list, '-b', self.bssid, './log/' + self.bssid + '-01.cap','-l', './log/' + self.bssid + '.key']
crack_proc = Popen(crack_cmd, stdout=DN)
self.proc_list.append(crack_proc)
crack_proc.wait()
try:
f = open('./log/' + self.bssid + '.key')
key = f.read()
f.close()
self.key = key
self.crack_success = True
self.stop()
except:
pass
else:
self.send_deauths()
time.sleep(5)
return self.key
def getDeviceState():
global deviceStatus
cmd = "adb devices"
devices = []
p = Popen(cmd,stdout=PIPE,shell=True)
for info in p.stdout.readlines():
info = info.decode()
if 'List' in info:
continue
elif 'offline' in info or 'unauthorized' in info or 'device' in info:
device = {}
name,state = [n.strip() for n in info.split('\t') if n.strip()]
device["deviceName"] = name
device["state"] = state
if ":" in name:
device["replacedName"] = name.replace(".","").replace(":","")
else:
device["replacedName"] = name
devices.append(device)
if name not in deviceStatus.keys():
deviceStatus[name] = False
else:
continue
p.kill()
return devices
def has_active_torrents():
""" Check for any active torrents on the local deluged server.
:return: True if one or more torrents are active
:rtype: Bool
"""
std_output, std_err_output = '', ''
proc = None
try:
proc = Popen(["deluge-console", "info"], stdout=PIPE, stderr=PIPE)
std_output, std_err_output = proc.communicate()
# Decode
std_output = std_output.decode("utf-8")
std_err_output = std_err_output.decode("utf-8")
except (OSError, ValueError):
if proc is not None:
proc.kill()
raise RuntimeError("deluge-console could not be found, " +
"make sure it is installed")
if ('fatal' in std_output) or (
'fatal' in std_err_output) or proc.returncode >= 1:
raise RuntimeError("Status check failed, " +
"make sure that deluged is running")
else:
# Success!
return std_output.strip() != ""
def runLocalhostBenchmark():
serverName = 'async_server'
serverPort = '22380'
clientName = 'bench_client'
hostName = 'localhost'
localhostCount = count / 50
p1 = Popen(['./'+serverName, serverPort], stdout=PIPE, stderr=PIPE, env=env)
then = time.time()
p2 = Popen(['./'+clientName, hostName+':'+serverPort, str(localhostCount)], stdout=PIPE, stderr=PIPE, env=env)
out2, err2 = p2.communicate()
t = time.time() - then
p1.kill()
out1, err1 = p1.communicate()
if p2.returncode != 0:
print '%s error. returncode: %d' % (executableName, p2.returncode)
print clientName, 'stdout:'
print out2
print clientName, 'stderr:'
print err2
print serverName, 'stdout:'
print out1
print serverName, 'stderr:'
print err1
sys.exit(1)
print('%s: Made %s AMP calls (back-to-back in series - over '
'localhost TCP socket) in %.4f seconds (%s calls/s)' %
(clientName, locale.format("%d", localhostCount, grouping=True),
t, locale.format("%d", int(localhostCount/t), grouping=True)))
class scp_proxy:
def __init__(self,userhost,verbose=0):
"""Opens a connection to the remote host and establishes the remote client. userhost should be of the form "user@host"""
self.verbose=verbose
# Open the remote SSH process
try :
self.ssh=Popen(("ssh",userhost,"e2scp.py --client"),stdin=PIPE,stdout=PIPE)
self.stdout=self.ssh.stdout # read from this
self.stdin=self.ssh.stdin # write to this
except:
print("ssh to remote machine failed : ",("ssh",host,"e2ssh.py --client"))
traceback.print_exc()
sys.exit(2)
while 1:
ln=self.stdout.readline().strip()
if len(ln)==0 :
print("Error running e2scp.py on the remote machine. EMAN2 installed ?")
sys.exit(3)
if ln=="HELO" :
if self.verbose : print("Connection established")
break
if self.verbose >1 : print("*** ",ln)
atexit.register(self.close)
def close(self):
"""Close the connection"""
if self.ssh!=None :
try:
self.stdin.write("exit\n")
# self.stdin.flush()
self.ssh.kill()
except: pass
self.ssh=None
def mkdir(self,path):
"""Create a path on the remote host, at all necessary levels"""
self.stdin.write("mkdir\n%s\n"%path)
self.stdin.flush()
r=self.stdout.readline().strip()
if r!="OK" : raise Exception("Error in creating remote path (%s)"%(r))
def listrecurse(self,path,basepath=""):
"""Recursively list the contents of a remote path, may be a directory or a BDB specifier. If specified
will reference paths with respect to basepath."""
self.stdin.write("listrecurse\n%s\n%s\n"%(path,basepath))
r=int(self.stdout.readline().strip())
ret=[]
for i in xrange(r):
ret.append(self.stdout.readline().strip())
return ret
def getheader(self,path,n):
"""Return the header of a remote image as a dictionary"""
self.stdin.write("getheader\n%s\n%d\n"%(path,n))
self.stdin.flush()
return read_obj(self.stdout,img)
def getimage(self,path,n):
"""Return a single reomote EMData object from a file"""
self.stdin.write("getimage\n%s\n%d\n"%(path,n))
self.stdin.flush()
return read_obj(self.stdout,img)
def putimage(self,path,n,img):
"""Write a single EMData object into a remote file"""
self.stdin.write("putimage\n%s\n%d\n"%(path,n))
self.write_obj(self.stdin,img)
self.stdin.flush()
def getfile(self,remotepath,localpath):
"""Retrieve a single binary remote file and write to path. Streams for minimal memory usage."""
self.stdin.write("getfile\n%s\n"%(remotepath))
self.stdin.flush()
recv_file(self.stdout,localpath)
def putfile(self,localpath,remotepath):
self.stdin.write("putfile\n%s\n"%(remotepath))
self.stdin.flush()
send_file(self.stdin,localpath)
def getbdb(self,remotepath,localpath):
"""Copyies a remote BDB to a local machine"""
self.stdin.write("getbdb\n%s\n"%(remotepath))
self.stdin.flush()
recv_bdb(self.stdout,localpath)
def putbdb(self,localpath,remotepath):
"""Copies a local BDB to a remote machine"""
self.stdin.write("putbdb\n%s\n"%(remotepath))
self.stdin.flush()
send_bdb(self.stdin,localpath)
class TestUDST(TestCase):
def setUp(self):
# Application, minimum, to define three trace points
app_text = b"""
#include <unistd.h>
#include <stdint.h>
#include <stdio.h>
#include <string.h>
#include "folly/tracing/StaticTracepoint.h"
int main() {
char s[100];
int i, a = 200, b = 40;
for (i = 0; i < 100; i++) s[i] = (i & 7) + (i & 6);
uint64_t j = 0;
char s1[64];
const char* str = "str";
size_t len = strlen(str);
while (1) {
FOLLY_SDT(test, probe_point_1, s[7], b);
FOLLY_SDT(test, probe_point_3, a, b);
FOLLY_SDT(test, probe_point_1, s[4], a);
FOLLY_SDT(test, probe_point_2, 5, s[10]);
FOLLY_SDT(test, probe_point_3, s[4], s[7]);
memset(&s1, '\0', sizeof(s1));
strncpy(s1, str, len);
snprintf(s1 + len, sizeof(s1) - len, "%d", j);
FOLLY_SDT(test, probe_point_4, j++, &s1);
memset(&s1, '\0', sizeof(s1));
strncpy(s1, str, len);
snprintf(s1 + len, sizeof(s1) - len, "%d", j);
FOLLY_SDT(test, probe_point_5, &s1, j++);
sleep(1);
}
return 1;
}
"""
# BPF program
self.bpf_text = """
#include <linux/blkdev.h>
#include <uapi/linux/ptrace.h>
struct probe_result_t1 {
char v1;
int v2;
};
struct probe_result_t2 {
int v1;
char v2;
};
struct probe_result_t3 {
int v1;
int v2;
};
struct probe_result_t4 {
u64 v1;
char v2[8];
};
struct probe_result_t5 {
char v1[8];
u64 v2;
};
BPF_PERF_OUTPUT(event1);
BPF_PERF_OUTPUT(event2);
BPF_PERF_OUTPUT(event3);
BPF_PERF_OUTPUT(event4);
BPF_PERF_OUTPUT(event5);
int do_trace1(struct pt_regs *ctx) {
struct probe_result_t1 result = {};
bpf_usdt_readarg(1, ctx, &result.v1);
bpf_usdt_readarg(2, ctx, &result.v2);
event1.perf_submit(ctx, &result, sizeof(result));
return 0;
};
int do_trace2(struct pt_regs *ctx) {
struct probe_result_t2 result = {};
bpf_usdt_readarg(1, ctx, &result.v1);
bpf_usdt_readarg(2, ctx, &result.v2);
event2.perf_submit(ctx, &result, sizeof(result));
return 0;
}
int do_trace3(struct pt_regs *ctx) {
struct probe_result_t3 result = {};
bpf_usdt_readarg(1, ctx, &result.v1);
bpf_usdt_readarg(2, ctx, &result.v2);
event3.perf_submit(ctx, &result, sizeof(result));
return 0;
}
int do_trace4(struct pt_regs *ctx) {
struct probe_result_t4 result = {};
bpf_usdt_readarg(1, ctx, &result.v1);
bpf_usdt_readarg_p(2, ctx, &result.v2, sizeof(result.v2));
event4.perf_submit(ctx, &result, sizeof(result));
return 0;
}
int do_trace5(struct pt_regs *ctx) {
struct probe_result_t5 result = {};
bpf_usdt_readarg_p(1, ctx, &result.v1, sizeof(result.v1));
bpf_usdt_readarg(2, ctx, &result.v2);
event5.perf_submit(ctx, &result, sizeof(result));
return 0;
}
"""
# Compile and run the application
self.ftemp = NamedTemporaryFile(delete=False)
self.ftemp.close()
comp = Popen([
"gcc", "-I",
"%s/include" % os.path.dirname(
os.path.abspath(inspect.getfile(inspect.currentframe()))),
"-x", "c", "-o", self.ftemp.name, "-"
],
stdin=PIPE)
comp.stdin.write(app_text)
comp.stdin.close()
self.assertEqual(comp.wait(), 0)
self.app = Popen([self.ftemp.name])
def test_attach1(self):
# enable USDT probe from given PID and verifier generated BPF programs
u = USDT(pid=int(self.app.pid))
u.enable_probe(probe="probe_point_1", fn_name="do_trace1")
u.enable_probe(probe="probe_point_2", fn_name="do_trace2")
u.enable_probe(probe="probe_point_3", fn_name="do_trace3")
u.enable_probe(probe="probe_point_4", fn_name="do_trace4")
u.enable_probe(probe="probe_point_5", fn_name="do_trace5")
b = BPF(text=self.bpf_text, usdt_contexts=[u], debug=4)
# Event states for each event:
# 0 - probe not caught, 1 - probe caught with correct value,
# 2 - probe caught with incorrect value
self.evt_st_1 = 0
self.evt_st_2 = 0
self.evt_st_3 = 0
# define output data structure in Python
class Data1(ct.Structure):
_fields_ = [("v1", ct.c_char), ("v2", ct.c_int)]
class Data2(ct.Structure):
_fields_ = [("v1", ct.c_int), ("v2", ct.c_char)]
class Data3(ct.Structure):
_fields_ = [("v1", ct.c_int), ("v2", ct.c_int)]
class Data4(ct.Structure):
_fields_ = [("v1", ct.c_ulonglong), ("v2", ct.c_char * 64)]
class Data5(ct.Structure):
_fields_ = [("v1", ct.c_char * 64), ("v2", ct.c_ulonglong)]
def check_event_val(event, event_state, v1, v2, v3, v4):
if ((event.v1 == v1 and event.v2 == v2)
or (event.v1 == v3 and event.v2 == v4)):
if (event_state == 0 or event_state == 1):
return 1
return 2
def print_event1(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data1)).contents
self.evt_st_1 = check_event_val(event, self.evt_st_1, b'\x0d', 40,
b'\x08', 200)
def print_event2(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data2)).contents
# pretend we have two identical probe points to simplify the code
self.evt_st_2 = check_event_val(event, self.evt_st_2, 5, b'\x04',
5, b'\x04')
def print_event3(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data3)).contents
self.evt_st_3 = check_event_val(event, self.evt_st_3, 200, 40, 8,
13)
def print_event4(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data4)).contents
print("%s" % event.v2)
def print_event5(cpu, data, size):
event = ct.cast(data, ct.POINTER(Data5)).contents
print("%s" % event.v1)
# loop with callback to print_event
b["event1"].open_perf_buffer(print_event1)
b["event2"].open_perf_buffer(print_event2)
b["event3"].open_perf_buffer(print_event3)
b["event4"].open_perf_buffer(print_event4)
b["event5"].open_perf_buffer(print_event5)
# three iterations to make sure we get some probes and have time to process them
for i in range(3):
b.perf_buffer_poll()
self.assertTrue(self.evt_st_1 == 1 and self.evt_st_2 == 1
and self.evt_st_3 == 1)
def tearDown(self):
# kill the subprocess, clean the environment
self.app.kill()
self.app.wait()
os.unlink(self.ftemp.name)
def __load_tar(filename,
mode,
allowTmpFile,
transformDataRawData,
tarMember,
extractAll,
eraseTmpTarMembers,
ignore_zeros,
logger=None):
"""
Internal method for reading tarfiles
"""
#useSubprocess = False
useSubprocess = True
if tarMember is None:
f = tarfile.open(filename, mode, ignore_zeros=ignore_zeros)
if not extractAll:
if logger:
logger.info("Retrieving tar file members (%s)...",
"full" if ignore_zeros else "fast")
memberList = f.getmembers()
elif type(tarMember) in (tarfile.TarInfo, str):
useSubprocess = True
memberList = [tarMember]
else:
raise TypeError("tarMember argument must be TarInfo or None.")
for entry in memberList if not extractAll else [None]:
if allowTmpFile:
tmpFolderPath = tempfile.mkdtemp()
if useSubprocess:
from subprocess import Popen, PIPE, CalledProcessError
from RingerCore import is_tool
tar_cmd = 'gtar' if is_tool('gtar') else 'tar'
# TODO This will crash if someday someone uses a member in file that is
# not in root path at the tarfile.
if extractAll:
start = time()
logger.info("Proceeding to untar all members.")
process_args = (
tar_cmd,
'--verbose',
'-xvzif',
filename,
)
untar_ps = Popen(process_args,
stdout=PIPE,
bufsize=1,
cwd=tmpFolderPath)
memberList = []
with untar_ps.stdout:
while True:
outputLine = untar_ps.stdout.readline().strip('\n')
if outputLine == '':
if untar_ps.poll() is not None:
break
else:
memberList.append(outputLine)
logger.debug(outputLine)
return_code = untar_ps.wait()
if return_code != 0:
raise CalledProcessError(return_code, process_args)
from re import compile
rexp = compile('\s+')
memberList = [
(int(size), name) for _, _, size, _, _, name in map(
lambda member: rexp.split(member), memberList)
]
end = time()
logger.info("Untar file content took %.2fs", end - start)
else:
memberName = entry.name if type(
entry) is tarfile.TarInfo else entry
untar_ps = Popen((
tar_cmd,
'--verbose',
'-xvzif',
filename,
memberName,
),
stdout=PIPE,
bufsize=1,
cwd=tmpFolderPath)
with untar_ps.stdout:
from re import compile
rexp = compile('\s+')
for line in iter(untar_ps.stdout.readline, b''):
line = line.strip('\n')
_, _, size, _, _, name = rexp.split(line)
memberList = [(int(size), name)]
break
for entry in memberList:
memberSize, memberName = (
entry.size,
entry.name,
) if type(entry) is tarfile.TarInfo else entry
oFile = os.path.join(tmpFolderPath, memberName)
while not os.path.isfile(oFile):
sleep(0.001)
while os.path.getsize(oFile) != memberSize:
sleep(0.001)
if not extractAll:
untar_ps.kill()
untar_ps.wait()
os.listdir(tmpFolderPath)
with open(oFile) as f_member:
data = transformDataRawData(
cPickle.load(f_member),
oFile if extractAll else filename, memberName)
yield data
if extractAll:
break
#else:
# if extractAll:
# logger.info("Untaring all members to %s...", tmpFolderPath)
# f.extractall(path=tmpFolderPath, )
# else:
# f.extractall(path=tmpFolderPath, members=(entry,))
# for entry in memberList if extractAll else [entry]:
# memberName = entry.name if type(entry) is tarfile.TarInfo else entry
# oFile = os.path.join(tmpFolderPath, memberName)
# print oFile
# with open(oFile) as f_member:
# yield transformDataRawData( cPickle.load(f_member), oFile if extractAll else filename, entry )
# if extractAll:
# break
if eraseTmpTarMembers:
shutil.rmtree(tmpFolderPath)
else:
fileobj = f.extractfile(entry)
if checkExtension(entry.name, 'gz|gzip'):
fio = StringIO.StringIO(fileobj.read())
fileobj = gzip.GzipFile(fileobj=fio)
yield transformDataRawData(cPickle.load(fileobj), filename,
memberName)
if not useSubprocess:
f.close()
def li_calc(calc_op, simnode, simacc, **kwargs):
scene = bpy.context.scene
pfs, epfs = [], []
context = simnode['coptions']['Context']
subcontext = simnode['coptions']['Type']
scene['liparams']['maxres'], scene['liparams']['minres'], scene['liparams']['avres'] = {}, {}, {}
frames = range(scene['liparams']['fs'], scene['liparams']['fe'] + 1) if not kwargs.get('genframe') else [kwargs['genframe']]
os.chdir(scene['viparams']['newdir'])
rtcmds, rccmds = [], []
builddict = {'0': ('School', 'Higher Education', 'Healthcare', 'Residential', 'Retail', 'Office & Other'), '2': ('School', 'Higher Education', 'Healthcare', 'Residential', 'Retail', 'Office & Other'), '3': ('Office/Education/Commercial', 'Healthcare')}
for f, frame in enumerate(frames):
if context == 'Basic' or (context == 'CBDM' and subcontext == '0') or (context == 'Compliance' and int(subcontext) < 3):
if os.path.isfile("{}-{}.af".format(scene['viparams']['filebase'], frame)):
os.remove("{}-{}.af".format(scene['viparams']['filebase'], frame))
if simnode.pmap:
pmappfile = open(os.path.join(scene['viparams']['newdir'], 'viprogress'), 'w')
pmappfile.close()
errdict = {'fatal - too many prepasses, no global photons stored\n': "Too many prepasses have ocurred. Make sure light sources can see your geometry",
'fatal - too many prepasses, no global photons stored, no caustic photons stored\n': "Too many prepasses have ocurred. Turn off caustic photons and encompass the scene",
'fatal - zero flux from light sources\n': "No light flux, make sure there is a light source and that photon port normals point inwards",
'fatal - no light sources\n': "No light sources. Photon mapping does not work with HDR skies"}
amentry, pportentry, cpentry, cpfileentry = retpmap(simnode, frame, scene)
pmcmd = ('mkpmap -e {1}.pmapmom -bv+ +fo -apD 0.001 {0} -apg {1}-{2}.gpm {3} {4} {5} {1}-{2}.oct'.format(pportentry, scene['viparams']['filebase'], frame, simnode.pmapgno, cpentry, amentry))
pmrun = Popen(pmcmd.split(), stderr = PIPE, stdout = PIPE)
while pmrun.poll() is None:
with open(os.path.join(scene['viparams']['newdir'], 'viprogress'), 'r') as pfile:
if 'CANCELLED' in pfile.read():
pmrun.kill()
return 'CANCELLED'
sleep(1)
with open('{}.pmapmon'.format(scene['viparams']['filebase']), 'r') as pmapfile:
for line in pmapfile.readlines():
if line in errdict:
calc_op.report({'ERROR'}, errdict[line])
return
# for line in pmrun.stdout:
# print(line)
# for line in pmrun.stderr:
# print(line)
# if line.decode() in errdict:
# calc_op.report({'ERROR'}, errdict[line.decode()])
# return
rtcmds.append("rtrace -n {0} -w {1} -ap {2}-{3}.gpm 50 {4} -faa -h -ov -I {2}-{3}.oct".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame, cpfileentry)) #+" | tee "+lexport.newdir+lexport.fold+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res"
else:
# if context == 'Compliance':
# rtcmds.append("rcontrib -w -n {} {} -m sky_glow -I+ -V+ {}-{}.oct ".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame))
#
# else:
rtcmds.append("rtrace -n {0} -w {1} -faa -h -ov -I {2}-{3}.oct".format(scene['viparams']['nproc'], simnode['radparams'], scene['viparams']['filebase'], frame)) #+" | tee "+lexport.newdir+lexport.fold+self.simlistn[int(lexport.metric)]+"-"+str(frame)+".res"
else:
rccmds.append("rcontrib -w -h -I -fo -bn 146 {} -n {} -f tregenza.cal -b tbin -m sky_glow {}-{}.oct".format(simnode['radparams'], scene['viparams']['nproc'], scene['viparams']['filebase'], frame))
# rccmds.append("rcontrib -w -h -I- -fo -bn 146 {} -n {} -f tregenza.cal -b tbin -m env_glow {}-{}.oct".format(simnode['radparams'], scene['viparams']['nproc'], scene['viparams']['filebase'], frame))
try:
tpoints = [bpy.data.objects[lc]['rtpnum'] for lc in scene['liparams']['livic']]
except:
calc_op.report({'ERROR'}, 'Re-export the LiVi geometry')
return
calcsteps = sum(tpoints) * len(frames)
pfile = progressfile(scene, datetime.datetime.now(), calcsteps)
kivyrun = progressbar(os.path.join(scene['viparams']['newdir'], 'viprogress'))
reslists = []
obs = [scene.objects[on] for on in scene['liparams']['livic']]
for oi, o in enumerate(obs):
curres = sum(tpoints[:oi])
selobj(scene, o)
o['omax'], o['omin'], o['oave'], totsensearea, totsdaarea, totasearea = {}, {}, {}, 0, 0, 0
if context == 'Basic':
bccout = o.basiccalcapply(scene, frames, rtcmds, simnode, curres, pfile)
if bccout == 'CANCELLED':
return 'CANCELLED'
else:
reslists += bccout
elif context == 'CBDM' and subcontext == '0':
lhout = o.lhcalcapply(scene, frames, rtcmds, simnode, curres, pfile)
if lhout == 'CANCELLED':
return 'CANCELLED'
else:
reslists += lhout
elif (context == 'CBDM' and subcontext in ('1', '2')) or (context == 'Compliance' and subcontext == '3'):
udiout = o.udidacalcapply(scene, frames, rccmds, simnode, curres, pfile)
if udiout == 'CANCELLED':
return 'CANCELLED'
else:
reslists += udiout[2]
pfs.append(udiout[0])
epfs.append(udiout[1])
elif context == 'Compliance':
compout = o.compcalcapply(scene, frames, rtcmds, simnode, curres, pfile)
if compout == 'CANCELLED':
return 'CANCELLED'
else:
reslists += compout[2]
pfs.append(compout[0])
epfs.append(compout[1])
for f, frame in enumerate(frames):
if context == 'Compliance':
tpf = 'FAIL' if 'FAIL' in pfs[f] or 'FAIL*' in pfs[f] else 'PASS'
if simnode['coptions']['canalysis'] == '0':
tpf = 'EXEMPLARY' if tpf == 'PASS' and ('FAIL' not in epfs[f] and 'FAIL*' not in epfs[f]) else tpf
cred = '0' if tpf == 'FAIL' else ('1', '2', '2', '1', '1', '1')[int(simnode['coptions']['buildtype'])]
ecred = '1' if tpf == 'EXEMPLARY' else '0'
simnode['tablecomp{}'.format(frame)] = [['Standard: BREEAM HEA1'],
['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Standard credits: ' + cred],
['Exemplary credits: '+ ecred]]
for o in obs:
o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])]), 'Standard credits: ' + cred, 'Exemplary credits: '+ ecred, '']]
elif simnode['coptions']['canalysis'] == '1':
cfshcred = 0
for pf in pfs[f]:
for stype in [0, 1, 2]:
if all([p[1] == 'Pass' for p in pf if p[0] == stype]) and [p for p in pf if p[0] == stype]:
cfshcred += 1
simnode['tablecomp{}'.format(frame)] = [['Standard: CfSH'],
['Build type: Residential'], [''], ['Standard credits: {}'.format(cfshcred)]]
for o in obs:
o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: Residential', 'Standard credits: {}'.format(cfshcred), '', '']]
elif simnode['coptions']['canalysis'] == '2':
gscred = max(len(p) for p in pfs[f]) - max([sum([(0, 1)[p == 'Fail'] for p in pf]) for pf in pfs[f]])
simnode['tablecomp{}'.format(frame)] = [['Standard: Green Star'],
['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Standard credits: {}'.format(gscred)]]
for o in obs:
o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']]), 'Standard credits: {}'.format(gscred), '', '']]
elif simnode['coptions']['canalysis'] == '3':
cred = 0
for z in list(zip(pfs[f], epfs[f])):
if all([pf == 'Pass' for pf in z[:-1]]):
cred += int(z[-1])
simnode['tablecomp{}'.format(frame)] = [['Standard: LEEDv4'],
['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])])], [''], ['Credits: {}'.format(cred)]]
for o in obs:
o['tablecomp{}'.format(frame)] += [['', '', '', ''], ['Build type: {}'.format(builddict[simnode['coptions']['canalysis']][int(simnode['coptions']['buildtype'])]), 'Credits: {}'.format(cred), '', '']]
if kivyrun.poll() is None:
kivyrun.kill()
return reslists
class SmtlibProc:
def __init__(self, command: str, debug: bool = False):
"""Single smtlib interactive process
:param command: the shell command to execute
:param debug: log all messaging
"""
self._proc: Optional[Popen] = None
self._command = command
self._debug = debug
def start(self):
"""Spawns POpen solver process"""
if self._proc is not None:
return
self._proc = Popen(
shlex.split(self._command),
stdin=PIPE,
stdout=PIPE,
bufsize=0,
universal_newlines=True,
close_fds=True,
)
def stop(self):
"""
Stops the solver process by:
- sending a SIGKILL signal,
- waiting till the process terminates (so we don't leave a zombie process)
"""
if self._proc is None:
return
# if it did not finished already
if self._proc.returncode is None:
self._proc.stdin.close()
self._proc.stdout.close()
# Kill the process
self._proc.kill()
self._proc.wait()
# No need to wait for termination, zombies avoided.
self._proc = None
def __readline_and_count(self):
assert self._proc
assert self._proc.stdout
buf = self._proc.stdout.readline() # No timeout enforced here
# If debug is enabled check if the solver reports a syntax error
# Error messages may contain an unbalanced parenthesis situation
if self._debug:
if "(error" in buf:
raise SolverException(f"Error in smtlib: {buf}")
lparen, rparen = map(sum, zip(*((c == "(", c == ")") for c in buf)))
return buf, lparen, rparen
def send(self, cmd: str) -> None:
"""
Send a string to the solver.
:param cmd: a SMTLIBv2 command (ex. (check-sat))
"""
if self._debug:
logger.debug(">%s", cmd)
self._proc.stdout.flush() # type: ignore
self._proc.stdin.write(f"{cmd}\n") # type: ignore
def recv(self) -> str:
"""Reads the response from the smtlib solver"""
buf, left, right = self.__readline_and_count()
bufl = [buf]
while left != right:
buf, l, r = self.__readline_and_count()
bufl.append(buf)
left += l
right += r
buf = "".join(bufl).strip()
if self._debug:
logger.debug("<%s", buf)
return buf
def _restart(self) -> None:
"""Auxiliary to start or restart the external solver"""
self.stop()
self.start()
def is_started(self):
return self._proc is not None
class MCDReforgedServer:
process: Optional[Popen]
def __init__(self):
# TODO console args
self.mcdr_state = MCDReforgedState.INITIALIZING
self.server_state = ServerState.STOPPED
self.process = None # type: Optional[PIPE]
self.flags = MCDReforgedFlag.NONE
self.starting_server_lock = Lock(
) # to prevent multiple start_server() call
self.stop_lock = Lock() # to prevent multiple stop() call
# will be assigned in on_config_changed()
self.encoding_method = None # type: Optional[str]
self.decoding_method = None # type: Optional[str]
# --- Constructing fields ---
self.logger = MCDReforgedLogger(self)
self.logger.set_file(constant.LOGGING_FILE)
self.server_interface = ServerInterface(self)
self.task_executor = TaskExecutor(self)
self.console_handler = ConsoleHandler(self)
self.watch_dog = WatchDog(self)
self.update_helper = UpdateHelper(self)
self.language_manager = LanguageManager(self.logger)
self.config = Config(self.logger)
self.rcon_manager = RconManager(self)
self.server_handler_manager = ServerHandlerManager(self)
self.reactor_manager = InfoReactorManager(self)
self.command_manager = CommandManager(self)
self.plugin_manager = PluginManager(self)
self.permission_manager = PermissionManager(self)
# --- Initialize fields instance ---
file_missing = False
try:
self.load_config(
allowed_missing_file=False) # loads config, language, handlers
except FileNotFoundError:
self.logger.error('Config is missing, default config generated')
self.config.save_default()
file_missing = True
try:
self.permission_manager.load_permission_file(
allowed_missing_file=False)
except FileNotFoundError:
self.logger.error(
'Permission file is missing, default permission file generated'
)
self.permission_manager.save_default()
file_missing = True
if file_missing:
self.on_first_start()
return
self.plugin_manager.register_permanent_plugins()
self.set_mcdr_state(MCDReforgedState.INITIALIZED)
def __del__(self):
try:
if self.process and self.process.poll() is None:
self.kill_server()
except:
pass
def on_first_start(self):
self.logger.info(
'Some of the user files are missing, check them before launch MCDR again'
)
default_config = self.config.get_default()
file_util.touch_directory(default_config['working_directory'])
self.plugin_manager.set_plugin_directories(
default_config['plugin_directories']) # to touch the directory
# --------------------------
# Translate info strings
# --------------------------
def tr(self, text, *args, allow_failure=True):
result = self.language_manager.translate(text,
allow_failure).strip('\r\n')
if len(args) > 0:
result = result.format(*args)
return result
# --------------------------
# Loaders
# --------------------------
def load_config(self, *, allowed_missing_file=True):
has_missing = self.config.read_config(allowed_missing_file)
# load the language first to make sure tr() is available
self.on_config_changed()
if has_missing:
for line in self.tr('config.missing_config').splitlines():
self.logger.warning(line)
def on_config_changed(self):
logger.console_color_disabled = self.config['disable_console_color']
self.logger.set_debug_options(self.config['debug'])
if self.config.is_debug_on():
self.logger.info(
self.tr('mcdr_server.on_config_changed.debug_mode_on'))
self.language_manager.set_language(self.config['language'])
self.logger.info(
self.tr('mcdr_server.on_config_changed.language_set',
self.config['language']))
self.encoding_method = self.config['encoding'] if self.config[
'encoding'] is not None else sys.getdefaultencoding()
self.decoding_method = self.config['decoding'] if self.config[
'decoding'] is not None else locale.getpreferredencoding()
self.logger.info(
self.tr('mcdr_server.on_config_changed.encoding_decoding_set',
self.encoding_method, self.decoding_method))
self.plugin_manager.set_plugin_directories(
self.config['plugin_directories'])
self.logger.info(
self.tr('mcdr_server.on_config_changed.plugin_directories_set',
self.encoding_method, self.decoding_method))
for directory in self.plugin_manager.plugin_directories:
self.logger.info('- {}'.format(directory))
self.reactor_manager.register_reactors(
self.config['custom_info_reactors'])
self.server_handler_manager.register_handlers(
self.config['custom_handlers'])
self.server_handler_manager.set_handler(self.config['handler'])
self.logger.info(
self.tr('mcdr_server.on_config_changed.handler_set',
self.config['handler']))
self.connect_rcon()
def load_plugins(self):
self.plugin_manager.refresh_all_plugins()
self.logger.info(
self.plugin_manager.last_operation_result.to_rtext(show_path=True))
def on_plugin_changed(self):
self.command_manager.clear_command()
self.plugin_manager.registry_storage.export_commands(
self.command_manager.register_command)
# ---------------------------
# State Getters / Setters
# ---------------------------
def is_server_running(self):
return self.server_state.in_state(
{ServerState.RUNNING, ServerState.STOPPING})
# Flags
def is_server_startup(self):
return MCDReforgedFlag.SERVER_STARTUP in self.flags
def is_server_rcon_ready(self):
return MCDReforgedFlag.SERVER_RCON_READY in self.flags
def is_interrupt(self):
return MCDReforgedFlag.INTERRUPT in self.flags
def is_mcdr_exit(self):
return self.mcdr_in_state(MCDReforgedState.STOPPED)
def is_exit_naturally(self):
return MCDReforgedFlag.EXIT_NATURALLY in self.flags
def with_flag(self, flag: MCDReforgedFlag):
self.flags |= flag
def remove_flag(self, flag: MCDReforgedFlag):
self.flags &= ~flag
def set_exit_naturally(self, flag):
if flag:
self.with_flag(MCDReforgedFlag.EXIT_NATURALLY)
else:
self.remove_flag(MCDReforgedFlag.EXIT_NATURALLY)
self.logger.debug('flag_exit_naturally has set to "{}"'.format(flag),
option=DebugOption.MCDR)
# State
def server_in_state(self, states):
return self.server_state.in_state(states)
def mcdr_in_state(self, states):
return self.mcdr_state.in_state(states)
def is_initialized(self):
return self.mcdr_in_state(MCDReforgedState.INITIALIZED)
def set_server_state(self, state):
self.server_state = state
self.logger.debug('Server state has set to "{}"'.format(state),
option=DebugOption.MCDR)
def set_mcdr_state(self, state):
self.mcdr_state = state
self.logger.debug('MCDR state has set to "{}"'.format(state),
option=DebugOption.MCDR)
def should_keep_looping(self):
"""
A criterion for sub threads to determine if it should keep looping
:rtype: bool
"""
if self.server_in_state(ServerState.STOPPED):
if self.is_interrupt(): # if interrupted and stopped
return False
return not self.is_exit_naturally(
) # if the sever exited naturally, exit MCDR
return not self.is_mcdr_exit()
# --------------------------
# Server Controls
# --------------------------
def start_server(self):
"""
try to start the server process
return True if the server process has started successfully
return False if the server is not able to start
:return: a bool as above
:rtype: bool
"""
with self.starting_server_lock:
if self.is_interrupt():
self.logger.warning(
self.tr('mcdr_server.start_server.already_interrupted'))
return False
if self.is_server_running():
self.logger.warning(
self.tr('mcdr_server.start_server.start_twice'))
return False
cwd = self.config['working_directory']
if not os.path.isdir(cwd):
self.logger.error(
self.tr('mcdr_server.start_server.cwd_not_existed', cwd))
return False
try:
start_command = self.config['start_command']
self.logger.info(
self.tr('mcdr_server.start_server.starting',
start_command))
self.process = Popen(start_command,
cwd=self.config['working_directory'],
stdin=PIPE,
stdout=PIPE,
stderr=STDOUT,
shell=True)
except:
self.logger.exception(
self.tr('mcdr_server.start_server.start_fail'))
return False
else:
self.on_server_start()
return True
def kill_server(self):
"""
Kill the server process group
"""
if self.process and self.process.poll() is None:
self.logger.info(self.tr('mcdr_server.kill_server.killing'))
try:
for child in psutil.Process(
self.process.pid).children(recursive=True):
child.kill()
self.logger.info(
self.tr('mcdr_server.kill_server.process_killed',
child.pid))
except psutil.NoSuchProcess:
pass
self.process.kill()
self.logger.info(
self.tr('mcdr_server.kill_server.process_killed',
self.process.pid))
else:
raise IllegalCallError(
"Server process has already been terminated")
def interrupt(self):
"""
Interrupt MCDR
The first call will softly stop the server and the later calls will kill the server
Return if it's the first try
:rtype: bool
"""
self.logger.info(
'Interrupting, first strike = {}'.format(not self.is_interrupt()))
self.stop(forced=self.is_interrupt())
ret = self.is_interrupt()
self.with_flag(MCDReforgedFlag.INTERRUPT)
return ret
def stop(self, forced=False):
"""
Stop the server
:param forced: an optional bool. If it's False (default) MCDR will stop the server by sending the STOP_COMMAND from the
current handler. If it's True MCDR will just kill the server process group
"""
with self.stop_lock:
if not self.is_server_running():
self.logger.warning(
self.tr('mcdr_server.stop.stop_when_stopped'))
return
self.set_server_state(ServerState.STOPPING)
if not forced:
try:
self.send(self.server_handler_manager.get_current_handler(
).get_stop_command())
except:
self.logger.error(self.tr('mcdr_server.stop.stop_fail'))
forced = True
if forced:
try:
self.kill_server()
except IllegalCallError:
pass
# --------------------------
# Server Logics
# --------------------------
def on_server_start(self):
self.set_exit_naturally(True)
self.logger.info(
self.tr('mcdr_server.start_server.pid_info', self.process.pid))
self.plugin_manager.dispatch_event(MCDRPluginEvents.SERVER_START, ())
self.set_server_state(ServerState.RUNNING)
def on_server_stop(self):
return_code = self.process.poll()
self.logger.info(
self.tr('mcdr_server.on_server_stop.show_stopcode', return_code))
self.process.stdin.close()
self.process.stdout.close()
self.process = None
self.remove_flag(
MCDReforgedFlag.SERVER_STARTUP
| MCDReforgedFlag.SERVER_RCON_READY) # removes this two
self.plugin_manager.dispatch_event(MCDRPluginEvents.SERVER_STOP,
(return_code, ))
self.set_server_state(ServerState.STOPPED)
def send(self, text, ending='\n', encoding=None):
"""
Send a text to server's stdin if the server is running
:param text: A str or a bytes you want to send. if text is a str then it will attach the ending parameter to its
back
:param str ending: The suffix of a command with a default value \n
:param str encoding: The encoding method for the text. If it's not given used the method in config
"""
if encoding is None:
encoding = self.encoding_method
if type(text) is str:
text = (text + ending).encode(encoding)
if self.is_server_running():
self.process.stdin.write(text)
self.process.stdin.flush()
else:
self.logger.warning(self.tr('mcdr_server.send.send_when_stopped'))
def receive(self):
"""
Try to receive a str from server's stdout. This will block the current thread
If server has stopped it will wait up to 10s for the server process to exit, then raise a ServerStopped exception
:rtype: str
:raise: ServerStopped
"""
while True:
try:
text = next(iter(self.process.stdout))
except StopIteration: # server process has stopped
for i in range(constant.WAIT_TIME_AFTER_SERVER_STDOUT_END_SEC *
10):
if self.process.poll() is not None:
break
time.sleep(0.1)
if i % 10 == 0:
self.logger.info(
self.tr('mcdr_server.receive.wait_stop'))
raise ServerStopped()
else:
try:
text = text.decode(self.decoding_method)
except:
self.logger.error(
self.tr('mcdr_server.receive.decode_fail', text))
raise
return text.rstrip('\n\r').lstrip('\n\r')
def tick(self):
"""
ticking MCDR:
try to receive a new line from server's stdout and parse / display / process the text
"""
try:
text = self.receive()
except ServerStopped:
self.on_server_stop()
return
try:
text = self.server_handler_manager.get_current_handler(
).pre_parse_server_stdout(text)
except:
self.logger.warning(self.tr('mcdr_server.tick.pre_parse_fail'))
parsed_result: Info
try:
parsed_result = self.server_handler_manager.get_current_handler(
).parse_server_stdout(text)
except:
if self.logger.should_log_debug(
option=DebugOption.HANDLER
): # traceback.format_exc() is costly
self.logger.debug(
'Fail to parse text "{}" from stdout of the server, using raw handler'
.format(text))
for line in traceback.format_exc().splitlines():
self.logger.debug(' {}'.format(line))
parsed_result = self.server_handler_manager.get_basic_handler(
).parse_server_stdout(text)
else:
if self.logger.should_log_debug(option=DebugOption.HANDLER):
self.logger.debug('Parsed text from server stdin:')
for line in parsed_result.format_text().splitlines():
self.logger.debug(' {}'.format(line))
self.server_handler_manager.detect_text(text)
self.reactor_manager.put_info(parsed_result)
def on_mcdr_start(self):
self.task_executor.start()
self.task_executor.enqueue_regular_task(self.load_plugins)
self.task_executor.wait_till_finish_all_task()
self.plugin_manager.dispatch_event(MCDRPluginEvents.MCDR_START, ())
if not self.config['disable_console_thread']:
self.console_handler.start()
else:
self.logger.info(
self.tr('mcdr_server.on_mcdr_start.console_disabled'))
if not self.start_server():
raise ServerStartError()
self.update_helper.start()
self.watch_dog.start()
self.server_handler_manager.start_handler_detection()
self.set_mcdr_state(MCDReforgedState.RUNNING)
def on_mcdr_stop(self):
try:
self.set_mcdr_state(MCDReforgedState.PRE_STOPPED)
if self.is_interrupt():
self.logger.info(
self.tr('mcdr_server.on_mcdr_stop.user_interrupted'))
else:
self.logger.info(
self.tr('mcdr_server.on_mcdr_stop.server_stop'))
self.watch_dog.stop() # it's ok for plugins to take some time
self.watch_dog.join()
self.plugin_manager.dispatch_event(MCDRPluginEvents.MCDR_STOP, ())
self.task_executor.wait_till_finish_all_task()
self.logger.info(self.tr('mcdr_server.on_mcdr_stop.bye'))
except KeyboardInterrupt: # I don't know why there sometimes will be a KeyboardInterrupt if MCDR is stopped by ctrl-c
pass
except:
self.logger.exception(
self.tr('mcdr_server.on_mcdr_stop.stop_error'))
finally:
self.set_mcdr_state(MCDReforgedState.STOPPED)
def start(self):
"""
The entry method to start MCDR
Try to start the server. if succeeded the console thread will start and MCDR will start ticking
:raise: IllegalStateError if MCDR is in wrong state
:raise: ServerStartError if the server is already running or start_server has been called by other
"""
if not self.mcdr_in_state(MCDReforgedState.INITIALIZED):
if self.mcdr_in_state(MCDReforgedState.INITIALIZING):
raise IllegalStateError(
'This instance is not fully initialized')
else:
raise IllegalStateError('MCDR can only start once')
self.main_loop()
return self.process
def main_loop(self):
"""
The main loop of MCDR
"""
self.on_mcdr_start()
while self.should_keep_looping():
try:
if self.is_server_running():
self.tick()
else:
time.sleep(0.01)
except KeyboardInterrupt:
self.interrupt()
except:
if self.is_interrupt():
break
else:
self.logger.critical(self.tr('mcdr_server.run.error'),
exc_info=True)
self.on_mcdr_stop()
def connect_rcon(self):
self.rcon_manager.disconnect()
if self.config['rcon']['enable'] and self.is_server_rcon_ready():
self.rcon_manager.connect(self.config['rcon']['address'],
self.config['rcon']['port'],
self.config['rcon']['password'])
class Z3Solver(Solver):
def __init__(self):
''' Build a Z3 solver instance.
This is implemented using an external z3 solver (via a subprocess).
'''
super(Z3Solver, self).__init__()
self._proc = None
self._log = '' # this should be enabled only if we are debugging
self.version = self._solver_version()
self.support_maximize = False
self.support_minimize = False
self.support_reset = True
logger.debug('Z3 version: %s', self.version)
if self.version >= Version(4, 5, 0):
self.support_maximize = False
self.support_minimize = False
self.support_reset = True
elif self.version >= Version(4, 4, 1):
self.support_maximize = True
self.support_minimize = True
self.support_reset = False
else:
logger.debug(
' Please install Z3 4.4.1 or newer to get optimization support'
)
self._command = 'z3 -t:120000 -smt2 -in'
self._init = [
'(set-logic QF_AUFBV)', '(set-option :global-decls false)'
]
self._get_value_fmt = (
re.compile('\(\((?P<expr>(.*))\ #x(?P<value>([0-9a-fA-F]*))\)\)'),
16)
@staticmethod
def _solver_version():
'''
If we
fail to parse the version, we assume z3's output has changed, meaning it's a newer
version than what's used now, and therefore ok.
Anticipated version_cmd_output format: 'Z3 version 4.4.2'
'Z3 version 4.4.5 - 64 bit - build hashcode $Z3GITHASH'
'''
their_version = Version(0, 0, 0)
try:
version_cmd_output = check_output('z3 -version'.split())
except OSError:
raise Z3NotFoundError
try:
version = version_cmd_output.split()[2]
their_version = Version(*map(int, version.split('.')))
except (IndexError, ValueError, TypeError):
pass
return their_version
def _start_proc(self):
''' Auxiliary method to spawn the external solver process'''
assert '_proc' not in dir(self) or self._proc is None
try:
self._proc = Popen(self._command.split(' '),
stdin=PIPE,
stdout=PIPE)
except OSError:
# Z3 was removed from the system in the middle of operation
raise Z3NotFoundError # TODO(mark) don't catch this exception in two places
# run solver specific initializations
for cfg in self._init:
self._send(cfg)
def _stop_proc(self):
''' Auxiliary method to stop the external solver process'''
if self._proc is not None:
self._send("(exit)")
self._proc.stdin.close()
self._proc.stdout.close()
self._proc.wait()
try:
self._proc.kill()
except BaseException:
pass
self._proc = None
# marshaling/pickle
def __getstate__(self):
raise Exception()
def __setstate__(self, state):
raise Exception()
def __del__(self):
try:
self._proc.stdin.writelines(('(exit)\n', ))
self._proc.wait()
except Exception:
pass
def _reset(self, constraints=None):
''' Auxiliary method to reset the smtlib external solver to initial defaults'''
if self._proc is None:
self._start_proc()
else:
if self.support_reset:
self._send("(reset)")
for cfg in self._init:
self._send(cfg)
else:
self._stop_proc()
self._start_proc()
if constraints is not None:
self._send(constraints)
def _send(self, cmd):
''' Send a string to the solver.
:param cmd: a SMTLIBv2 command (ex. (check-sat))
'''
logger.debug('>%s', cmd)
self._log += str(cmd) + '\n'
try:
buf = str(cmd)
self._proc.stdin.write(buf + '\n')
except IOError as e:
raise SolverException(e)
def _recv(self):
''' Reads the response from the solver '''
def readline():
buf = self._proc.stdout.readline()
return buf, buf.count('('), buf.count(')')
bufl = []
left = 0
right = 0
buf, l, r = readline()
bufl.append(buf)
left += l
right += r
while left != right:
buf, l, r = readline()
bufl.append(buf)
left += l
right += r
buf = ''.join(bufl).strip()
logger.debug('<%s', buf)
if '(error' in bufl[0]:
raise Exception("Error in smtlib: {}".format(bufl[0]))
return buf
# UTILS: check-sat get-value
def _check(self):
''' Check the satisfiability of the current state '''
logger.debug("Solver.check() ")
start = time.time()
self._send('(check-sat)')
_status = self._recv()
logger.debug("Check took %s seconds (%s)",
time.time() - start, _status)
if _status not in ('sat', 'unsat', 'unknown'):
raise SolverException(_status)
if consider_unknown_as_unsat:
if _status == 'unknown':
logger.warning(
'Found an unknown core, probably a solver timeout')
_status = 'unsat'
if _status == 'unknown':
raise SolverUnknown(_status)
return _status
def _assert(self, expression):
''' Auxiliary method to send an assert '''
assert isinstance(expression, Bool)
smtlib = translate_to_smtlib(expression)
self._send('(assert %s)' % smtlib)
def _getvalue(self, expression):
''' Ask the solver for one possible assignment for val using current set
of constraints.
The current set of assertions must be sat.
:param val: an expression or symbol '''
if not issymbolic(expression):
return expression
assert isinstance(expression, Variable)
self._send('(get-value (%s))' % expression.name)
ret = self._recv()
assert ret.startswith('((') and ret.endswith('))')
if isinstance(expression, Bool):
return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]]
elif isinstance(expression, BitVec):
pattern, base = self._get_value_fmt
m = pattern.match(ret)
expr, value = m.group('expr'), m.group('value')
return int(value, base)
raise NotImplementedError(
"_getvalue only implemented for Bool and BitVec")
# push pop
def _push(self):
''' Pushes and save the current constraint store and state.'''
self._send('(push 1)')
def _pop(self):
''' Recall the last pushed constraint store and state. '''
self._send('(pop 1)')
#@memoized
def can_be_true(self, constraints, expression):
''' Check if two potentially symbolic values can be equal '''
if isinstance(expression, bool):
if not expression:
return expression
else:
expression = BoolConstant(expression)
assert isinstance(expression, Bool)
with constraints as temp_cs:
temp_cs.add(expression)
self._reset(temp_cs.related_to(expression))
return self._check() == 'sat'
# get-all-values min max minmax
#@memoized
def get_all_values(self,
constraints,
expression,
maxcnt=3000,
silent=False):
''' Returns a list with all the possible values for the symbol x'''
if not isinstance(expression, Expression):
return [expression]
assert isinstance(constraints, ConstraintSet)
assert isinstance(expression, Expression)
with constraints as temp_cs:
if isinstance(expression, Bool):
var = temp_cs.new_bool()
elif isinstance(expression, BitVec):
var = temp_cs.new_bitvec(expression.size)
else:
raise NotImplementedError(
"get_all_values only implemted for Bool and BitVec")
temp_cs.add(var == expression)
self._reset(temp_cs.related_to(var))
result = []
val = None
while self._check() == 'sat':
value = self._getvalue(var)
result.append(value)
# Reset the solver to avoid the incremental mode
# Triggered with two consecutive calls to check-sat
# Yet, if the number of solution is large, sending back
# the whole formula is more expensive
if len(result) < 50:
self._reset(temp_cs.related_to(var))
for value in result:
self._assert(var != value)
else:
self._assert(var != value)
if len(result) >= maxcnt:
if silent:
# do not throw an exception if set to silent
# Default is not silent, assume user knows
# what they are doing and will check the size
# of returned vals list (previous smtlib behavior)
break
else:
raise TooManySolutions(result)
return result
#@memoized
def optimize(self, constraints, x, goal, M=10000):
''' Iterativelly finds the maximum or minimal value for the operation
(Normally Operators.UGT or Operators.ULT)
:param X: a symbol or expression
:param M: maximum number of iterations allowed
'''
assert goal in ('maximize', 'minimize')
assert isinstance(x, BitVec)
operation = {
'maximize': Operators.UGE,
'minimize': Operators.ULE
}[goal]
with constraints as temp_cs:
X = temp_cs.new_bitvec(x.size)
temp_cs.add(X == x)
aux = temp_cs.new_bitvec(X.size, name='optimized_')
self._reset(temp_cs)
self._send(aux.declaration)
if getattr(self, 'support_{}'.format(goal)):
self._push()
try:
self._assert(operation(X, aux))
self._send('(%s %s)' % (goal, aux.name))
self._send('(check-sat)')
_status = self._recv()
if _status not in ('sat', 'unsat', 'unknown'):
# Minimize (or Maximize) sometimes prints the objective before the status
# This will be a line like NAME |-> VALUE
maybe_sat = self._recv()
if maybe_sat == 'sat':
pattern = re.compile(
'(?P<expr>.*?)\s+\|->\s+(?P<value>.*)',
re.DOTALL)
m = pattern.match(_status)
expr, value = m.group('expr'), m.group('value')
assert expr == aux.name
return int(value)
elif _status == 'sat':
ret = self._recv()
if not (ret.startswith('(') and ret.endswith(')')):
raise SolverException(
'bad output on max, z3 may have been killed')
pattern = re.compile(
'\(objectives.*\((?P<expr>.*) (?P<value>\d*)\).*\).*',
re.MULTILINE | re.DOTALL)
m = pattern.match(ret)
expr, value = m.group('expr'), m.group('value')
assert expr == aux.name
return int(value)
finally:
self._pop()
self._reset(temp_cs)
self._send(aux.declaration)
operation = {
'maximize': Operators.UGT,
'minimize': Operators.ULT
}[goal]
self._assert(aux == X)
last_value = None
i = 0
while self._check() == 'sat':
last_value = self._getvalue(aux)
self._assert(operation(aux, last_value))
i = i + 1
if (i > M):
raise SolverException(
"Optimizing error, maximum number of iterations was reached"
)
if last_value is not None:
return last_value
raise SolverException("Optimizing error, unsat or unknown core")
#@memoized
def get_value(self, constraints, expression):
''' Ask the solver for one possible assignment for val using current set
of constraints.
The current set of assertions must be sat.
:param val: an expression or symbol '''
if not issymbolic(expression):
return expression
assert isinstance(expression, (Bool, BitVec, Array))
with constraints as temp_cs:
if isinstance(expression, Bool):
var = temp_cs.new_bool()
elif isinstance(expression, BitVec):
var = temp_cs.new_bitvec(expression.size)
elif isinstance(expression, Array):
var = []
result = ''
for i in xrange(expression.index_max):
subvar = temp_cs.new_bitvec(expression.value_bits)
var.append(subvar)
temp_cs.add(subvar == expression[i])
self._reset(temp_cs)
if self._check() != 'sat':
raise SolverException('Model is not available')
for i in xrange(expression.index_max):
self._send('(get-value (%s))' % var[i].name)
ret = self._recv()
assert ret.startswith('((') and ret.endswith('))')
pattern, base = self._get_value_fmt
m = pattern.match(ret)
expr, value = m.group('expr'), m.group('value')
result += chr(int(value, base))
return result
temp_cs.add(var == expression)
self._reset(temp_cs)
if self._check() != 'sat':
raise SolverException('Model is not available')
self._send('(get-value (%s))' % var.name)
ret = self._recv()
if not (ret.startswith('((') and ret.endswith('))')):
raise SolverException('SMTLIB error parsing response: %s' % ret)
if isinstance(expression, Bool):
return {'true': True, 'false': False}[ret[2:-2].split(' ')[1]]
if isinstance(expression, BitVec):
pattern, base = self._get_value_fmt
m = pattern.match(ret)
expr, value = m.group('expr'), m.group('value')
return int(value, base)
raise NotImplementedError(
"get_value only implemented for Bool and BitVec")
class CloudSqlProxyRunner(LoggingMixin):
"""
Downloads and runs cloud-sql-proxy as subprocess of the Python process.
The cloud-sql-proxy needs to be downloaded and started before we can connect
to the Google Cloud SQL instance via database connection. It establishes
secure tunnel connection to the database. It authorizes using the
GCP credentials that are passed by the configuration.
More details about the proxy can be found here:
https://cloud.google.com/sql/docs/mysql/sql-proxy
"""
def __init__(self,
path_prefix,
instance_specification,
gcp_conn_id='google_cloud_default',
project_id=None,
sql_proxy_version=None,
sql_proxy_binary_path=None):
"""
Creates the proxy runner class.
:param path_prefix: Unique path prefix where proxy will be downloaded and
directories created for unix sockets.
:type path_prefix: str
:param instance_specification: Specification of the instance to connect the
proxy to. It should be specified in the form that is described in
https://cloud.google.com/sql/docs/mysql/sql-proxy#multiple-instances in
-instances parameter (typically in the form of ``<project>:<region>:<instance>``
for UNIX socket connections and in the form of
``<project>:<region>:<instance>=tcp:<port>`` for TCP connections.
:type instance_specification: str
:param gcp_conn_id: Id of Google Cloud Platform connection to use for
authentication
:type gcp_conn_id: str
:param project_id: Optional id of the GCP project to connect to - it overwrites
default project id taken from the GCP connection.
:type project_id: str
:param sql_proxy_version: Specific version of SQL proxy to download
(for example 'v1.13'). By default latest version is downloaded.
:type sql_proxy_version: str
:param sql_proxy_binary_path: If specified, then proxy will be
used from the path specified rather than dynamically generated. This means
that if the binary is not present in that path it will also be downloaded.
:type sql_proxy_binary_path: str
"""
super(CloudSqlProxyRunner, self).__init__()
self.path_prefix = path_prefix
if not self.path_prefix:
raise AirflowException("The path_prefix must not be empty!")
self.sql_proxy_was_downloaded = False
self.sql_proxy_version = sql_proxy_version
self.download_sql_proxy_dir = None
self.sql_proxy_process = None
self.instance_specification = instance_specification
self.project_id = project_id
self.gcp_conn_id = gcp_conn_id
self.command_line_parameters = []
self.cloud_sql_proxy_socket_directory = self.path_prefix
self.sql_proxy_path = sql_proxy_binary_path if sql_proxy_binary_path \
else self.path_prefix + "_cloud_sql_proxy"
self.credentials_path = self.path_prefix + "_credentials.json"
self._build_command_line_parameters()
def _build_command_line_parameters(self):
self.command_line_parameters.extend(
['-dir', self.cloud_sql_proxy_socket_directory])
self.command_line_parameters.extend(
['-instances', self.instance_specification])
@staticmethod
def _is_os_64bit():
return platform.machine().endswith('64')
def _download_sql_proxy_if_needed(self):
if os.path.isfile(self.sql_proxy_path):
self.log.info("cloud-sql-proxy is already present")
return
system = platform.system().lower()
processor = "amd64" if CloudSqlProxyRunner._is_os_64bit() else "386"
if not self.sql_proxy_version:
download_url = CLOUD_SQL_PROXY_DOWNLOAD_URL.format(
system, processor)
else:
download_url = CLOUD_SQL_PROXY_VERSION_DOWNLOAD_URL.format(
self.sql_proxy_version, system, processor)
proxy_path_tmp = self.sql_proxy_path + ".tmp"
self.log.info("Downloading cloud_sql_proxy from %s to %s",
download_url, proxy_path_tmp)
r = requests.get(download_url, allow_redirects=True)
# Downloading to .tmp file first to avoid case where partially downloaded
# binary is used by parallel operator which uses the same fixed binary path
with open(proxy_path_tmp, 'wb') as f:
f.write(r.content)
if r.status_code != 200:
raise AirflowException(
"The cloud-sql-proxy could not be downloaded. Status code = {}. "
"Reason = {}".format(r.status_code, r.reason))
self.log.info("Moving sql_proxy binary from %s to %s", proxy_path_tmp,
self.sql_proxy_path)
shutil.move(proxy_path_tmp, self.sql_proxy_path)
os.chmod(self.sql_proxy_path, 0o744) # Set executable bit
self.sql_proxy_was_downloaded = True
@provide_session
def _get_credential_parameters(self, session):
connection = session.query(Connection). \
filter(Connection.conn_id == self.gcp_conn_id).first()
session.expunge_all()
if GCP_CREDENTIALS_KEY_PATH in connection.extra_dejson:
credential_params = [
'-credential_file',
connection.extra_dejson[GCP_CREDENTIALS_KEY_PATH]
]
elif GCP_CREDENTIALS_KEYFILE_DICT in connection.extra_dejson:
credential_file_content = json.loads(
connection.extra_dejson[GCP_CREDENTIALS_KEYFILE_DICT])
self.log.info("Saving credentials to %s", self.credentials_path)
with open(self.credentials_path, "w") as f:
json.dump(credential_file_content, f)
credential_params = ['-credential_file', self.credentials_path]
else:
self.log.info(
"The credentials are not supplied by neither key_path nor "
"keyfile_dict of the gcp connection %s. Falling back to "
"default activated account", self.gcp_conn_id)
credential_params = []
if not self.instance_specification:
project_id = connection.extra_dejson.get(
'extra__google_cloud_platform__project')
if self.project_id:
project_id = self.project_id
if not project_id:
raise AirflowException(
"For forwarding all instances, the project id "
"for GCP should be provided either "
"by project_id extra in the GCP connection or by "
"project_id provided in the operator.")
credential_params.extend(['-projects', project_id])
return credential_params
def start_proxy(self):
"""
Starts Cloud SQL Proxy.
You have to remember to stop the proxy if you started it!
"""
self._download_sql_proxy_if_needed()
if self.sql_proxy_process:
raise AirflowException(
"The sql proxy is already running: {}".format(
self.sql_proxy_process))
else:
command_to_run = [self.sql_proxy_path]
command_to_run.extend(self.command_line_parameters)
try:
self.log.info("Creating directory %s",
self.cloud_sql_proxy_socket_directory)
os.makedirs(self.cloud_sql_proxy_socket_directory)
except OSError:
# Needed for python 2 compatibility (exists_ok missing)
pass
command_to_run.extend(self._get_credential_parameters())
self.log.info("Running the command: `%s`",
" ".join(command_to_run))
self.sql_proxy_process = Popen(command_to_run,
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
self.log.info("The pid of cloud_sql_proxy: %s",
self.sql_proxy_process.pid)
while True:
line = self.sql_proxy_process.stderr.readline().decode('utf-8')
return_code = self.sql_proxy_process.poll()
if line == '' and return_code is not None:
self.sql_proxy_process = None
raise AirflowException(
"The cloud_sql_proxy finished early with return code {}!"
.format(return_code))
if line != '':
self.log.info(line)
if "googleapi: Error" in line or "invalid instance name:" in line:
self.stop_proxy()
raise AirflowException(
"Error when starting the cloud_sql_proxy {}!".format(
line))
if "Ready for new connections" in line:
return
def stop_proxy(self):
"""
Stops running proxy.
You should stop the proxy after you stop using it.
"""
if not self.sql_proxy_process:
raise AirflowException("The sql proxy is not started yet")
else:
self.log.info("Stopping the cloud_sql_proxy pid: %s",
self.sql_proxy_process.pid)
self.sql_proxy_process.kill()
self.sql_proxy_process = None
# Cleanup!
self.log.info("Removing the socket directory: %s",
self.cloud_sql_proxy_socket_directory)
shutil.rmtree(self.cloud_sql_proxy_socket_directory,
ignore_errors=True)
if self.sql_proxy_was_downloaded:
self.log.info("Removing downloaded proxy: %s", self.sql_proxy_path)
# Silently ignore if the file has already been removed (concurrency)
try:
os.remove(self.sql_proxy_path)
except OSError as e:
if not e.errno == errno.ENOENT:
raise
else:
self.log.info("Skipped removing proxy - it was not downloaded: %s",
self.sql_proxy_path)
if os.path.isfile(self.credentials_path):
self.log.info("Removing generated credentials file %s",
self.credentials_path)
# Here file cannot be delete by concurrent task (each task has its own copy)
os.remove(self.credentials_path)
def get_proxy_version(self):
"""
Returns version of the Cloud SQL Proxy.
"""
self._download_sql_proxy_if_needed()
command_to_run = [self.sql_proxy_path]
command_to_run.extend(['--version'])
command_to_run.extend(self._get_credential_parameters())
result = subprocess.check_output(command_to_run).decode('utf-8')
pattern = re.compile("^.*[V|v]ersion ([^;]*);.*$")
m = pattern.match(result)
if m:
return m.group(1)
else:
return None
def get_socket_path(self):
"""
Retrieves UNIX socket path used by Cloud SQL Proxy.
:return: The dynamically generated path for the socket created by the proxy.
:rtype: str
"""
return self.cloud_sql_proxy_socket_directory + "/" + self.instance_specification
def importdata(self):
env = os.environ.copy()
env["PROJ_LIB"] = "c:\\OSGeo4W64\\share\\proj"
sql = "select id from datafile where filename=? and md5=?"
self.db.cursor.execute(sql, [self.filename, self.md5])
fileid = self.db.cursor.fetchall()[0][0]
args = [self.proj]
args.extend(self.projparams)
with open(self.filename, 'r') as csvfile:
breaking = True
breaking = False
csvreader = csv.reader(csvfile, delimiter=",")
i = 0
header = []
insmeas = "insert into detectormeasure (measurementid,vd,ndet, livetime, rawdose, dose,totcount,spectra) values(?,?,?,?,?,?,?,?) "
vds = ['VD1', 'VD2', 'VD3', 'VD4']
for row in csvreader:
i = i + 1
if i == 2:
top = row
if not "GPS Sample" in top:
print(top)
print("No GPS data")
break # Makes no sense to continue if no GPS data
roistarts = [
i for i, s in enumerate(top)
if 'ROI for Virtual Detector' in s
]
spectrumstarts = [
i for i, s in enumerate(top) if 'Spectrum VD' in s
]
try:
specchs = spectrumstarts[1] - spectrumstarts[0]
except IndexError as error:
specchs = 1025 # Must be one extra
vdstart = [idx for idx, s in enumerate(top)
if '[1]' in s][0]
print(roistarts)
print(spectrumstarts)
print(vdstart)
print("===")
if i == 3:
header = row
meashead = dict(zip(header[0:vdstart], range(0, vdstart)))
print(meashead)
insert = "insert into measurement(datafileid,seqnum,systemtime,aqutimeus,GPSSmplFlags,GpsError,GpsTime,PDOP,longitude,latitude,GPSAltitude,LineNum"
values = " values(?,?,convert(datetime,?,103),?,?,?,convert(datetime,?,103),?,?,?,?,?"
adcs = False
pres = False
if "ADC 1" in header: # ADCs - for altitude measurement May or may not be exported
insert = insert + ",adc1,adc2"
values = values + ",?,?"
adcs = True
if "Pres" in header: # Pressure and temperature
insert = insert + ",Pres,Temp"
values = values + ",?,?"
pres = True
#insert=insert+",location,location_utm) "+values+",geometry::STGeomFromText('POINT ({} {})',4326),geometry::STGeomFromText('POINT ({})',32633)) "
insert = insert + ",location_utm) " + values + ",geometry::STGeomFromText('POINT ({})',32633)) "
# Analyze header to see that all that is needed is there...
if i > 3:
store = False
for vd in vds:
if self.req[vd][0] == '':
continue
vd = int(self.req[vd][0])
store = store or int(row[roistarts[vd - 1]]) > 0
# no reason to store data where the detector not yet is up
if not store:
continue
lon = row[meashead["Long"]]
lat = row[meashead["Lat"]]
projcoord = lon + " " + lat + ""
projcoord = projcoord.encode()
process = Popen(args, env=env, stdin=PIPE, stdout=PIPE)
utmcoord = process.communicate(input=projcoord)[0].decode()
if breaking:
print(projcoord)
print(utmcoord)
process.kill()
params = [fileid]
params.append(row[meashead["SeqNum"]])
params.append(row[meashead["UtcDate"]] + " " +
row[meashead["UtcTime"]])
params.append(row[meashead["µs"]])
# nothing in row[4]
params.append(row[meashead["SmplFlags"]])
params.append(row[meashead["GpsError"]])
params.append(row[meashead["GpsUtcDate"]] + " " +
row[meashead["GpsUtcTime"]])
params.append(row[meashead["PDOP"]])
params.append(lon)
params.append(lat)
params.append(row[meashead["Alt[m]"]])
params.append(row[meashead["LineNum"]])
if adcs:
params.append(row[meashead["ADC 1"]])
params.append(row[meashead["ADC 2"]])
if pres:
params.append(row[meashead["Pres"]])
params.append(row[meashead["Temp"]])
params = list(map(lambda x: 0 if x == '' else x, params))
#self.db.cursor.execute(insert.format(lon,lat,utmcoord),params)
self.db.cursor.execute(
insert.format(utmcoord), params
) # Using format to insert coordinates in the Geom from text - normal parameters do not work here
self.db.cursor.execute(
"select IDENT_CURRENT('measurement')"
) # Get last id - may fail
id = self.db.cursor.fetchall()
measid = id[0][0]
if i % 100 == 0:
print(i, measid)
for vd in vds:
if self.req[vd][0] == '':
continue
vd = int(self.req[vd][0])
roi = roistarts[vd - 1]
params = [measid, vd]
params.extend(
row[roi:roi + 5]
) # DetCount, LiveTime, Raw Doserate, Doserate TotCount[cps]
specst = spectrumstarts[vd - 1]
spectrum = ":".join(row[specst:specst + specchs])
params.append(spectrum)
# print("VD"+str(vd))
self.db.cursor.execute(insmeas, params)
#break # Jumps out if no data for this VD
if i > 10 and breaking:
break
sql = "update datafile set finished = 1 where id = ?"
self.db.cursor.execute(sql, fileid)
self.db.cursor.commit()
message = "Uploaded OK"
return (message)
class Pqos(object):
"""
The Pqos class defines methods to interact with pqos_wrapper cli.
"""
CAP_SYS_RAWIO = "cap_sys_rawio"
def __init__(self, show_warnings=False):
self.reset_required = False
self.show_warnings = show_warnings
self.mon_process = None
self.executable_path = find_executable2("pqos_wrapper")
if self.executable_path is None:
if self.show_warnings:
logging.info(
"Unable to find pqos_wrapper, please install it for "
"cache allocation and monitoring if your CPU supports Intel RDT "
"(cf. https://gitlab.com/sosy-lab/software/pqos-wrapper).")
def execute_command(self, __type, function, suppress_warning, *args):
"""
Execute a given pqos_wrapper command and log the output
@__type: The type of command being executed (monitoring or l3ca)
@function_name: The name of the function being executed in pqos_wrapper
@suppress_warning: A boolean to decide wether to print warning on failing execution
"""
if self.executable_path:
args_list = [self.executable_path] + list(args)
try:
if "-m" in args_list:
self.mon_process = Popen(args_list,
stdout=PIPE,
stderr=PIPE)
else:
ret = json.loads(
check_output(args_list, stderr=STDOUT).decode())
logging.debug(ret[function]["message"])
return True
except CalledProcessError as e:
if self.show_warnings and (not suppress_warning):
self.print_error_message(e.output.decode(), __type,
args_list)
return False
def print_error_message(self, err, __type, args_list):
"""
Prints error message returned from pqos_wrapper
@err: The error output returned by pqos_wrapper
@__type: The type of command being executed (monitoring or l3ca)
@args_list: The command being executed as a list
"""
msg_prefix = {
"mon": "Could not monitor events",
"l3ca": "Could not set cache allocation",
}
try:
ret = json.loads(err)
logging.warning("{0}...{1}".format(msg_prefix[__type],
ret["message"]))
self.check_for_errors()
except ValueError:
logging.warning("{0}...Unable to execute command {1}".format(
msg_prefix[__type], " ".join(args_list)))
def check_capacity(self, technology):
"""
Check if given intel rdt is supported.
@technology: The intel rdt to be tested
"""
return self.execute_command(technology, "check_capability", False,
"-c", technology)
@staticmethod
def convert_core_list(core_assignment):
"""
Convert a double list to a string.
@core_assignment: The double list of cores
"""
ret = []
for benchmark in core_assignment:
ret.append("[" + ",".join(str(core) for core in benchmark) + "]")
return "[" + ",".join(ret) + "]"
def allocate_l3ca(self, core_assignment):
"""
This method checks if L3CAT is available and calls pqos_wrapper to
allocate equal cache to each thread.
@core_assignment: The list of cores assigned to each run
"""
if self.check_capacity("l3ca"):
core_string = self.convert_core_list(core_assignment)
if self.execute_command("l3ca", "allocate_resource", False, "-a",
"l3ca", core_string):
self.reset_required = True
else:
self.reset_resources()
def start_monitoring(self, core_assignment):
"""
This method checks if monitoring capability is available and calls pqos_wrapper to
monitor events on given lists of cores.
@core_assignment: The list of cores assigned to each run
"""
if self.check_capacity("mon"):
core_string = self.convert_core_list(core_assignment)
self.execute_command("mon", "monitor_events", False, "-m",
core_string)
def stop_monitoring(self):
"""
This method stops monitoring by sending SIGINT to the monitoring process
and resets the RMID for monitored cores to 0
"""
ret = {}
if self.mon_process:
self.mon_process.send_signal(SIGINT)
mon_output = self.mon_process.communicate()
if self.mon_process.returncode == 0:
mon_data = json.loads(mon_output[0].decode())
logging.debug(mon_data["monitor_events"]["message"])
ret = self.flatten_mon_data(
mon_data["monitor_events"]["function_output"]
["monitoring_data"])
else:
if self.show_warnings:
self.print_error_message(mon_output[1].decode(), "mon",
self.mon_process.args)
self.mon_process.kill()
self.mon_process = None
else:
if self.show_warnings:
logging.warning("No monitoring process started")
return ret
def reset_monitoring(self):
"""
Reset monitoring RMID to 0 for all cores
"""
self.execute_command("mon", "reset_monitoring", True, "-rm")
@staticmethod
def flatten_mon_data(mon_data):
"""
Converts the monitoring data array received from pqos_wrapper
to a flattened dictionary
@mon_data: The array of data received from pqos_wrapper monitoring cli
"""
flatten_dict = {}
for data in mon_data:
core_str = ",".join(str(core) for core in data["cores"])
data.pop("cores", None)
for key, val in data.items():
if isinstance(val, dict):
for sub_key, sub_val in val.items():
if len(mon_data) > 1:
flatten_key = "{0}_{1}_cpus{2}".format(
key, sub_key, core_str)
else:
flatten_key = "{0}_{1}".format(key, sub_key)
flatten_dict[flatten_key] = sub_val
else:
if len(mon_data) > 1:
flatten_key = "{0}_cpus{1}".format(key, core_str)
else:
flatten_key = key
flatten_dict[flatten_key] = val
return flatten_dict
def reset_resources(self):
"""
This method resets all resources to default.
"""
if self.reset_required:
self.execute_command("l3ca", "reset_resources", True, "-r")
self.reset_required = False
def check_for_errors(self):
"""
This method logs a detailed error on a failed pqos_error command.
"""
cap = get_capability(self.executable_path)
if cap["error"] is False:
if self.CAP_SYS_RAWIO in cap["capabilities"]:
if not all(x in cap["set"] for x in ["e", "p"]):
logging.warning(
"Insufficient capabilities for pqos_wrapper, Please add e,p in cap_sys_rawio capability set of pqos_wrapper"
)
else:
logging.warning(
"Insufficient capabilities for pqos_wrapper, Please set capabilitiy cap_sys_rawio with e,p for pqos_wrapper"
)
msr = check_msr()
if msr["loaded"]:
current_user = grp.getgrgid(os.getegid()).gr_name
if msr["read"]:
if not msr["write"]:
logging.warning(
"Add write permissions for msr module for {}".format(
current_user))
else:
logging.warning(
"Add read and write permissions for msr module for {}".
format(current_user))
else:
logging.warning(
"Load msr module for using cache allocation/monitoring")
class ORIGAMIMS(object):
def __init__(self, *args, **kwargs):
self.__wx_app = wx.App(redirect=False)
self.run = None
self.view = None
self.init(*args, **kwargs)
def start(self):
self.view.Show()
self.__wx_app.MainLoop()
def quit(self):
self.__wx_app.ProcessIdle()
self.__wx_app.ExitMainLoop()
self.view.Destroy()
return True
def endSession(self):
wx.CallAfter(self.quit, force=True)
def init(self, *args, **kwargs):
self.config = config.config()
self.icons = config.IconContainer()
self.view = mainWindow.MyFrame(self,
config=self.config,
icons=self.icons,
title="ORIGAMI-MS")
self.wrensCMD = None
self.wrensRun = None
self.wrensReset = None
self.currentPath = ""
self.wrensInput = {
'polarity': None,
'activationZone': None,
'method': None,
'command': None
}
self.logging = True
self.config.startTime = (strftime("%H-%M-%S_%d-%m-%Y", gmtime()))
self.__wx_app.SetTopWindow(self.view)
self.__wx_app.SetAppName("ORIGAMI-MS")
self.__wx_app.SetVendorName("Lukasz G Migas, University of Manchester")
# Log all events to
if self.logging == True:
sys.stdin = self.view.panelPlots.log
sys.stdout = self.view.panelPlots.log
sys.stderr = self.view.panelPlots.log
self.importConfigFileOnStart(evt=None)
#===============================================================================
# ACQUISITION FUNCTIONS
#===============================================================================
def onLibraryLink(self, evt):
"""Open selected webpage."""
# set link
links = {
ID_helpHomepage: 'home',
ID_helpGitHub: 'github',
ID_helpCite: 'cite',
ID_helpNewVersion: 'newVersion'
}
link = self.config.links[links[evt.GetId()]]
# open webpage
try:
webbrowser.open(link, autoraise=1)
except:
pass
def onCheckParameters(self, evt):
"""
This function checks that all variables are in correct format
"""
print("version", self.config.version)
if isinstance(self.config.iSPV, (int, long)): pass
else:
msg = ("SPV value should be an integer!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if isinstance(self.config.iScanTime, (int, float)): pass
else:
msg = ("Scan time value should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if isinstance(self.config.iStartVoltage, (int, float)): pass
else:
msg = ("Start voltage should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if isinstance(self.config.iEndVoltage, (int, float)): pass
else:
msg = ("End voltage should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if isinstance(self.config.iStepVoltage, (int, float)): pass
else:
msg = ("Step voltage should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if self.config.iActivationMode == "Exponential":
if isinstance(self.config.iExponentPerct, (int, float)): pass
else:
msg = ("Exponential % value should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if isinstance(self.config.iExponentIncre, (int, float)): pass
else:
msg = ("Exponential increment value should be an float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
elif self.config.iActivationMode == "Boltzmann":
if isinstance(self.config.iBoltzmann, (int, float)): pass
else:
msg = ("Boltzmann offset value should be an integer or float!")
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return False
if (abs(self.config.iEndVoltage) <= abs(self.config.iStartVoltage)):
msg = ('End voltage has to be larger than starting voltage')
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if (abs(self.config.iEndVoltage) > 200):
msg = (
'The highest possible voltage is 200 V. Set to default: 200')
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iEndVoltage = 200
self.view.panelControls.endVoltage_input.SetValue(
str(self.config.iEndVoltage))
if (abs(self.config.iStartVoltage) < 0):
msg = ('The lowest possible voltage is 0 V. Set to default: 0')
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iStartVoltage = 0
self.view.panelControls.startVoltage_input.SetValue(
str(self.config.iStartVoltage))
if self.config.iSPV <= 0:
msg = ('SPV must be larger than 0! Set to default: 3')
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iSPV = 3
self.view.panelControls.spv_input.SetValue(str(self.config.iSPV))
if self.config.iScanTime <= 0:
msg = ('Scan time must be larger than 0! Set to default: 5')
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iScanTime = 5
self.view.panelControls.scanTime_input.SetValue(
str(self.config.iScanTime))
if self.config.iActivationMode == "Exponential":
if self.config.iExponentPerct < 0:
msg = (
'Exponential % must be larger or equal to 0! Set to default: 0'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iExponentPerct = 0
elif self.config.iExponentPerct >= 100:
msg = (
'Exponential % must be smaller than 100! Set to default: 0'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iExponentPerct = 0
self.view.panelControls.exponentialPerct_input.SetValue(
str(self.config.iExponentPerct))
if self.config.iExponentIncre <= 0:
msg = (
'Exponential increment must be larger than 0! Set to default: 0.01'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iExponentIncre = 0.01
elif self.config.iExponentIncre > 0.075:
msg = (
'Exponential increment must be smaller than 0.075! Set to default: 0.075'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iExponentIncre = 0.075
self.view.panelControls.exponentialIncrm_input.SetValue(
str(self.config.iExponentIncre))
elif self.config.iActivationMode == "Boltzmann":
if self.config.iBoltzmann < 10:
msg = (
'Boltzmann offset must be larger than 10! Set to default: 10'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iBoltzmann = 10
elif self.config.iBoltzmann >= 100:
msg = (
'Boltzmann offset must be smaller than 100! Set to default: 25'
)
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
self.config.iBoltzmann = 25
self.view.panelControls.boltzmann_input.SetValue(
str(self.config.iBoltzmann))
# All good
return True
def onCalculateParameters(self, evt):
"""
This function is to be used to setup path to save origami parameters
"""
if not self.config.iActivationMode == "User-defined":
if self.onCheckParameters(evt=None) == False:
print('Please fill in all necessary fields first!')
return
divisibleCheck = abs(
self.config.iEndVoltage -
self.config.iStartVoltage) / self.config.iStepVoltage
divisibleCheck2 = divisibleCheck % 1
if divisibleCheck2 != 0:
msg = "Are you sure your collision voltage range is divisible by your increment?"
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
else:
if self.config.iScanTime == None or self.config.iScanTime == "":
msg = 'Please make sure you to fill in the scan time input box.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if self.config.iActivationMode == "Linear":
self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareLinearMethod(
)
elif self.config.iActivationMode == "Exponential":
self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareExponentialMethod(
)
elif self.config.iActivationMode == "Boltzmann":
self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareBoltzmannMethod(
)
elif self.config.iActivationMode == "User-defined":
self.wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime = self.onPrepareListMethod(
)
self.wrensCMD = self.wrensInput.get('command', None)
# Setup status:
self.view.SetStatusText(''.join(
['Acq. time: ', str(totalAcqTime), ' mins']),
number=0)
self.view.SetStatusText(''.join(
[str(len(scanPerVoltageList)), ' steps']),
number=1)
# Add wrensCMD to config file
self.config.wrensCMD = self.wrensCMD
self.onPlotSPV(ColEnergyX, scanPerVoltageList)
self.onPlotTime(ColEnergyX, timeList)
print(''.join(["Your submission code: ", self.wrensCMD]))
def onPrepareLinearMethod(self):
startScansPerVoltage = self.config.iSPV
scanPerVoltageList, timeList = [], []
timeFit = 3
numberOfCEsteps = (self.config.iEndVoltage - self.config.iStartVoltage
) / self.config.iStepVoltage + 1
ColEnergyX = np.linspace(self.config.iStartVoltage,
self.config.iEndVoltage, numberOfCEsteps)
for i in range(int(numberOfCEsteps)):
scanPerVoltageFit = startScansPerVoltage
scanPerVoltageList.append(scanPerVoltageFit)
timeFit = timeFit + scanPerVoltageFit
timeList.append(timeFit * self.config.iScanTime)
numberOfCEScans = (numberOfCEsteps * self.config.iSPV)
timeOfCEScans = (numberOfCEScans +
self.config.iSPV) * self.config.iScanTime
totalAcqTime = round(
float((6 + numberOfCEScans + self.config.iSPV) *
self.config.iScanTime) / 60, 2)
if (totalAcqTime > 300):
msg = "The acquisition will take more than 5 hours. Consider reducing " \
"your collision voltage range or adjusting the parameters!"
dialogs.dlgBox(exceptionTitle='Very long acquisition warning',
exceptionMsg=msg,
type="Warning")
wrensCMD = ''.join([
self.config.wrensLinearPath, self.config.iActivationZone, ',',
self.config.iPolarity, ',',
str(self.config.iSPV), ',',
str(self.config.iScanTime), ',',
str(self.config.iStartVoltage), ',',
str(self.config.iEndVoltage), ',',
str(self.config.iStepVoltage), ',',
str(totalAcqTime)
])
wrensInput = {
'polarity': self.config.iPolarity,
'activationZone': self.config.iActivationZone,
'method': self.config.iActivationMode,
'command': wrensCMD
}
if self.config.iPolarity == 'POSITIVE': polarity = "+VE"
else: polarity = "-VE"
self.view.SetStatusText(''.join([
"Current method: ", polarity, " mode in the ",
self.config.iActivationZone, " using the ",
self.config.iActivationMode, " method"
]),
number=2)
return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime
def onPrepareExponentialMethod(self):
startScansPerVoltage = self.config.iSPV
scanPerVoltageList, timeList = [], []
timeFit = 3
expAccumulator = 0
numberOfCEsteps = (self.config.iEndVoltage - self.config.iStartVoltage
) / self.config.iStepVoltage + 1
ColEnergyX = np.linspace(self.config.iStartVoltage,
self.config.iEndVoltage, numberOfCEsteps)
for i in xrange(int(numberOfCEsteps)):
if abs(ColEnergyX[i]) >= abs(self.config.iEndVoltage *
self.config.iExponentPerct / 100):
expAccumulator = expAccumulator + self.config.iExponentIncre
scanPerVoltageFit = np.round(
startScansPerVoltage * np.exp(expAccumulator), 0)
else:
scanPerVoltageFit = startScansPerVoltage
scanPerVoltageList.append(scanPerVoltageFit)
timeFit = timeFit + scanPerVoltageFit
timeList.append(timeFit * self.config.iScanTime)
numberOfCEScans = sum(scanPerVoltageList)
timeOfCEScans = (numberOfCEScans +
scanPerVoltageList[-1]) * self.config.iScanTime
totalAcqTime = round(
float((6 + numberOfCEScans + self.config.iSPV) *
self.config.iScanTime) / 60, 2)
if (totalAcqTime > 300):
msg = "The acquisition will take more than 5 hours. Consider reducing " \
"your collision voltage range or adjusting the parameters!"
dialogs.dlgBox(exceptionTitle='Very long acquisition warning',
exceptionMsg=msg,
type="Warning")
wrensCMD = ''.join([
self.config.wrensExponentPath, self.config.iActivationZone, ',',
self.config.iPolarity, ',',
str(self.config.iSPV), ',',
str(self.config.iScanTime), ',',
str(self.config.iStartVoltage), ',',
str(self.config.iEndVoltage), ',',
str(self.config.iStepVoltage), ',',
str(self.config.iExponentPerct), ',',
str(self.config.iExponentIncre), ',',
str(totalAcqTime)
])
wrensInput = {
'polarity': self.config.iPolarity,
'activationZone': self.config.iActivationZone,
'method': self.config.iActivationMode,
'command': wrensCMD
}
if self.config.iPolarity == 'POSITIVE': polarity = "+VE"
else: polarity = "-VE"
self.view.SetStatusText(''.join([
"Current method: ", polarity, " mode in the ",
self.config.iActivationZone, " using the ",
self.config.iActivationMode, " method"
]),
number=2)
return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime
def onPrepareBoltzmannMethod(self):
startScansPerVoltage = self.config.iSPV
scanPerVoltageList, timeList = [], []
timeFit = 3
A1 = 2
A2 = 0.07
x0 = 47
numberOfCEsteps = int(
(self.config.iEndVoltage - self.config.iStartVoltage) /
self.config.iStepVoltage + 1)
ColEnergyX = np.linspace(self.config.iStartVoltage,
self.config.iEndVoltage, numberOfCEsteps)
for i in range(int(numberOfCEsteps)):
scanPerVoltageFit = np.round(
1 / (A2 + (A1 - A2) / (1 + np.exp(
(ColEnergyX[i] - x0) / self.config.iBoltzmann))), 0)
scanPerVoltageList.append(scanPerVoltageFit * startScansPerVoltage)
timeFit = timeFit + scanPerVoltageFit
timeList.append(timeFit * self.config.iScanTime)
numberOfCEScans = sum(scanPerVoltageList)
timeOfCEScans = (numberOfCEScans +
scanPerVoltageList[-1]) * self.config.iScanTime
totalAcqTime = round(
float((6 + numberOfCEScans + self.config.iSPV) *
self.config.iScanTime) / 60, 2)
if (totalAcqTime > 300):
msg = "The acquisition will take more than 5 hours. Consider reducing " \
"your collision voltage range or adjusting the parameters!"
dialogs.dlgBox(exceptionTitle='Very long acquisition warning',
exceptionMsg=msg,
type="Warning")
wrensCMD = ''.join([
self.config.wrensBoltzmannPath, self.config.iActivationZone, ',',
self.config.iPolarity, ',',
str(self.config.iSPV), ',',
str(self.config.iScanTime), ',',
str(self.config.iStartVoltage), ',',
str(self.config.iEndVoltage), ',',
str(self.config.iStepVoltage), ',',
str(self.config.iBoltzmann), ',',
str(totalAcqTime)
])
wrensInput = {
'polarity': self.config.iPolarity,
'activationZone': self.config.iActivationZone,
'method': self.config.iActivationMode,
'command': wrensCMD
}
if self.config.iPolarity == 'POSITIVE': polarity = "+VE"
else: polarity = "-VE"
self.view.SetStatusText(''.join([
"Current method: ", polarity, " mode in the ",
self.config.iActivationZone, " using the ",
self.config.iActivationMode, " method"
]),
number=2)
return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime
def onPrepareListMethod(self):
if self.config.CSVFilePath == None:
msg = 'Please load a CSV file first.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if self.config.iScanTime == None or self.config.iScanTime == "":
msg = 'Please fill in appropriate fields. The scan time is empty or incorrect'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
print(self.config.CSVFilePath)
try:
spvCVlist = np.genfromtxt(self.config.CSVFilePath,
skip_header=1,
delimiter=',',
filling_values=0)
except:
return
# Read table
scanPerVoltageList = spvCVlist[:, 0].astype(int)
ColEnergyX = spvCVlist[:, 1]
timeFit = 3
timeList = []
for i in scanPerVoltageList:
timeFit = timeFit + i
timeList.append(timeFit * self.config.iScanTime)
if len(scanPerVoltageList) != len(ColEnergyX): return
totalAcqTime = np.round(
(sum(scanPerVoltageList) * self.config.iScanTime) / 60, 2)
if (totalAcqTime > 300):
msg = "The acquisition will take more than 5 hours. Consider reducing " \
"your collision voltage range or adjusting the parameters!"
dialogs.dlgBox(exceptionTitle='Very long acquisition warning',
exceptionMsg=msg,
type="Warning")
SPV_list = ' '.join(str(spv) for spv in scanPerVoltageList.tolist())
SPV_list = ''.join(["[", SPV_list, "]"])
CV_list = ' '.join(str(cv) for cv in ColEnergyX.tolist())
CV_list = ''.join(["[", CV_list, "]"])
self.config.SPVsList = scanPerVoltageList
self.config.CVsList = ColEnergyX
print(SPV_list, CV_list)
wrensCMD = ''.join([
self.config.wrensUserDefinedPath, self.config.iActivationZone, ',',
self.config.iPolarity, ',',
str(self.config.iScanTime), ',', SPV_list, ',', CV_list
])
wrensInput = {
'polarity': self.config.iPolarity,
'activationZone': self.config.iActivationZone,
'method': self.config.iActivationMode,
'command': wrensCMD
}
if self.config.iPolarity == 'POSITIVE': polarity = "+VE"
else: polarity = "-VE"
self.view.SetStatusText(''.join([
"Current method: ", polarity, " mode in the ",
self.config.iActivationZone, " using the ",
self.config.iActivationMode, " method"
]),
number=2)
return wrensInput, ColEnergyX, scanPerVoltageList, timeList, totalAcqTime
def onStartWREnSRun(self, evt):
if self.wrensCMD == None:
msg = "Are you sure you filled in correct details or pressed calculate?"
dialogs.dlgBox(
exceptionTitle=
'Please complete all necessary fields and press Calculate',
exceptionMsg=msg,
type="Error")
return
# A couple of checks to ensure the method in the settings is the one
# currently available in memory..
if self.wrensInput.get("polarity", None) != self.config.iPolarity:
msg = 'The polarity of the current method and the one in the window do not agree. Consider re-calculating.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if self.wrensInput.get("activationZone",
None) != self.config.iActivationZone:
msg = 'The activation zone of the current method and the one in the window do not agree. Consider re-calculating.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if self.wrensInput.get("method", None) != self.config.iActivationMode:
msg = 'The acquisition mode of the current method and the one in the window do not agree. Consider re-calculating.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
if self.wrensInput.get("command", None) != self.wrensCMD:
msg = 'The command in the memory and the current method and the one in the window do not agree. Consider re-calculating.'
dialogs.dlgBox(exceptionTitle='Mistake in the input',
exceptionMsg=msg,
type="Error")
return
print(''.join(["Your code: ", self.wrensCMD]))
self.wrensRun = Popen(self.wrensCMD)
def onStopWREnSRun(self, evt):
if self.wrensRun:
print('Stopped acquisition and reset the property banks')
self.wrensRun.kill()
self.wrensReset = Popen(self.config.wrensResetPath)
self.view.panelControls.goBtn.Enable()
else:
print('You have to start acquisition first!')
def onFillInDefaults(self, evt=None):
"""
This function fills in default values in case you are being lazy!
"""
pass
def onGetMassLynxPath(self, evt):
"""
Select path to the MassLynx folder
"""
dlg = wx.DirDialog(self.view,
"Select MassLynx directory",
style=wx.DD_DEFAULT_STYLE)
if dlg.ShowModal() == wx.ID_OK:
print "You chose %s" % dlg.GetPath()
self.view.panelControls.path_value.SetValue(dlg.GetPath())
self.currentPath = dlg.GetPath()
def importConfigFile(self, evt):
"""
This function imports configuration file
"""
dlg = wx.FileDialog(self.view,
"Open Configuration File",
wildcard="*.ini",
style=wx.FD_DEFAULT_STYLE | wx.FD_CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
fileName = dlg.GetPath()
self.config.importConfig(fileName=fileName, e=None)
print(''.join(["WREnS runner path: ",
self.config.wrensRunnerPath]))
print(''.join(["Linear path: ", self.config.wrensLinearPath]))
print(''.join(["Exponent path: ", self.config.wrensExponentPath]))
print(''.join(["Boltzmann path: ",
self.config.wrensBoltzmannPath]))
print(''.join(["List path: ", self.config.wrensUserDefinedPath]))
print(''.join(["Reset path: ", self.config.wrensResetPath]))
def importConfigFileOnStart(self, evt):
print("Importing origamiConfig.ini")
self.config.importConfig(fileName="origamiConfig.ini", e=None)
print(''.join(["WREnS runner path: ", self.config.wrensRunnerPath]))
print(''.join(["Linear path: ", self.config.wrensLinearPath]))
print(''.join(["Exponent path: ", self.config.wrensExponentPath]))
print(''.join(["Boltzmann path: ", self.config.wrensBoltzmannPath]))
print(''.join(["List path: ", self.config.wrensUserDefinedPath]))
print(''.join(["Reset path: ", self.config.wrensResetPath]))
def onExportConfig(self, evt):
"""
This function exports configuration file
"""
dlg = wx.FileDialog(self.view,
"Save As Configuration File",
wildcard="*.ini",
style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
if dlg.ShowModal() == wx.ID_OK:
fileName = dlg.GetPath()
self.config.exportConfig(fileName=fileName, e=None)
def onExportParameters(self, evt):
dlg = wx.FileDialog(self.view,
"Save As ORIGAMI configuration File",
wildcard="*.conf",
style=wx.FD_SAVE | wx.FD_OVERWRITE_PROMPT)
dlg.SetFilename('origami.conf')
if dlg.ShowModal() == wx.ID_OK:
fileName = dlg.GetPath()
self.config.exportOrigamiConfig(fileName=fileName, e=None)
def onLoadCSVList(self, evt):
"""
This function loads a two column list with Collision voltage | number of scans
"""
dlg = wx.FileDialog(self.view,
"Choose a file:",
wildcard="*.txt; *.csv",
style=wx.FD_DEFAULT_STYLE | wx.FD_CHANGE_DIR)
if dlg.ShowModal() == wx.ID_OK:
print "You chose %s" % dlg.GetPath()
self.config.CSVFilePath = dlg.GetPath()
def onPlotSPV(self, xvals, yvals):
self.view.panelPlots.plot1.clearPlot()
self.view.panelPlots.plot1.plot1Ddata(xvals=xvals,
yvals=yvals,
title="",
xlabel="Collision Voltage (V)",
ylabel="SPV")
# Show the mass spectrum
self.view.panelPlots.plot1.repaint()
def onPlotTime(self, xvals, yvals):
self.view.panelPlots.plot2.clearPlot()
self.view.panelPlots.plot2.plot1Ddata(xvals=xvals,
yvals=yvals,
title="",
xlabel="Collision Voltage (V)",
ylabel="Accumulated Time (s)")
# Show the mass spectrum
self.view.panelPlots.plot2.repaint()
#!/usr/bin/env python3
from subprocess import Popen
from socket import *
tcp = Popen(['./cqotdd-tcp', 'This is a test purely intended to test.'])
udp = Popen(['./cqotdd-udp', 'This is a test purely intended to test.'])
sock = socket(AF_INET, SOCK_STREAM)
sock.connect(('127.0.0.1', 17))
print(sock.recv(512).decode('UTF-8'), end='')
sock.close()
sock = socket(AF_INET, SOCK_DGRAM)
sock.sendto(b'\0' * 22, ('127.0.0.1', 17))
print(sock.recv(512).decode('UTF-8'), end='')
sock.close()
tcp.kill()
udp.kill()
def test_reload_wrapper_preservation(self):
# This test verifies that when `python -m tornado.autoreload`
# is used on an application that also has an internal
# autoreload, the reload wrapper is preserved on restart.
main = """\
import os
import sys
# This import will fail if path is not set up correctly
import testapp
if 'tornado.autoreload' not in sys.modules:
raise Exception('started without autoreload wrapper')
import tornado.autoreload
print('Starting')
sys.stdout.flush()
if 'TESTAPP_STARTED' not in os.environ:
os.environ['TESTAPP_STARTED'] = '1'
# Simulate an internal autoreload (one not caused
# by the wrapper).
tornado.autoreload._reload()
else:
# Exit directly so autoreload doesn't catch it.
os._exit(0)
"""
# Create temporary test application
os.mkdir(os.path.join(self.path, 'testapp'))
init_file = os.path.join(self.path, 'testapp', '__init__.py')
open(init_file, 'w').close()
main_file = os.path.join(self.path, 'testapp', '__main__.py')
with open(main_file, 'w') as f:
f.write(main)
# Make sure the tornado module under test is available to the test
# application
pythonpath = os.getcwd()
if 'PYTHONPATH' in os.environ:
pythonpath += os.pathsep + os.environ['PYTHONPATH']
autoreload_proc = Popen(
[sys.executable, '-m', 'tornado.autoreload', '-m', 'testapp'],
stdout=subprocess.PIPE,
cwd=self.path,
env=dict(os.environ, PYTHONPATH=pythonpath),
universal_newlines=True)
# This timeout needs to be fairly generous for pypy due to jit
# warmup costs.
for i in range(40):
if autoreload_proc.poll() is not None:
break
time.sleep(0.1)
else:
autoreload_proc.kill()
raise Exception("subprocess failed to terminate")
out = autoreload_proc.communicate()[0]
self.assertEqual(out, 'Starting\n' * 2)
class LocalhostRun:
def __init__(self, port, id=None, interval=30, retries=30):
import os
filePath = "/usr/local/sessionSettings/localhostDB.json"
if not os.path.exists(filePath):
os.makedirs(filePath[:-16], exist_ok=True)
open(filePath, 'w').close()
installAutoSSH()
if not id: id = str(uuid.uuid4())[:8]
self.connection = None
self.id = id
self.port = port
self.interval = interval
self.retries = retries
def start(self):
if self.connection: self.connection.kill()
self.connection = Popen(
f"ssh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no"
.split(),
stdout=PIPE,
stdin=PIPE)
try:
return re.findall(
"http://(.*?.localhost.run)",
self.connection.stdout.readline().decode("utf-8"))[0]
except:
raise Exception(self.connection.stdout.readline().decode("utf-8"))
def keep_alive(self):
# if self.connection:self.connection.kill()
import urllib
try:
localhostOpenDB = dict(
accessSettingFile("localhostDB.json", v=False))
except TypeError:
localhostOpenDB = dict()
if findProcess("autossh", f"80:localhost:{self.port}"):
try:
oldAddr = localhostOpenDB[str(self.port)]
urllib.request.urlopen("http://" + oldAddr)
return oldAddr
except:
pass
for _ in range(2):
self.connection = Popen(
f"autossh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no -o ServerAliveInterval={self.interval} -o ServerAliveCountMax={self.retries}"
.split(),
stdout=PIPE,
stdin=PIPE,
stderr=PIPE)
#print("ssh -R 80:localhost:{self.port} {self.id}@ssh.localhost.run -o StrictHostKeyChecking=no -o ServerAliveInterval={self.interval} -o ServerAliveCountMax={self.retries}")
try:
newAddr = re.findall(
"http://(.*?.localhost.run)",
self.connection.stdout.readline().decode("utf-8"))[0]
localhostOpenDB[str(self.port)] = newAddr
accessSettingFile("localhostDB.json", localhostOpenDB, v=False)
return newAddr
except:
outs, errs = self.connection.communicate(timeout=15)
self.connection.kill()
# print(outs)
# print(errs)
if re.search(r"Permission\sdenied\s\(publickey\)",
errs.decode("utf-8")):
os.system("ssh-keygen -q -t rsa -N '' -f ~/.ssh/id_rsa")
continue
raise Exception(errs.decode("utf-8"))
break
def kill(self):
self.connection.kill()
def run_attach(cmd, output_cb=None):
""" run command and attach output to cb
Arguments:
cmd: shell command
output_cb: where to display output
"""
stdoutmaster, stdoutslave = pty.openpty()
subproc = Popen(cmd, shell=True,
stdout=stdoutslave,
stderr=PIPE)
os.close(stdoutslave)
decoder = codecs.getincrementaldecoder('utf-8')()
def last_ten_lines(s):
chunk = s[-1500:]
lines = chunk.splitlines(True)
return ''.join(lines[-10:]).replace('\r', '')
decoded_output = ""
try:
while subproc.poll() is None:
try:
b = os.read(stdoutmaster, 512)
except OSError as e:
if e.errno != errno.EIO:
raise
break
else:
final = False
if not b:
final = True
decoded_chars = decoder.decode(b, final)
if decoded_chars is None:
continue
decoded_output += decoded_chars
if output_cb:
ls = last_ten_lines(decoded_output)
output_cb(ls)
if final:
break
finally:
os.close(stdoutmaster)
if subproc.poll() is None:
subproc.kill()
subproc.wait()
errors = [l.decode('utf-8') for l in subproc.stderr.readlines()]
if output_cb:
output_cb(last_ten_lines(decoded_output))
errors = ''.join(errors)
if subproc.returncode == 0:
return decoded_output.strip()
else:
raise Exception("Problem running {0} "
"{1}:{2}".format(cmd,
subproc.returncode))
DEPENDENCIES = ('requests', 'bs4', 'cryptography', 'git', 'psutil')
assert all(map(find_spec, DEPENDENCIES)),\
'Missing dependencies are required to continue. Check to make sure you have {} installed.'.format(', '.join(DEPENDENCIES[:-1]).title()+', and '+DEPENDENCIES[-1].title())
print('Dependency check passed.')
HERE = os.path.dirname(os.path.realpath(__file__))
os.chdir(HERE)
for folder in ('logs', 'data', 'config', 'web/assets'):
try:
os.mkdir(folder)
except FileExistsError:
pass
assert os.path.exists('.env'), 'Toplevel env file not found.'
while True:
process = Popen([sys.executable, '_main.py'],
stdin=sys.stdin,
stdout=sys.stdout,
stderr=sys.stderr)
process.wait()
if process.returncode == 37:
process.kill()
print('Rebooting.')
continue
elif process.returncode == 0:
print('Shut down gracefully.')
process.kill()
break
class W3MImageDisplayer(ImageDisplayer, FileManagerAware):
"""Implementation of ImageDisplayer using w3mimgdisplay, an utilitary
program from w3m (a text-based web browser). w3mimgdisplay can display
images either in virtual tty (using linux framebuffer) or in a Xorg session.
Does not work over ssh.
w3m need to be installed for this to work.
"""
is_initialized = False
def __init__(self):
self.binary_path = None
self.process = None
def initialize(self):
"""start w3mimgdisplay"""
self.binary_path = None
self.binary_path = self._find_w3mimgdisplay_executable() # may crash
self.process = Popen([self.binary_path] + W3MIMGDISPLAY_OPTIONS,
stdin=PIPE,
stdout=PIPE,
universal_newlines=True)
self.is_initialized = True
@staticmethod
def _find_w3mimgdisplay_executable():
paths = [os.environ.get(W3MIMGDISPLAY_ENV, None)] + W3MIMGDISPLAY_PATHS
for path in paths:
if path is not None and os.path.exists(path):
return path
raise ImageDisplayError(
"No w3mimgdisplay executable found. Please set "
"the path manually by setting the %s environment variable. (see "
"man page)" % W3MIMGDISPLAY_ENV)
def _get_font_dimensions(self):
# Get the height and width of a character displayed in the terminal in
# pixels.
if self.binary_path is None:
self.binary_path = self._find_w3mimgdisplay_executable()
farg = struct.pack("HHHH", 0, 0, 0, 0)
fd_stdout = sys.stdout.fileno()
fretint = fcntl.ioctl(fd_stdout, termios.TIOCGWINSZ, farg)
rows, cols, xpixels, ypixels = struct.unpack("HHHH", fretint)
if xpixels == 0 and ypixels == 0:
process = Popen([self.binary_path, "-test"],
stdout=PIPE,
universal_newlines=True)
output, _ = process.communicate()
output = output.split()
xpixels, ypixels = int(output[0]), int(output[1])
# adjust for misplacement
xpixels += 2
ypixels += 2
return (xpixels // cols), (ypixels // rows)
def draw(self, path, start_x, start_y, width, height):
if not self.is_initialized or self.process.poll() is not None:
self.initialize()
try:
input_gen = self._generate_w3m_input(path, start_x, start_y, width,
height)
except ImageDisplayError:
raise
# Mitigate the issue with the horizontal black bars when
# selecting some images on some systems. 2 milliseconds seems
# enough. Adjust as necessary.
if self.fm.settings.w3m_delay > 0:
from time import sleep
sleep(self.fm.settings.w3m_delay)
self.process.stdin.write(input_gen)
self.process.stdin.flush()
self.process.stdout.readline()
self.quit()
self.is_initialized = False
def clear(self, start_x, start_y, width, height):
if not self.is_initialized or self.process.poll() is not None:
self.initialize()
fontw, fonth = self._get_font_dimensions()
cmd = "6;{x};{y};{w};{h}\n4;\n3;\n".format(
x=int((start_x - 0.2) * fontw),
y=start_y * fonth,
# y = int((start_y + 1) * fonth), # (for tmux top status bar)
w=int((width + 0.4) * fontw),
h=height * fonth + 1,
# h = (height - 1) * fonth + 1, # (for tmux top status bar)
)
try:
self.process.stdin.write(cmd)
except IOError as ex:
if ex.errno == errno.EPIPE:
return
raise
self.process.stdin.flush()
self.process.stdout.readline()
def _generate_w3m_input(self, path, start_x, start_y, max_width,
max_height):
"""Prepare the input string for w3mimgpreview
start_x, start_y, max_height and max_width specify the drawing area.
They are expressed in number of characters.
"""
fontw, fonth = self._get_font_dimensions()
if fontw == 0 or fonth == 0:
raise ImgDisplayUnsupportedException
max_width_pixels = max_width * fontw
max_height_pixels = max_height * fonth - 2
# (for tmux top status bar)
# max_height_pixels = (max_height - 1) * fonth - 2
# get image size
cmd = "5;{}\n".format(path)
self.process.stdin.write(cmd)
self.process.stdin.flush()
output = self.process.stdout.readline().split()
if len(output) != 2:
raise ImageDisplayError('Failed to execute w3mimgdisplay', output)
width = int(output[0])
height = int(output[1])
# get the maximum image size preserving ratio
if width > max_width_pixels:
height = (height * max_width_pixels) // width
width = max_width_pixels
if height > max_height_pixels:
width = (width * max_height_pixels) // height
height = max_height_pixels
return "0;1;{x};{y};{w};{h};;;;;{filename}\n4;\n3;\n".format(
x=int((start_x - 0.2) * fontw),
y=start_y * fonth,
# y = (start_y + 1) * fonth, # (for tmux top status bar)
w=width,
h=height,
filename=path,
)
def quit(self):
if self.is_initialized and self.process and self.process.poll(
) is None:
self.process.kill()
def remote_access_service(
local_host="127.0.0.1",
local_port=22267,
server="app.pywebio.online",
server_port=1022,
remote_port="/",
setup_timeout=60,
):
"""
Wait at most one minute to get the ssh output, if it gets a normal out, the connection is successfully established.
Otherwise report error and kill ssh process.
:param local_port: ssh local listen port
:param server: ssh server domain
:param server_port: ssh server port
:param setup_timeout: If the service can't setup successfully in `setup_timeout` seconds, then exit.
"""
global _ssh_process, _ssh_notfound
bin = State.deploy_config.SSHExecutable
cmd = f"{bin} -oStrictHostKeyChecking=no -R {remote_port}:{local_host}:{local_port} -p {server_port} {server} -- --output json"
args = shlex.split(cmd)
logger.debug(f"remote access service command: {cmd}")
if _ssh_process is not None and _ssh_process.poll() is None:
logger.warning(f"Kill previous ssh process [{_ssh_process.pid}]")
_ssh_process.kill()
try:
_ssh_process = Popen(args, stdout=PIPE, stderr=PIPE)
except FileNotFoundError as e:
logger.critical(
f"Cannot find SSH executable {bin}, please install OpenSSH or specify SSHExecutable in deploy.yaml"
)
_ssh_notfound = True
return
logger.info(f"remote access process pid: {_ssh_process.pid}")
success = False
def timeout_killer(wait_sec):
time.sleep(wait_sec)
if not success and _ssh_process.poll() is None:
logger.info("Connection timeout, kill ssh process")
_ssh_process.kill()
threading.Thread(target=timeout_killer,
kwargs=dict(wait_sec=setup_timeout),
daemon=True).start()
stdout = _ssh_process.stdout.readline().decode("utf8")
logger.info(f"ssh server stdout: {stdout}")
connection_info = {}
try:
connection_info = json.loads(stdout)
success = True
except json.decoder.JSONDecodeError:
if not success and _ssh_process.poll() is None:
_ssh_process.kill()
if success:
if connection_info.get("status", "fail") != "success":
logger.info(
f"Failed to establish remote access, this is the error message from service provider: {connection_info.get('message', '')}"
)
new_username = connection_info.get("change_username", None)
if new_username:
logger.info(
f"Server requested to change username, change it to: {new_username}"
)
State.deploy_config.SSHUser = new_username
else:
global address
address = connection_info["address"]
logger.debug(f"Remote access url: {address}")
# wait ssh or main thread exit
while not am_i_the_only_thread() and _ssh_process.poll() is None:
# while _ssh_process.poll() is None:
time.sleep(1)
if _ssh_process.poll() is None: # main thread exit, kill ssh process
logger.info("App process exit, killing ssh process")
_ssh_process.kill()
else: # ssh process exit by itself or by timeout killer
stderr = _ssh_process.stderr.read().decode("utf8")
if stderr:
logger.error(
f"PyWebIO application remote access service error: {stderr}")
else:
logger.info("PyWebIO application remote access service exit.")
address = None
class Py3:
"""
Helper object that gets injected as ``self.py3`` into Py3status
modules that have not got that attribute set already.
This allows functionality like:
- User notifications
- Forcing module to update (even other modules)
- Triggering events for modules
Py3 is also used for testing in which case it does not get a module when
being created. All methods should work in this situation.
"""
CACHE_FOREVER = PY3_CACHE_FOREVER
"""
Special constant that when returned for ``cached_until`` will cause the
module to not update unless externally triggered.
"""
LOG_ERROR = PY3_LOG_ERROR
"""Show as Error"""
LOG_INFO = PY3_LOG_INFO
"""Show as Informational"""
LOG_WARNING = PY3_LOG_WARNING
"""Show as Warning"""
# Shared by all Py3 Instances
_formatter = None
_gradients = Gradients()
_none_color = NoneColor()
_storage = Storage()
# Exceptions
Py3Exception = exceptions.Py3Exception
CommandError = exceptions.CommandError
RequestException = exceptions.RequestException
RequestInvalidJSON = exceptions.RequestInvalidJSON
RequestTimeout = exceptions.RequestTimeout
RequestURLError = exceptions.RequestURLError
def __init__(self, module=None):
self._audio = None
self._config_setting = {}
self._english_env = dict(os.environ)
self._english_env["LC_ALL"] = "C"
self._english_env["LANGUAGE"] = "C"
self._format_placeholders = {}
self._format_placeholders_cache = {}
self._is_python_2 = sys.version_info < (3, 0)
self._module = module
self._report_exception_cache = set()
self._thresholds = None
self._threshold_gradients = {}
self._uid = uuid4()
if module:
self._i3s_config = module._py3_wrapper.config["py3_config"][
"general"]
self._module_full_name = module.module_full_name
self._output_modules = module._py3_wrapper.output_modules
self._py3status_module = module.module_class
self._py3_wrapper = module._py3_wrapper
# create formatter we only if need one but want to pass py3_wrapper so
# that we can do logging etc.
if not self._formatter:
self.__class__._formatter = Formatter(module._py3_wrapper)
def __getattr__(self, name):
"""
Py3 can provide COLOR constants
eg COLOR_GOOD, COLOR_BAD, COLOR_DEGRADED
but also any constant COLOR_XXX we find this color in the config
if it exists
"""
if not name.startswith("COLOR_"):
raise AttributeError("Attribute `%s` not in Py3" % name)
return self._get_config_setting(name.lower())
def _get_config_setting(self, name, default=None):
try:
return self._config_setting[name]
except KeyError:
fn = self._py3_wrapper.get_config_attribute
param = fn(self._module_full_name, name)
# colors are special we want to make sure that we treat a color
# that was explicitly set to None as a True value. Ones that are
# not set should be treated as None
if name.startswith("color_"):
if hasattr(param, "none_setting"):
# see if named color and use if it is
param = COLOR_NAMES.get(name[6:].lower())
elif param is None:
param = self._none_color
# if a non-color parameter and was not set then set to default
elif hasattr(param, "none_setting"):
param = default
self._config_setting[name] = param
return self._config_setting[name]
def _get_color(self, color):
if not color:
return
# fix any hex colors so they are #RRGGBB
if color.startswith("#"):
color = color.upper()
if len(color) == 4:
color = ("#" + color[1] + color[1] + color[2] + color[2] +
color[3] + color[3])
return color
name = "color_%s" % color
return self._get_config_setting(name)
def _thresholds_init(self):
"""
Initiate and check any thresholds set
"""
thresholds = getattr(self._py3status_module, "thresholds", [])
self._thresholds = {}
if isinstance(thresholds, list):
try:
thresholds.sort()
except TypeError:
pass
self._thresholds[None] = [(x[0], self._get_color(x[1]))
for x in thresholds]
return
elif isinstance(thresholds, dict):
for key, value in thresholds.items():
if isinstance(value, list):
try:
value.sort()
except TypeError:
pass
self._thresholds[key] = [(x[0], self._get_color(x[1]))
for x in value]
def _get_module_info(self, module_name):
"""
THIS IS PRIVATE AND UNSUPPORTED.
Get info for named module. Info comes back as a dict containing.
'module': the instance of the module,
'position': list of places in i3bar, usually only one item
'type': module type py3status/i3status
"""
return self._output_modules.get(module_name)
def _report_exception(self, msg, frame_skip=2):
"""
THIS IS PRIVATE AND UNSUPPORTED.
logs an exception that occurs inside of a Py3 method. We only log the
exception once to prevent spamming the logs and we do not notify the
user.
frame_skip is used to change the place in the code that the error is
reported as coming from. We want to show it as coming from the
py3status module where the Py3 method was called.
"""
# We use a hash to see if the message is being repeated.
msg_hash = hash(msg)
if msg_hash in self._report_exception_cache:
return
self._report_exception_cache.add(msg_hash)
# If we just report the error the traceback will end in the try
# except block that we are calling from.
# We want to show the traceback originating from the module that
# called the Py3 method so get the correct error frame and pass this
# along.
error_frame = sys._getframe(0)
while frame_skip:
error_frame = error_frame.f_back
frame_skip -= 1
self._py3_wrapper.report_exception(msg,
notify_user=False,
error_frame=error_frame)
def error(self, msg, timeout=None):
"""
Raise an error for the module.
:param msg: message to be displayed explaining the error
:param timeout: how long before we should retry. For permanent errors
`py3.CACHE_FOREVER` should be returned. If not supplied then the
modules `cache_timeout` will be used.
"""
raise ModuleErrorException(msg, timeout)
def flatten_dict(self,
d,
delimiter="-",
intermediates=False,
parent_key=None):
"""
Flatten a dictionary.
Values that are dictionaries are flattened using delimiter in between
(eg. parent-child)
Values that are lists are flattened using delimiter
followed by the index (eg. parent-0)
example:
.. code-block:: python
{
'fish_facts': {
'sharks': 'Most will drown if they stop moving',
'skates': 'More than 200 species',
},
'fruits': ['apple', 'peach', 'watermelon'],
'number': 52
}
# becomes
{
'fish_facts-sharks': 'Most will drown if they stop moving',
'fish_facts-skates': 'More than 200 species',
'fruits-0': 'apple',
'fruits-1': 'peach',
'fruits-2': 'watermelon',
'number': 52
}
# if intermediates is True then we also get unflattened elements
# as well as the flattened ones.
{
'fish_facts': {
'sharks': 'Most will drown if they stop moving',
'skates': 'More than 200 species',
},
'fish_facts-sharks': 'Most will drown if they stop moving',
'fish_facts-skates': 'More than 200 species',
'fruits': ['apple', 'peach', 'watermelon'],
'fruits-0': 'apple',
'fruits-1': 'peach',
'fruits-2': 'watermelon',
'number': 52
}
"""
items = []
if isinstance(d, list):
d = dict(enumerate(d))
for k, v in d.items():
if parent_key:
k = u"{}{}{}".format(parent_key, delimiter, k)
if intermediates:
items.append((k, v))
if isinstance(v, list):
v = dict(enumerate(v))
if isinstance(v, collections.Mapping):
items.extend(
self.flatten_dict(v, delimiter, intermediates,
str(k)).items())
else:
items.append((str(k), v))
return dict(items)
def format_units(self, value, unit="B", optimal=5, auto=True, si=False):
"""
Takes a value and formats it for user output, we can choose the unit to
use eg B, MiB, kbits/second. This is mainly for use with bytes/bits it
converts the value into a human readable form. It has various
additional options but they are really only for special cases.
The function returns a tuple containing the new value (this is a number
so that the user can still format it if required) and a unit that is
the units that we have been converted to.
By supplying unit to the function we can force those units to be used
eg ``unit=KiB`` would force the output to be in Kibibytes. By default we
use non-si units but if the unit is si eg kB then we will switch to si
units. Units can also be things like ``Mbit/sec``.
If the auto parameter is False then we use the unit provided. This
only makes sense when the unit is singular eg 'Bytes' and we want the
result in bytes and not say converted to MBytes.
optimal is used to control the size of the output value. We try to
provide an output value of that number of characters (including decimal
point), it may also be less due to rounding. If a fixed unit is used
the output may be more than this number of characters.
"""
UNITS = "KMGTPEZY"
DECIMAL_SIZE = 1000
BINARY_SIZE = 1024
CUTOFF = 1000
can_round = False
if unit:
# try to guess the unit. Do we have a known prefix too it?
if unit[0].upper() in UNITS:
index = UNITS.index(unit[0].upper()) + 1
post = unit[1:]
si = len(unit) > 1 and unit[1] != "i"
if si:
post = post[1:]
auto = False
else:
index = 0
post = unit
if si:
size = DECIMAL_SIZE
else:
size = BINARY_SIZE
if auto:
# we will try to use an appropriate prefix
if value < CUTOFF:
unit_out = post
else:
value /= size
for prefix in UNITS:
if abs(value) < CUTOFF:
break
value /= size
if si:
# si kilo is lowercase
if prefix == "K":
prefix = "k"
else:
post = "i" + post
unit_out = prefix + post
can_round = True
else:
# we are using a fixed unit
unit_out = unit
size = pow(size, index)
if size:
value /= size
can_round = True
if can_round and optimal and value:
# we will try to make the output value the desired size
# we need to keep out value as a numeric type
places = int(log10(abs(value)))
if places >= optimal - 2:
value = int(value)
else:
value = round(value, max(optimal - places - 2, 0))
return value, unit_out
def is_color(self, color):
"""
Tests to see if a color is defined.
Because colors can be set to None in the config and we want this to be
respected in an expression like.
color = self.py3.COLOR_MUTED or self.py3.COLOR_BAD
The color is treated as True but sometimes we want to know if the color
has a value set in which case the color should count as False. This
function is a helper for this second case.
"""
return not (color is None or hasattr(color, "none_setting"))
def i3s_config(self):
"""
returns the i3s_config dict.
"""
return self._i3s_config
def is_python_2(self):
"""
True if the version of python being used is 2.x
Can be helpful for fixing python 2 compatibility issues
"""
return self._is_python_2
def is_my_event(self, event):
"""
Checks if an event triggered belongs to the module receiving it. This
is mainly for containers who will also receive events from any children
they have.
Returns True if the event name and instance match that of the module
checking.
"""
return (event.get("name") == self._module.module_name
and event.get("instance") == self._module.module_inst)
def log(self, message, level=LOG_INFO):
"""
Log the message.
The level must be one of LOG_ERROR, LOG_INFO or LOG_WARNING
"""
assert level in [
self.LOG_ERROR,
self.LOG_INFO,
self.LOG_WARNING,
], "level must be LOG_ERROR, LOG_INFO or LOG_WARNING"
# nicely format logs if we can using pretty print
if isinstance(message, (dict, list, set, tuple)):
message = pformat(message)
# start on new line if multi-line output
try:
if "\n" in message:
message = "\n" + message
except: # noqa e722
pass
message = "Module `{}`: {}".format(self._module.module_full_name,
message)
self._py3_wrapper.log(message, level)
def update(self, module_name=None):
"""
Update a module. If module_name is supplied the module of that
name is updated. Otherwise the module calling is updated.
"""
if not module_name:
return self._module.force_update()
else:
module_info = self._get_module_info(module_name)
if module_info:
module_info["module"].force_update()
def get_output(self, module_name):
"""
Return the output of the named module. This will be a list.
"""
output = []
module_info = self._get_module_info(module_name)
if module_info:
output = module_info["module"].get_latest()
# we do a deep copy so that any user does not change the actual output
# of the module.
return deepcopy(output)
def trigger_event(self, module_name, event):
"""
Trigger an event on a named module.
"""
if module_name:
self._py3_wrapper.events_thread.process_event(module_name, event)
def prevent_refresh(self):
"""
Calling this function during the on_click() method of a module will
request that the module is not refreshed after the event. By default
the module is updated after the on_click event has been processed.
"""
self._module.prevent_refresh = True
def notify_user(self,
msg,
level="info",
rate_limit=5,
title=None,
icon=None):
"""
Send a notification to the user.
level must be 'info', 'error' or 'warning'.
rate_limit is the time period in seconds during which this message
should not be repeated.
icon must be an icon path or icon name.
"""
module_name = self._module.module_full_name
if isinstance(msg, Composite):
msg = msg.text()
if title is None:
title = "py3status: {}".format(module_name)
elif isinstance(title, Composite):
title = title.text()
# force unicode for python2 str
if self._is_python_2:
if isinstance(msg, str):
msg = msg.decode("utf-8")
if isinstance(title, str):
title = title.decode("utf-8")
if msg:
self._py3_wrapper.notify_user(
msg=msg,
level=level,
rate_limit=rate_limit,
module_name=module_name,
title=title,
icon=icon,
)
def register_function(self, function_name, function):
"""
Register a function for the module.
The following functions can be registered
.. py:function:: content_function()
Called to discover what modules a container is displaying. This is
used to determine when updates need passing on to the container and
also when modules can be put to sleep.
the function must return a set of module names that are being
displayed.
.. note::
This function should only be used by containers.
.. py:function:: urgent_function(module_names)
This function will be called when one of the contents of a container
has changed from a non-urgent to an urgent state. It is used by the
group module to switch to displaying the urgent module.
``module_names`` is a list of modules that have become urgent
.. note::
This function should only be used by containers.
"""
my_info = self._get_module_info(self._module.module_full_name)
my_info[function_name] = function
def time_in(self, seconds=None, sync_to=None, offset=0):
"""
Returns the time a given number of seconds into the future. Helpful
for creating the ``cached_until`` value for the module output.
.. note::
from version 3.1 modules no longer need to explicitly set a
``cached_until`` in their response unless they wish to directly control
it.
:param seconds: specifies the number of seconds that should occur before the
update is required. Passing a value of ``CACHE_FOREVER`` returns
``CACHE_FOREVER`` which can be useful for some modules.
:param sync_to: causes the update to be synchronized to a time period. 1 would
cause the update on the second, 60 to the nearest minute. By default we
synchronize to the nearest second. 0 will disable this feature.
:param offset: is used to alter the base time used. A timer that started at a
certain time could set that as the offset and any synchronization would
then be relative to that time.
"""
# if called with CACHE_FOREVER we just return this
if seconds is self.CACHE_FOREVER:
return self.CACHE_FOREVER
if seconds is None:
# If we have a sync_to then seconds can be 0
if sync_to and sync_to > 0:
seconds = 0
else:
try:
# use py3status modules cache_timeout
seconds = self._py3status_module.cache_timeout
except AttributeError:
# use default cache_timeout
seconds = self._module.config["cache_timeout"]
# Unless explicitly set we sync to the nearest second
# Unless the requested update is in less than a second
if sync_to is None:
if seconds and seconds < 1:
if 1 % seconds == 0:
sync_to = seconds
else:
sync_to = 0
else:
sync_to = 1
if seconds:
seconds -= 0.1
requested = time() + seconds - offset
# if sync_to then we find the sync time for the requested time
if sync_to:
requested = (requested + sync_to) - (requested % sync_to)
return requested + offset
def format_contains(self, format_string, names):
"""
Determines if ``format_string`` contains a placeholder string ``names``
or a list of placeholders ``names``.
``names`` is tested against placeholders using fnmatch so the following
patterns can be used:
.. code-block:: none
* matches everything
? matches any single character
[seq] matches any character in seq
[!seq] matches any character not in seq
This is useful because a simple test like
``'{placeholder}' in format_string``
will fail if the format string contains placeholder formatting
eg ``'{placeholder:.2f}'``
"""
# We cache things to prevent parsing the format_string more than needed
if isinstance(names, list):
key = str(names)
else:
key = names
names = [names]
try:
return self._format_placeholders_cache[format_string][key]
except KeyError:
pass
if format_string not in self._format_placeholders:
placeholders = self._formatter.get_placeholders(format_string)
self._format_placeholders[format_string] = placeholders
else:
placeholders = self._format_placeholders[format_string]
if format_string not in self._format_placeholders_cache:
self._format_placeholders_cache[format_string] = {}
for name in names:
for placeholder in placeholders:
if fnmatch(placeholder, name):
self._format_placeholders_cache[format_string][key] = True
return True
self._format_placeholders_cache[format_string][key] = False
return False
def get_color_names_list(self, format_strings):
"""
Returns a list of color names in ``format_string``.
:param format_strings: Accepts a format string or a list of format strings.
"""
if not format_strings:
return []
if not getattr(self._py3status_module, "thresholds", None):
return []
if isinstance(format_strings, basestring):
format_strings = [format_strings]
names = set()
for string in format_strings:
for color in string.replace("&", " ").split("color=")[1::1]:
color = color.split()[0]
if "#" in color:
continue
if color in ["good", "bad", "degraded", "None", "threshold"]:
continue
if color in COLOR_NAMES:
continue
names.add(color)
return list(names)
def get_placeholders_list(self, format_string, match=None):
"""
Returns a list of placeholders in ``format_string``.
If ``match`` is provided then it is used to filter the result using
fnmatch so the following patterns can be used:
.. code-block:: none
* matches everything
? matches any single character
[seq] matches any character in seq
[!seq] matches any character not in seq
This is useful because we just get simple placeholder without any
formatting that may be applied to them
eg ``'{placeholder:.2f}'`` will give ``['{placeholder}']``
"""
if format_string not in self._format_placeholders:
placeholders = self._formatter.get_placeholders(format_string)
self._format_placeholders[format_string] = placeholders
else:
placeholders = self._format_placeholders[format_string]
if not match:
return list(placeholders)
# filter matches
found = []
for placeholder in placeholders:
if fnmatch(placeholder, match):
found.append(placeholder)
return found
def get_placeholder_formats_list(self, format_string):
"""
Parses the format_string and returns a list of tuples
[(placeholder, format), ...].
eg ``'{placeholder:.2f}'`` will give ``[('placeholder', ':.2f')]``
"""
return self._formatter.get_placeholder_formats_list(format_string)
def update_placeholder_formats(self, format_string, formats):
"""
Update a format string adding formats if they are not already present.
This is useful when for example a placeholder has a floating point
value but by default we only want to show it to a certain precision.
"""
return self._formatter.update_placeholder_formats(
format_string, formats)
def safe_format(self,
format_string,
param_dict=None,
force_composite=False,
attr_getter=None):
r"""
Parser for advanced formatting.
Unknown placeholders will be shown in the output eg ``{foo}``.
Square brackets ``[]`` can be used. The content of them will be removed
from the output if there is no valid placeholder contained within.
They can also be nested.
A pipe (vertical bar) ``|`` can be used to divide sections the first
valid section only will be shown in the output.
A backslash ``\`` can be used to escape a character eg ``\[`` will show ``[``
in the output.
``\?`` is special and is used to provide extra commands to the format
string, example ``\?color=#FF00FF``. Multiple commands can be given
using an ampersand ``&`` as a separator, example ``\?color=#FF00FF&show``.
``\?if=<placeholder>`` can be used to check if a placeholder exists. An
exclamation mark ``!`` after the equals sign ``=`` can be used to negate
the condition.
``\?if=<placeholder>=<value>`` can be used to determine if {<placeholder>}
would be replaced with <value>. ``[]`` in <value> don't need to be escaped.
``{<placeholder>}`` will be converted, or removed if it is None or empty.
Formatting can also be applied to the placeholder Eg
``{number:03.2f}``.
example format_string:
``"[[{artist} - ]{title}]|{file}"``
This will show ``artist - title`` if artist is present,
``title`` if title but no artist,
and ``file`` if file is present but not artist or title.
param_dict is a dictionary of placeholders that will be substituted.
If a placeholder is not in the dictionary then if the py3status module
has an attribute with the same name then it will be used.
.. note::
Added in version 3.3
Composites can be included in the param_dict.
The result returned from this function can either be a string in the
case of simple parsing or a Composite if more complex.
If force_composite parameter is True a composite will always be
returned.
attr_getter is a function that will when called with an attribute name
as a parameter will return a value.
"""
try:
return self._formatter.format(
format_string,
self._py3status_module,
param_dict,
force_composite=force_composite,
attr_getter=attr_getter,
)
except Exception:
self._report_exception(
u"Invalid format `{}`".format(format_string))
return "invalid format"
def build_composite(self,
format_string,
param_dict=None,
composites=None,
attr_getter=None):
"""
.. note::
deprecated in 3.3 use safe_format().
Build a composite output using a format string.
Takes a format_string and treats it the same way as ``safe_format()`` but
also takes a composites dict where each key/value is the name of the
placeholder and either an output eg ``{'full_text': 'something'}`` or a
list of outputs.
"""
if param_dict is None:
param_dict = {}
# merge any composites into the param_dict.
# as they are no longer dealt with separately
if composites:
for key, value in composites.items():
param_dict[key] = Composite(value)
try:
return self._formatter.format(
format_string,
self._py3status_module,
param_dict,
force_composite=True,
attr_getter=attr_getter,
)
except Exception:
self._report_exception(
u"Invalid format `{}`".format(format_string))
return [{"full_text": "invalid format"}]
def composite_update(self, item, update_dict, soft=False):
"""
Takes a Composite (item) if item is a type that can be converted into a
Composite then this is done automatically. Updates all entries it the
Composite with values from update_dict. Updates can be soft in which
case existing values are not overwritten.
A Composite object will be returned.
"""
return Composite.composite_update(item, update_dict, soft)
def composite_join(self, separator, items):
"""
Join a list of items with a separator.
This is used in joining strings, responses and Composites.
A Composite object will be returned.
"""
return Composite.composite_join(separator, items)
def composite_create(self, item):
"""
Create and return a Composite.
The item may be a string, dict, list of dicts or a Composite.
"""
return Composite(item)
def is_composite(self, item):
"""
Check if item is a Composite and return True if it is.
"""
return isinstance(item, Composite)
def get_composite_string(self, format_string):
"""
Return a string from a Composite.
"""
if not isinstance(format_string, Composite):
return ""
return format_string.text()
def check_commands(self, cmd_list):
"""
Checks to see if commands in list are available using ``which``.
returns the first available command.
If a string is passed then that command will be checked for.
"""
# if a string is passed then convert it to a list. This prevents an
# easy mistake that could be made
if isinstance(cmd_list, basestring):
cmd_list = [cmd_list]
for cmd in cmd_list:
if self.command_run("which {}".format(cmd)) == 0:
return cmd
def command_run(self, command):
"""
Runs a command and returns the exit code.
The command can either be supplied as a sequence or string.
An Exception is raised if an error occurs
"""
# convert the command to sequence if a string
if isinstance(command, basestring):
command = shlex.split(command)
try:
return Popen(command, stdout=PIPE, stderr=PIPE,
close_fds=True).wait()
except Exception as e:
# make a pretty command for error loggings and...
if isinstance(command, basestring):
pretty_cmd = command
else:
pretty_cmd = " ".join(command)
msg = "Command `{cmd}` {error}".format(cmd=pretty_cmd,
error=e.errno)
raise exceptions.CommandError(msg, error_code=e.errno)
def command_output(self,
command,
shell=False,
capture_stderr=False,
localized=False):
"""
Run a command and return its output as unicode.
The command can either be supplied as a sequence or string.
:param command: command to run can be a str or list
:param shell: if `True` then command is run through the shell
:param capture_stderr: if `True` then STDERR is piped to STDOUT
:param localized: if `False` then command is forced to use its default (English) locale
A CommandError is raised if an error occurs
"""
# make a pretty command for error loggings and...
if isinstance(command, basestring):
pretty_cmd = command
else:
pretty_cmd = " ".join(command)
# convert the non-shell command to sequence if it is a string
if not shell and isinstance(command, basestring):
command = shlex.split(command)
stderr = STDOUT if capture_stderr else PIPE
env = self._english_env if not localized else None
try:
process = Popen(
command,
stdout=PIPE,
stderr=stderr,
close_fds=True,
universal_newlines=True,
shell=shell,
env=env,
)
except Exception as e:
msg = "Command `{cmd}` {error}".format(cmd=pretty_cmd, error=e)
raise exceptions.CommandError(msg, error_code=e.errno)
output, error = process.communicate()
if self._is_python_2 and isinstance(output, str):
output = output.decode("utf-8")
error = error.decode("utf-8")
retcode = process.poll()
if retcode:
# under certain conditions a successfully run command may get a
# return code of -15 even though correct output was returned see
# #664. This issue seems to be related to arch linux but the
# reason is not entirely clear.
if retcode == -15:
msg = "Command `{cmd}` returned SIGTERM (ignoring)"
self.log(msg.format(cmd=pretty_cmd))
else:
msg = "Command `{cmd}` returned non-zero exit status {error}"
output_oneline = output.replace("\n", " ")
if output_oneline:
msg += " ({output})"
msg = msg.format(cmd=pretty_cmd,
error=retcode,
output=output_oneline)
raise exceptions.CommandError(msg,
error_code=retcode,
error=error,
output=output)
return output
def _storage_init(self):
"""
Ensure that storage is initialized.
"""
if not self._storage.initialized:
self._storage.init(self._module._py3_wrapper, self._is_python_2)
def storage_set(self, key, value):
"""
Store a value for the module.
"""
if not self._module:
return
self._storage_init()
module_name = self._module.module_full_name
return self._storage.storage_set(module_name, key, value)
def storage_get(self, key):
"""
Retrieve a value for the module.
"""
if not self._module:
return
self._storage_init()
module_name = self._module.module_full_name
return self._storage.storage_get(module_name, key)
def storage_del(self, key=None):
"""
Remove the value stored with the key from storage.
If key is not supplied then all values for the module are removed.
"""
if not self._module:
return
self._storage_init()
module_name = self._module.module_full_name
return self._storage.storage_del(module_name, key=key)
def storage_keys(self):
"""
Return a list of the keys for values stored for the module.
Keys will contain the following metadata entries:
- '_ctime': storage creation timestamp
- '_mtime': storage last modification timestamp
"""
if not self._module:
return []
self._storage_init()
module_name = self._module.module_full_name
return self._storage.storage_keys(module_name)
def storage_items(self):
"""
Return key, value pairs of the stored data for the module.
Keys will contain the following metadata entries:
- '_ctime': storage creation timestamp
- '_mtime': storage last modification timestamp
"""
if not self._module:
return {}.items()
self._storage_init()
items = []
module_name = self._module.module_full_name
for key in self._storage.storage_keys(module_name):
value = self._storage.storage_get(module_name, key)
items.add((key, value))
return items
def play_sound(self, sound_file):
"""
Plays sound_file if possible.
"""
self.stop_sound()
cmd = self.check_commands(["ffplay", "paplay", "play"])
if cmd:
if cmd == "ffplay":
cmd = "ffplay -autoexit -nodisp -loglevel 0"
sound_file = os.path.expanduser(sound_file)
c = shlex.split("{} {}".format(cmd, sound_file))
self._audio = Popen(c)
def stop_sound(self):
"""
Stops any currently playing sounds for this module.
"""
if self._audio:
self._audio.kill()
self._audio = None
def threshold_get_color(self, value, name=None):
"""
Obtain color for a value using thresholds.
The value will be checked against any defined thresholds. These should
have been set in the i3status configuration. If more than one
threshold is needed for a module then the name can also be supplied.
If the user has not supplied a named threshold but has defined a
general one that will be used.
If the gradients config parameter is True then rather than sharp
thresholds we will use a gradient between the color values.
:param value: numerical value to be graded
:param name: accepts a string, otherwise 'threshold'
accepts 3-tuples to allow name with different
values eg ('name', 'key', 'thresholds')
"""
# If first run then process the threshold data.
if self._thresholds is None:
self._thresholds_init()
# allow name with different values
if isinstance(name, tuple):
name_used = "{}/{}".format(name[0], name[1])
if name[2]:
self._thresholds[name_used] = [(x[0], self._get_color(x[1]))
for x in name[2]]
name = name[0]
else:
# if name not in thresholds info then use defaults
name_used = name
if name_used not in self._thresholds:
name_used = None
# convert value to int/float
thresholds = self._thresholds.get(name_used)
color = None
try:
value = float(value)
except (TypeError, ValueError):
pass
# skip on empty thresholds/values
if not thresholds or value in [None, ""]:
pass
elif isinstance(value, basestring):
# string
for threshold in thresholds:
if value == threshold[0]:
color = threshold[1]
break
else:
# int/float
try:
if self._get_config_setting("gradients"):
try:
colors, minimum, maximum = self._threshold_gradients[
name_used]
except KeyError:
colors = self._gradients.make_threshold_gradient(
self, thresholds)
minimum = min(thresholds)[0]
maximum = max(thresholds)[0]
self._threshold_gradients[name_used] = (
colors,
minimum,
maximum,
)
if value < minimum:
color = colors[0]
elif value > maximum:
color = colors[-1]
else:
value -= minimum
col_index = int(
((len(colors) - 1) / (maximum - minimum)) * value)
color = colors[col_index]
else:
color = thresholds[0][1]
for threshold in thresholds:
if value >= threshold[0]:
color = threshold[1]
else:
break
except TypeError:
color = None
# save color so it can be accessed via safe_format()
if name:
color_name = "color_threshold_%s" % name
else:
color_name = "color_threshold"
setattr(self._py3status_module, color_name, color)
return color
def request(
self,
url,
params=None,
data=None,
headers=None,
timeout=None,
auth=None,
cookiejar=None,
):
"""
Make a request to a url and retrieve the results.
If the headers parameter does not provide an 'User-Agent' key, one will
be added automatically following the convention:
py3status/<version> <per session random uuid>
:param url: url to request eg `http://example.com`
:param params: extra query string parameters as a dict
:param data: POST data as a dict. If this is not supplied the GET method will be used
:param headers: http headers to be added to the request as a dict
:param timeout: timeout for the request in seconds
:param auth: authentication info as tuple `(username, password)`
:param cookiejar: an object of a CookieJar subclass
:returns: HttpResponse
"""
# The aim of this function is to be a limited lightweight replacement
# for the requests library but using only pythons standard libs.
# IMPORTANT NOTICE
# This function is excluded from private variable hiding as it is
# likely to need api keys etc which people may have obfuscated.
# Therefore it is important that no logging is done in this function
# that might reveal this information.
if headers is None:
headers = {}
if timeout is None:
timeout = getattr(self._py3status_module, "request_timeout", 10)
if "User-Agent" not in headers:
headers["User-Agent"] = "py3status/{} {}".format(
version, self._uid)
return HttpResponse(
url,
params=params,
data=data,
headers=headers,
timeout=timeout,
auth=auth,
cookiejar=cookiejar,
)
class BaseTestServer(object):
def __init__(self,
host='localhost',
port=None,
cwd=None,
shell=False,
stdin=DEVNULL,
stdout=DEVNULL,
stderr=DEVNULL):
self.host = host
self.port = port or port_for.select_random()
self.proc = None
self.shell = shell
self.cwd = cwd
self.stdin = stdin
self.stdout = stdout
self.stderr = stderr
self.arguments = [
sys.executable, '-u', '-m', 'SimpleHTTPServer',
str(self.port)
]
def start(self):
self.proc = Popen(self.arguments,
stdin=self.stdin,
stdout=self.stdout,
stderr=self.stderr,
shell=self.shell,
cwd=self.cwd,
env=get_testenv())
self.proc.poll()
if self.proc.returncode:
msg = (
"unable to start server. error code: %d - stderr follows: \n%s"
) % (self.proc.returncode, self.proc.stderr.read())
raise RuntimeError(msg)
try:
self._wait_for_port()
finally:
print(self._non_block_read(self.proc.stderr))
pass
def stop(self):
if self.proc is None:
raise RuntimeError("Server wasn't started")
self.proc.kill()
self.proc.wait()
self.proc = None
def __enter__(self):
self.start()
return self
def __exit__(self, exc_type, exc_val, exc_tb):
self.stop()
def url(self, path=''):
return urljoin('http://{}:{}'.format(self.host, self.port), path)
def _wait_for_port(self, delay=0.1, attempts=20):
"""Imports could take some time, server opens port with some delay."""
while attempts > 0:
s = socket.socket()
try:
s.connect((self.host, self.port))
except Exception:
time.sleep(delay)
attempts -= 1
else:
return
finally:
s.close()
raise RuntimeError("Port %d is not open" % self.port)
@staticmethod
def _non_block_read(output):
if output is None:
return ''
fd = output.fileno()
fl = fcntl.fcntl(fd, fcntl.F_GETFL)
fcntl.fcntl(fd, fcntl.F_SETFL, fl | os.O_NONBLOCK)
try:
return output.read()
except Exception:
return ''
class P2P:
"""
Forks a child process and executes p2pd command with given arguments.
Can be used for peer to peer communication and procedure calls.
Sends SIGKILL to the child in destructor.
"""
HEADER_LEN = 8
BYTEORDER = 'big'
PB_HEADER_LEN = 1
RESULT_MESSAGE = b'\x00'
ERROR_MESSAGE = b'\x01'
DHT_MODE_MAPPING = {
'dht': {
'dht': 1
},
'dht_server': {
'dhtServer': 1
},
'dht_client': {
'dhtClient': 1
},
}
FORCE_REACHABILITY_MAPPING = {
'public': {
'forceReachabilityPublic': 1
},
'private': {
'forceReachabilityPrivate': 1
},
}
def __init__(self):
self._child = None
self._alive = False
self._listen_task = None
self._server_stopped = asyncio.Event()
@classmethod
async def create(cls,
*args,
quic: bool = True,
tls: bool = True,
conn_manager: bool = True,
dht_mode: str = 'dht_server',
force_reachability: Optional[str] = None,
nat_port_map: bool = True,
auto_nat: bool = True,
bootstrap: bool = False,
bootstrap_peers: Optional[List[str]] = None,
use_global_ipfs: bool = False,
host_port: int = None,
daemon_listen_port: int = None,
**kwargs):
"""
Start a new p2pd process and connect to it.
:param args:
:param quic: Enables the QUIC transport
:param tls: Enables TLS1.3 channel security protocol
:param conn_manager: Enables the Connection Manager
:param dht_mode: DHT mode (dht_client/dht_server/dht)
:param force_reachability: Force reachability mode (public/private)
:param nat_port_map: Enables NAT port mapping
:param auto_nat: Enables the AutoNAT service
:param bootstrap: Connects to bootstrap peers and bootstraps the dht if enabled
:param bootstrap_peers: List of bootstrap peers; defaults to the IPFS DHT peers
:param use_global_ipfs: Bootstrap to global ipfs (works only if bootstrap=True and bootstrap_peers=None)
:param host_port: port for p2p network
:param daemon_listen_port: port for connection daemon and client binding
:param kwargs:
:return: new wrapper for p2p daemon
"""
assert not (bootstrap and bootstrap_peers is None and not use_global_ipfs), \
'Trying to create with bootstrap node without bootstrap nodes list. ' \
'It is very dangerous, because p2pd connects to global ipfs and it is very unstable. ' \
'If you really want this, pass use_global_ipfs=True'
assert not (bootstrap_peers is not None and use_global_ipfs), \
'Non empty bootstrap_nodes and use_global_ipfs=True are incompatible.' \
'Choose one option: your nodes list (preferable) or global ipfs (very unstable)'
self = cls()
with path(cli, P2PD_FILENAME) as p:
p2pd_path = p
bootstrap_peers = cls._make_bootstrap_peers(bootstrap_peers)
dht = cls.DHT_MODE_MAPPING.get(dht_mode, {'dht': 0})
force_reachability = cls.FORCE_REACHABILITY_MAPPING.get(
force_reachability, {})
proc_args = self._make_process_args(str(p2pd_path),
*args,
quic=quic,
tls=tls,
connManager=conn_manager,
natPortMap=nat_port_map,
autonat=auto_nat,
b=bootstrap,
**{
**bootstrap_peers,
**dht,
**force_reachability,
**kwargs
})
self._assign_daemon_ports(host_port, daemon_listen_port)
for try_count in range(NUM_RETRIES):
try:
self._initialize(proc_args)
await self._wait_for_client(RETRY_DELAY * (2**try_count))
break
except Exception as e:
logger.debug(f"Failed to initialize p2p daemon: {e}")
self._terminate()
if try_count == NUM_RETRIES - 1:
raise
self._assign_daemon_ports()
return self
@classmethod
async def replicate(cls, daemon_listen_port: int, host_port: int):
"""
Connect to existing p2p daemon
:param daemon_listen_port: port for connection daemon and client binding
:param host_port: port for p2p network
:return: new wrapper for existing p2p daemon
"""
self = cls()
# There is no child under control
# Use external already running p2pd
self._child = None
self._alive = True
self._assign_daemon_ports(host_port, daemon_listen_port)
self._client_listen_port = find_open_port()
self._client = p2pclient.Client(
Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'),
Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}'))
await self._wait_for_client()
return self
async def wait_for_at_least_n_peers(self, n_peers, attempts=3, delay=1):
for _ in range(attempts):
peers = await self._client.list_peers()
if len(peers) >= n_peers:
return
await asyncio.sleep(delay)
raise RuntimeError('Not enough peers')
def _initialize(self, proc_args: List[str]) -> None:
proc_args = deepcopy(proc_args)
proc_args.extend(
self._make_process_args(
hostAddrs=
f'/ip4/0.0.0.0/tcp/{self._host_port},/ip4/0.0.0.0/udp/{self._host_port}/quic',
listen=f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'))
self._child = Popen(args=proc_args, encoding="utf8")
self._alive = True
self._client_listen_port = find_open_port()
self._client = p2pclient.Client(
Multiaddr(f'/ip4/127.0.0.1/tcp/{self._daemon_listen_port}'),
Multiaddr(f'/ip4/127.0.0.1/tcp/{self._client_listen_port}'))
async def _wait_for_client(self, delay=0):
await asyncio.sleep(delay)
encoded = await self._client.identify()
self.id = encoded[0].to_base58()
def _assign_daemon_ports(self, host_port=None, daemon_listen_port=None):
if host_port is None:
host_port = find_open_port()
if daemon_listen_port is None:
daemon_listen_port = find_open_port()
while daemon_listen_port == host_port:
daemon_listen_port = find_open_port()
self._host_port, self._daemon_listen_port = host_port, daemon_listen_port
@staticmethod
async def send_raw_data(byte_str, writer):
request = len(byte_str).to_bytes(P2P.HEADER_LEN,
P2P.BYTEORDER) + byte_str
writer.write(request)
@staticmethod
async def send_msgpack(data, writer):
raw_data = MSGPackSerializer.dumps(data)
await P2P.send_raw_data(raw_data, writer)
@staticmethod
async def send_protobuf(protobuf, out_proto_type, writer):
if type(protobuf) != out_proto_type:
raise TypeError('Unary handler returned protobuf of wrong type.')
if out_proto_type == p2pd_pb2.RPCError:
await P2P.send_raw_data(P2P.ERROR_MESSAGE, writer)
else:
await P2P.send_raw_data(P2P.RESULT_MESSAGE, writer)
await P2P.send_raw_data(protobuf.SerializeToString(), writer)
@staticmethod
async def receive_raw_data(reader: asyncio.StreamReader,
header_len=HEADER_LEN):
header = await reader.readexactly(header_len)
content_length = int.from_bytes(header, P2P.BYTEORDER)
data = await reader.readexactly(content_length)
return data
@staticmethod
async def receive_msgpack(reader):
return MSGPackSerializer.loads(await P2P.receive_raw_data(reader))
@staticmethod
async def receive_protobuf(in_proto_type, reader):
msg_type = await P2P.receive_raw_data(reader)
if msg_type == P2P.RESULT_MESSAGE:
protobuf = in_proto_type()
protobuf.ParseFromString(await P2P.receive_raw_data(reader))
return protobuf, None
elif msg_type == P2P.ERROR_MESSAGE:
protobuf = p2pd_pb2.RPCError()
protobuf.ParseFromString(await P2P.receive_raw_data(reader))
return None, protobuf
else:
raise TypeError('Invalid Protobuf message type')
@staticmethod
def _handle_stream(handle):
async def do_handle_stream(stream_info, reader, writer):
try:
request = await P2P.receive_raw_data(reader)
except asyncio.IncompleteReadError:
logger.debug(
"Incomplete read while receiving request from peer")
writer.close()
return
try:
result = handle(request)
await P2P.send_raw_data(result, writer)
finally:
writer.close()
return do_handle_stream
@staticmethod
def _handle_unary_stream(handle, context, in_proto_type, out_proto_type):
async def watchdog(reader: asyncio.StreamReader):
await reader.read(n=1)
raise P2PInterruptedError()
async def do_handle_unary_stream(stream_info: StreamInfo,
reader: asyncio.StreamReader,
writer: asyncio.StreamWriter) -> None:
try:
try:
request = await P2P.receive_protobuf(in_proto_type, reader)
except asyncio.IncompleteReadError:
logger.debug(
"Incomplete read while receiving request from peer")
return
except google.protobuf.message.DecodeError as error:
logger.exception(error)
return
context.peer_id, context.peer_addr = stream_info.peer_id, stream_info.addr
done, pending = await asyncio.wait(
[watchdog(reader),
handle(request, context)],
return_when=asyncio.FIRST_COMPLETED)
try:
result = done.pop().result()
await P2P.send_protobuf(result, out_proto_type, writer)
except P2PInterruptedError:
pass
except Exception as exc:
error = p2pd_pb2.RPCError(message=str(exc))
await P2P.send_protobuf(error, p2pd_pb2.RPCError, writer)
finally:
pending_task = pending.pop()
pending_task.cancel()
try:
await pending_task
except asyncio.CancelledError:
pass
finally:
writer.close()
return do_handle_unary_stream
def start_listening(self):
async def listen():
async with self._client.listen():
await self._server_stopped.wait()
self._listen_task = asyncio.create_task(listen())
async def stop_listening(self):
if self._listen_task is not None:
self._server_stopped.set()
self._listen_task.cancel()
try:
await self._listen_task
except asyncio.CancelledError:
self._listen_task = None
self._server_stopped.clear()
async def add_stream_handler(self, name, handle):
if self._listen_task is None:
self.start_listening()
await self._client.stream_handler(name, self._handle_stream(handle))
async def add_unary_handler(self, name, handle, in_proto_type,
out_proto_type):
if self._listen_task is None:
self.start_listening()
context = P2PContext(id=self.id,
port=self._host_port,
handle_name=name)
await self._client.stream_handler(
name,
P2P._handle_unary_stream(handle, context, in_proto_type,
out_proto_type))
async def call_peer_handler(self, peer_id, handler_name, input_data):
libp2p_peer_id = PeerID.from_base58(peer_id)
stream_info, reader, writer = await self._client.stream_open(
libp2p_peer_id, (handler_name, ))
try:
await P2P.send_raw_data(input_data, writer)
return await P2P.receive_raw_data(reader)
finally:
writer.close()
def __del__(self):
self._terminate()
@property
def is_alive(self):
return self._alive
async def shutdown(self):
await asyncio.get_event_loop().run_in_executor(None, self._terminate)
def _terminate(self):
self._alive = False
if self._child is not None and self._child.poll() is None:
self._child.kill()
self._child.wait()
@staticmethod
def _make_process_args(*args, **kwargs) -> List[str]:
proc_args = []
proc_args.extend(str(entry) for entry in args)
proc_args.extend(f'-{key}={P2P._convert_process_arg_type(value)}'
if value is not None else f'-{key}'
for key, value in kwargs.items())
return proc_args
@staticmethod
def _convert_process_arg_type(val):
if isinstance(val, bool):
return 1 if val else 0
return val
@staticmethod
def _make_bootstrap_peers(nodes):
if nodes is None:
return {}
return {'bootstrapPeers': ','.join(nodes)}
from subprocess import Popen
from time import sleep
p = Popen("gedit")
sleep(15)
p.kill()
# p.terminate()
class Airodump(Air):
"""
TODO
This accepts the following parameters from airodump-ng's help.
* ivs
* gpsd
* beacons
* manufacturer
* uptime
* ignore_negative_one
* a
* showack
* h
* f
* update
* berlin
* r
* x
* encrypt
* netmask
* bssid
* essid
* output_format
* write
* essid_regex
"""
_aps = []
_clients = []
_stop = False
_allowed_arguments = (
('ivs', False),
('gpsd', False),
('beacons', False),
('manufacturer', False),
('uptime', False),
('ignore_negative_one', False),
('a', False),
('showack', False),
('h', False),
('f', False),
('update', False),
('berlin', False),
('r', False),
('x', False),
('encrypt', False),
('netmask', False),
('bssid', False),
('essid', False),
('output_format', False),
('write', False),
('essid_regex', False),
('cswitch', False),
('channel', False),
)
def __init__(self, interface=False, **kwargs):
self.interface = interface
super(self.__class__, self).__init__(**kwargs)
@property
def tree(self):
"""
Returns currently reported aps
"""
keys = [
'FirstTimeSeen', 'LastTimeSeen', 'channel', 'Speed', 'Privacy',
'Cipher', 'Authentication', 'Power', 'beacons', 'IV', 'LANIP',
'IDlength', 'ESSID', 'Key'
]
c_keys = [
'Station MAC', 'FirstTimeSeen', 'LastTimeSeen', 'Power', 'Packets',
'BSSID', 'ProbedESSIDs'
]
self.update_results()
aps = {}
for ap_ in self._aps:
bssid = ap_.pop(0)
aps[bssid] = dict(zip(keys, ap_))
aps[bssid]['clients'] = []
for client in self.clients:
if client[0] == bssid:
aps[bssid]['clients'].append(dict(zip(c_keys, client)))
return aps
@property
def clients(self):
"""
Returns currently reported clients
"""
self.update_results()
return self._clients
def scan(self):
"""
Get next result: implement in childrens
Both this and previous one must be
responsible for duplicates
"""
self.start()
while not os.path.exists(self.curr_csv):
time.sleep(5)
def watch_process(self):
"""
Watcher thread.
This one relaunches airodump eatch time it dies until
we call stop()
"""
psutil.wait_procs([psutil.Process(self._proc.pid)],
callback=self.start)
def start(self, _=False):
"""
Start process.
psutil sends an argument (that we don't actually need...)
interface defaults to monitor interface 0 as started by Airmon
"""
if not self._stop:
self._current_execution += 1
flags = self.flags
if '--write' not in flags:
flags.extend(['--write', self.writepath])
if '--output-format' not in flags:
flags.extend(['--output-format', 'csv'])
line = ["airodump-ng"] + flags + self.arguments + [self.interface]
self._proc = Popen(line,
bufsize=0,
env={'PATH': os.environ['PATH']},
stderr=DEVNULL,
stdin=DEVNULL,
stdout=DEVNULL)
os.system('stty sane')
time.sleep(5)
watcher = threading.Thread(target=self.watch_process)
watcher.start()
def stop(self):
"""
Stop proc.
"""
self._stop = True
return self._proc.kill()
def update_results(self):
"""
Updates self.clients and self.aps
"""
clis = []
aps = []
with open(self.curr_csv) as fileo:
file_ = fileo.readlines()
file_enum = enumerate(file_)
num = 0
for num, line in file_enum:
line = line.replace('\0', '')
if not line:
continue
if line.startswith('BSSID'):
continue
if line.startswith('Station'):
num += 1
break
aps.append(line)
for line in file_[num:]:
clis.append(line)
def clean_rows(reader):
"""
Airodump-ng's csv info comes a bit unclean.
Strip each line of its extra blank spaces
"""
return [[a.strip() for a in row] for row in reader if row]
self._aps = clean_rows(csv.reader(StringIO('\n'.join(aps))))
self._clients = clean_rows(csv.reader(StringIO('\n'.join(clis))))
def gradeGroup(group_dir, question_ID):
#student_question_ID = student_ID + question_ID
os.chdir("./Groups/"+group_dir)
#-----------------------------------------------------
# Edit the .bochsrc-no-gui, replace the partport1 line to this question timestamped output file
# We do this to make sure that Autograde doesn't mistakenly take the same file for the next question
#-----------------------------------------------------
# fraction of a second time stamp
QTimestamp = str(time.time()).replace(".","")
# This question specific student_stdout file name
student_stdout_default = "student_stdout"
student_stdout_Q_specific = student_stdout_default+"_"+QTimestamp
bochsrcDefault = open(os.getcwd()+"/.bochsrc-no-gui", 'r')
bochsrcAutograde = open(os.getcwd()+"/.bochsrc_autograde_"+QTimestamp, 'w')
for line in bochsrcDefault:
bochsrcAutograde.write(line.replace(student_stdout_default, student_stdout_Q_specific))
bochsrcDefault.close()
bochsrcAutograde.close()
#-----------------------------------------------------
# TODO: Improvment: If a compilation error occurs to a student folder, let the script save the output
# of "make" into a disk file (just like it does for Bochs output)
# According to the number of cores in your machine, you can change the -j number to compile faster
# example: I use -j4 on my quad core machine
makeProc = subprocess.Popen("make -j2", shell=True, stdout=PIPE, stderr=PIPE)
#makeProc.wait()
makeProc.communicate() #wait() caused the deadlock with stdout, err = PIPE, online documentation warns against that, and suggested the use of communicate()
question_grade = -100
if (os.path.isfile(os.getcwd()+"/obj/kern/bochs.img") == True):
myCommand = 'bochs-no-gui -q -f .bochsrc_autograde_' + QTimestamp
#proc = Popen(myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE, preexec_fn=os.setsid)
#si = subprocess.STARTUPINFO()
#si.dwFlags |= subprocess.STARTF_USESTDHANDLES
#proc = Popen(myCommand, shell=True, startupinfo=si, stdout=PIPE, stderr=STDOUT) #shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)
proc = Popen("exec " + myCommand, shell=True, stdout=PIPE, stderr=PIPE, stdin=PIPE)
#===== WARNING: Unix only work-around: non-blocking streams
# default read() and readline() in Python block until a
# valid output is available.
# This makes it difficult to stop the bochs process
# when we reach the FOS>, because we will not know when to check
# as long as read() is working and doesn't return
#
# The workaround using fcntl non-blocking flag is Unix only solution
# we should find an equivalent library to other platforms, in case
# we migrate the course to windows or Mac
#import fcntl
#file_flags = fcntl.fcntl(proc.stdout.fileno(), fcntl.F_GETFL)
#fcntl.fcntl(proc.stdout.fileno(), fcntl.F_SETFL, file_flags | os.O_NONBLOCK)
# UPDATE: We didn't use this non-blocking work-around, and we returned
# to using the default blocking read(1), but we made sure
# to always check for the FOS> after every character read
# from the stream. if for some unusual reason the FOS>
# prompt isn't printed, we will have to manually close the
# bochs window to allow the script to continue working.
# - JUST MAKE SURE to run the script with python - u (unbuffered)
#=============
stream_data = ""
question_grade = 0
#A closure (inner function) that I will use later to terminate any stucking threads or processes
# and finialize remainig resources
def finializeAndCleanUp():
try:
if(proc.poll() is None): #kill the Bochs process if not already killed
proc.kill()
#os.kill(proc.pid, signal.SIGKILL)
thefile.close()
except Exception, msg:
print msg
try:
startTimeOfQuestion = datetime.now()
while os.path.isfile(os.getcwd() + "/" + student_stdout_Q_specific) == False:
#keep waiting till we find the new student_stdout file generated by Bochs
if ((datetime.now() - startTimeOfQuestion).seconds > 40):
print "Timeout, no console output from Bochs"
shutil.rmtree('./obj', ignore_errors=True)
os.chdir("../../")
if(proc.poll() is None): #kill the Bochs process if not already killed
proc.kill()
return -100, (datetime.now() - startTimeOfQuestion).seconds
time.sleep(0.001)
#TODO: Consider removing the try catch statements as a running time optimization
thefile = open(os.getcwd() + "/"+student_stdout_Q_specific,"r") #+ "/Students"+"/"+student_dir+
while True:
timeNow = datetime.now()
if((timeNow - startTimeOfQuestion).seconds > 60):
question_grade = -6
finializeAndCleanUp()
print "Timeout"
break
try:
#TODO: Revise this part to enable time out for project correction
#def signal_handler(signum, frame):
# raise Exception("Timed out!")
#signal.signal(signal.SIGALRM, signal_handler)
#signal.alarm(60*1) # 1 Min
#try:
# chunk = thefile.read(1)
#except Exception, msg:
# question_grade = -6
# finializeAndCleanUp()
# print msg
# break
#time.sleep(0.00005) #this slowed down the loop consideralby, taking about 5 seconds
chunk = thefile.read(1)
if not chunk:
continue
#else:
# sys.stdout.write(chunk)
# sys.stdout.flush()
stream_data += chunk
#print stream_data
if "InitialWSError1" in stream_data:
question_grade = -4
finializeAndCleanUp()
break
elif "InitialWSError2" in stream_data:
question_grade = -5
finializeAndCleanUp()
break
elif "[EVAL_FINAL]panic" in stream_data:
if tests[question_ID] == 'tia' and "kernel [EVAL_FINAL]panic" in stream_data:
question_grade = 1
finializeAndCleanUp()
break
else:
question_grade = -7 #Panic Case
finializeAndCleanUp()
break
elif stream_data.count('!! FCIS says HELLO !!') > 1:
question_grade = -3 #Restart Case
finializeAndCleanUp()
break
elif outputs[question_ID] == "TESTING MULTIPLE TIMES" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 3:
question_grade = 1
finializeAndCleanUp()
break
elif outputs[question_ID] == "TESTING MULTIPLE TIMES S2" and "[AUTO_GR@DING]Fibonacci #30 = 1346269" in stream_data and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 4:
question_grade = 1
finializeAndCleanUp()
break
elif outputs[question_ID] == "TESTING MULTIPLE TIMES S3" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 2:
question_grade = 1
finializeAndCleanUp()
break
elif outputs[question_ID] == "TESTING MULTIPLE TIMES S4" and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 1:
question_grade = 1
finializeAndCleanUp()
break
elif outputs[question_ID] == "TESTING ENVFREE SC 2" and stream_data.count('[AUTO_GR@DING]Congratulations!! test Scenario 2 completed successfully.') == 1 and stream_data.count('[AUTO_GR@DING]Congratulations!! The array is sorted correctly') == 2:
question_grade = 1
finializeAndCleanUp()
break
elif outputs[question_ID] in stream_data:
question_grade = 1
finializeAndCleanUp()
break
elif "FOS>" in stream_data:
finializeAndCleanUp()
break
elif proc.poll() is not None: #if process is finished, break the loop
finializeAndCleanUp()
break
#elif chunk is None:
#break
#elif chunk == "":
#break
except IOError:
if proc.poll() is not None:
#print "exiting loop, io error"
finializeAndCleanUp()
break
except Exception, msg:
#print msg
if proc.poll() is not None:
#print "exiting loop"
finializeAndCleanUp()
break
except IOError, msg:
#print "exiting, io error opening student_stdout_..."
print msg
finializeAndCleanUp()
def inner_run(self, *args, **options):
import sys
shutdown_message = options.get('shutdown_message', '')
do_reload = options.get('use_reloader', True)
#We use the old dev appserver if threading is disabled or --old was passed
use_old_dev_appserver = options.get(
'use_old_dev_appserver') or not options.get("use_threading")
quit_command = 'CTRL-BREAK' if sys.platform == 'win32' else 'CONTROL-C'
from djangae.utils import find_project_root, data_root
from djangae.sandbox import _find_sdk_from_python_path
from django.conf import settings
from django.utils import translation
#Check for app.yaml
expected_path = os.path.join(find_project_root(), "app.yaml")
if not os.path.exists(expected_path):
sys.stderr.write("Unable to find app.yaml at '%s'\n" %
expected_path)
sys.exit(1)
self.stdout.write("Validating models...\n\n")
self.validate(display_num_errors=True)
self.stdout.write(
("%(started_at)s\n"
"Django version %(version)s, using settings %(settings)r\n"
"Starting development server at http://%(addr)s:%(port)s/\n"
"Quit the server with %(quit_command)s.\n") % {
"started_at": datetime.now().strftime('%B %d, %Y - %X'),
"version": self.get_version(),
"settings": settings.SETTINGS_MODULE,
"addr": self._raw_ipv6 and '[%s]' % self.addr or self.addr,
"port": self.port,
"quit_command": quit_command,
})
sys.stdout.write("\n")
sys.stdout.flush()
# django.core.management.base forces the locale to en-us. We should
# set it up correctly for the first request (particularly important
# in the "--noreload" case).
translation.activate(settings.LANGUAGE_CODE)
#Will have been set by setup_paths
sdk_path = _find_sdk_from_python_path()
if use_old_dev_appserver:
dev_appserver = os.path.join(sdk_path, "old_dev_appserver.py")
command = [
dev_appserver,
find_project_root(),
"-p",
self.port,
"-h",
self.addr,
"--use_sqlite",
"--high_replication",
"--allow_skipped_files",
]
else:
dev_appserver = os.path.join(sdk_path, "dev_appserver.py")
command = [
dev_appserver,
find_project_root(), "--port", self.port, "--host", self.addr,
"--admin_port",
str(int(self.port) + 1), "--automatic_restart",
"True" if do_reload else "False", "--allow_skipped_files"
]
process = Popen(command,
stdout=sys.__stdout__,
stderr=sys.__stderr__,
cwd=find_project_root())
#This makes sure that dev_appserver gets killed on reload
import atexit
atexit.register(process.kill)
try:
process.wait()
except KeyboardInterrupt:
#Tell the dev appserver to shutdown and forcibly kill
#if it takes too long
process.send_signal(signal.SIGTERM)
time.sleep(2)
process.kill()
if shutdown_message:
sys.stdout.write(shutdown_message)
#Some weird race condition crazy sometimes makes this None...
if sys:
sys.exit(process.returncode)
class DockerizedJobServer(object):
"""
Spins up the JobServer in a docker container for local execution
"""
def __init__(self,
job_host="localhost",
job_port=None,
artifact_port=None,
expansion_port=None,
harness_port_range=(8100, 8200),
max_connection_retries=5):
self.job_host = job_host
self.job_port = job_port
self.expansion_port = expansion_port
self.artifact_port = artifact_port
self.harness_port_range = harness_port_range
self.max_connection_retries = max_connection_retries
self.docker_process = None
self.process_lock = Lock()
def start(self):
# TODO This is hardcoded to Flink at the moment but should be changed
job_server_image_name = os.environ['USER'] + \
"-docker-apache.bintray.io/beam/flink-job-server:latest"
docker_path = check_output(['which', 'docker']).strip().decode('utf-8')
cmd = [
"docker",
"run",
# We mount the docker binary and socket to be able to spin up
# "sibling" containers for the SDK harness.
"-v",
':'.join([docker_path, "/bin/docker"]),
"-v",
"/var/run/docker.sock:/var/run/docker.sock"
]
self.job_port, self.artifact_port, self.expansion_port = \
DockerizedJobServer._pick_port(self.job_port,
self.artifact_port,
self.expansion_port)
args = [
'--job-host', self.job_host, '--job-port',
str(self.job_port), '--artifact-port',
str(self.artifact_port), '--expansion-port',
str(self.expansion_port)
]
if sys.platform == "darwin":
# Docker-for-Mac doesn't support host networking, so we need to explictly
# publish ports from the Docker container to be able to connect to it.
# Also, all other containers need to be aware that they run Docker-on-Mac
# to connect against the internal Docker-for-Mac address.
cmd += ["-e", "DOCKER_MAC_CONTAINER=1"]
cmd += ["-p", "{}:{}".format(self.job_port, self.job_port)]
cmd += [
"-p", "{}:{}".format(self.artifact_port, self.artifact_port)
]
cmd += [
"-p", "{}:{}".format(self.expansion_port, self.expansion_port)
]
cmd += [
"-p", "{0}-{1}:{0}-{1}".format(self.harness_port_range[0],
self.harness_port_range[1])
]
else:
# This shouldn't be set for MacOS because it detroys port forwardings,
# even though host networking is not supported on MacOS.
cmd.append("--network=host")
cmd.append(job_server_image_name)
cmd += args
logging.debug("Starting container with %s", cmd)
try:
self.docker_process = Popen(cmd)
atexit.register(self.stop)
signal.signal(signal.SIGINT, self.stop)
except: # pylint:disable=bare-except
logging.exception("Error bringing up container")
self.stop()
return "{}:{}".format(self.job_host, self.job_port)
def stop(self):
with self.process_lock:
if not self.docker_process:
return
num_retries = 0
while self.docker_process.poll() is None and \
num_retries < self.max_connection_retries:
logging.debug("Sending SIGINT to job_server container")
self.docker_process.send_signal(signal.SIGINT)
num_retries += 1
time.sleep(1)
if self.docker_process.poll is None:
self.docker_process.kill()
@staticmethod
def _pick_port(*ports):
"""
Returns a list of ports, same length as input ports list, but replaces
all None or 0 ports with a random free port.
"""
sockets = []
def find_free_port(port):
if port:
return port
else:
s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
sockets.append(s)
s.bind(('localhost', 0))
_, free_port = s.getsockname()
return free_port
ports = list(map(find_free_port, ports))
# Close sockets only now to avoid the same port to be chosen twice
for s in sockets:
s.close()
return ports
def run_game(client1, client2, name):
c1_status = update(client1)
if c1_status != "good!":
return ("Error", c1_status)
c2_status = update(client2)
if c2_status != "good!":
return ("Error", c2_status)
# if not update('server'):
# return -13
startTime = time.time()
#now start the server...
servergood = False
port = startport
#slog = open(rootdir+"server.log","w")
while not servergood:
serverp = pexpect.spawn('python main.py ' + str(port),
cwd=rootdir + 'server/',
timeout=600)
i = serverp.expect([
'Unable to open socket!', 'Server Started', pexpect.EOF,
pexpect.TIMEOUT
],
timeout=10)
if i == 0 or i == 3 or i == 2:
#print "bad socket"
port += 1
serverp.close(True)
elif i == 1:
#print tentacle+" server should be started on port "+str(port)
servergood = True
else:
print 'THE WORLD IS AT AN END'
return ("Error", "fail")
#serverp.logfile = slog
time.sleep(2)
#now client 1..
client1p = Popen('/bin/bash ./run localhost:' + str(port),
cwd=rootdir + repositories[client1],
shell=True,
stdout=file('/dev/null', 'w'),
stderr=file('/dev/null', 'w'))
#client1p = pexpect.spawn('/bin/bash ./run localhost:'+str(port)+'> /dev/null 2>&1', cwd = rootdir+repositories[client1], timeout = 600)
#i = client1p.expect(['Creating game 1',pexpect.EOF,pexpect.TIMEOUT], timeout = 10)
i = serverp.expect(['Creating game 1', pexpect.EOF, pexpect.TIMEOUT],
timeout=10)
if i == 0:
pass
#print tentacle+" game created!"
else:
#print client1p.returncode
#print tentacle+" game failed to create:"+str(i)
return ("Error", "Game failed to create " + client1)
#and client 2...
client2p = Popen(['/bin/bash ./run localhost:' + str(port) + ' 0'],
cwd=rootdir + repositories[client2],
shell=True,
stdout=file('/dev/null', 'w'),
stderr=file('/dev/null', 'w'))
#client2p = pexpect.spawn('/bin/bash ./run localhost:'+str(port)+' 0 > /dev/null 2>&1', cwd = rootdir+repositories[client2], timeout = 600)
#i = client2p.expect([pexpect.EOF,pexpect.TIMEOUT],timeout=5)
i = serverp.expect(['Starting', pexpect.EOF, pexpect.TIMEOUT], timeout=5)
if i != 0:
#print tentacle+' '+client2+" couldn't connect"
return ("Error", client2 + " couldn't connect to server")
print tentacle + " game started! (" + client1 + " vs. " + client2 + " on port " + str(
port) + ")"
result = ''
try:
while time.time() < startTime + 600 and len(result) < 5:
#result = serverp.expect(["Tie game!", "1 Wins!", "2 Wins", pexpect.TIMEOUT], timeout = 5)
#try:
# client1p.read_nonblocking(1024, timeout = 0)
#except:
# pass
#try:
# client2p.read_nonblocking(1024, timeout = 0)
#except:
# pass
try:
result = serverp.readline()
if "win" not in result.lower() and "tie" not in result.lower():
result = ''
except pexpect.TIMEOUT:
pass
except SoftTimeLimitExceeded:
print tentacle + " TIMEOUT!"
serverp.close(True)
client1p.kill()
client2p.kill()
return ("Error", "Timeout!")
#result = serverp.readlines()
#print result
#print tentacle+" game completed!"
c1_score = 0
c2_score = 0
if "1" in result: # == "1 Wins!":
c1_score = 1
winner = client1
#print client1, "wins"
elif "2" in result or result == '': # == "2 Wins!":
c2_score = 1
winner = client2
#print client2, "wins"
elif "tie" in result.lower(): # == "Tie game!":
#print "tie"
winner = client2
c2_score = 1
elif result == '':
pass
#print "someone crashed!"
else:
print "wat?", result
return ("Error", "Unknown results from server:" + result)
serverp.close(True)
client1p.kill()
client2p.kill()
#slog.close()
#client1p.close(True)
#client2p.close(True)
#Popen("cp "+rootdir+"server.log ~/"+tentacle+str(time.time())+".server.log", shell = True).wait()
#logfile = open(rootdir+'server/logs/1.gamelog.bz2','rb')
#log = logfile.read()
s3conn = S3Connection(AWS_ACCESS_KEY, AWS_SECRET_KEY)
logbucket = s3conn.get_bucket("megaminer7")
newkey = Key(logbucket)
logname = 'logs/arena/' + str(tentacle) + '-' + str(
time.time()) + '.gamelog.bz2'
newkey.key = logname
newkey.set_contents_from_filename(rootdir + "server/logs/1.gamelog.bz2")
webserver.set_game_stat(client1, client2, c1_score, c2_score,
working_copies[client1], working_copies[client2],
logname)
remove(rootdir + "server/logs/1.gamelog")
remove(rootdir + "server/logs/1.gamelog.bz2")
return (winner, logname)
class Client(object):
def __init__(self, python_path, max_pickle_version, config_dir):
# Make sure to init these variables (used in __del__) early on in case we
# have an exception
self._poller = None
self._server_process = None
self._socket_path = None
data_transferer.defaultProtocol = max_pickle_version
self._config_dir = config_dir
# The client launches the server when created; we use
# Unix sockets for now
server_module = ".".join([__package__, "server"])
self._socket_path = "/tmp/%s_%d" % (os.path.basename(config_dir),
os.getpid())
if os.path.exists(self._socket_path):
raise RuntimeError("Existing socket: %s" % self._socket_path)
env = os.environ.copy()
#env["PYTHONPATH"] = ":".join(sys.path)
self._server_process = Popen(
[
python_path, "-u", "-m", server_module,
str(max_pickle_version), config_dir, self._socket_path
],
env=env,
stdout=PIPE,
stderr=PIPE,
bufsize=1,
universal_newlines=True, # Forces text as well
)
# Read override configuration
sys.path.insert(0, config_dir)
override_module = importlib.import_module("overrides")
sys.path = sys.path[1:]
# Determine all overrides
self._overrides = {}
self._getattr_overrides = {}
self._setattr_overrides = {}
for override in override_module.__dict__.values():
if isinstance(override, (LocalOverride, LocalAttrOverride)):
for obj_name, obj_funcs in override.obj_mapping.items():
if isinstance(override, LocalOverride):
override_dict = self._overrides.setdefault(
obj_name, {})
elif override.is_setattr:
override_dict = self._setattr_overrides.setdefault(
obj_name, {})
else:
override_dict = self._getattr_overrides.setdefault(
obj_name, {})
if isinstance(obj_funcs, str):
obj_funcs = (obj_funcs, )
for name in obj_funcs:
if name in override_dict:
raise ValueError(
"%s was already overriden for %s" %
(name, obj_name))
override_dict[name] = override.func
self._proxied_objects = {}
# Wait for the socket to be up on the other side; we also check if the
# server had issues starting up in which case we report that and crash out
while not os.path.exists(self._socket_path):
returncode = self._server_process.poll()
if returncode is not None:
raise RuntimeError(
"Server did not properly start: %s" %
self._server_process.stderr.read(), )
time.sleep(1)
# Open up the channel and setup the datastransfer pipeline
self._channel = Channel(SocketByteStream.unixconnect(
self._socket_path))
self._datatransferer = DataTransferer(self)
# Make PIPEs non-blocking; this is helpful to be able to
# order the messages properly
for f in (self._server_process.stdout, self._server_process.stderr):
fl = fcntl.fcntl(f, fcntl.F_GETFL)
fcntl.fcntl(f, fcntl.F_SETFL, fl | os.O_NONBLOCK)
# Set up poller
self._poller = select.poll()
self._poller.register(self._server_process.stdout, select.POLLIN)
self._poller.register(self._server_process.stderr, select.POLLIN)
self._poller.register(self._channel, select.POLLIN | select.POLLHUP)
# Get all exports that we are proxying
response = self._communicate({
FIELD_MSGTYPE: MSG_CONTROL,
FIELD_OPTYPE: CONTROL_GETEXPORTS
})
self._proxied_classes = {
k: None
for k in itertools.chain(response[FIELD_CONTENT]["classes"],
response[FIELD_CONTENT]["proxied"])
}
# Proxied standalone functions are functions that are proxied
# as part of other objects like defaultdict for which we create a
# on-the-fly simple class that is just a callable. This is therefore
# a special type of self._proxied_classes
self._proxied_standalone_functions = {}
self._export_info = {
"classes": response[FIELD_CONTENT]["classes"],
"functions": response[FIELD_CONTENT]["functions"],
"values": response[FIELD_CONTENT]["values"],
"exceptions": response[FIELD_CONTENT]["exceptions"],
"aliases": response[FIELD_CONTENT]["aliases"]
}
self._aliases = response[FIELD_CONTENT]["aliases"]
def __del__(self):
# Clean up the server; we drain all messages if any
if self._poller is not None:
# If we have self._poller, we have self._server_process
self._poller.unregister(self._channel)
last_evts = self._poller.poll(5)
for fd, _ in last_evts:
# Readlines will never block here because bufsize is set to
# 1 (line buffering)
if fd == self._server_process.stdout.fileno():
sys.stdout.write(self._server_process.stdout.readline())
elif fd == self._server_process.stderr.fileno():
sys.stderr.write(self._server_process.stderr.readline())
sys.stdout.flush()
sys.stderr.flush()
if self._server_process is not None:
# Attempt to send it a terminate signal and then wait and kill
try:
self._channel.send({
FIELD_MSGTYPE: MSG_CONTROL,
FIELD_OPTYPE: CONTROL_SHUTDOWN
})
self._channel.recv(timeout=10) # If we receive, we are sure we
# are good
except: # noqa E722
pass # If there is any issue sending this message, just ignore it
self._server_process.kill()
if self._socket_path is not None and os.path.exists(self._socket_path):
os.unlink(self._socket_path)
@property
def name(self):
return self._config_dir
def get_exports(self):
return self._export_info
def stub_request(self, stub, request_type, *args, **kwargs):
# Encode the operation to send over the wire and wait for the response
target = self.encode(stub)
encoded_args = []
for arg in args:
encoded_args.append(self.encode(arg))
encoded_kwargs = []
for k, v in kwargs.items():
encoded_kwargs.append((self.encode(k), self.encode(v)))
response = self._communicate({
FIELD_MSGTYPE:
MSG_OP,
FIELD_OPTYPE:
request_type,
FIELD_TARGET:
target,
FIELD_ARGS:
self.encode(args),
FIELD_KWARGS:
self.encode([(k, v) for k, v in kwargs.items()]),
})
response_type = response[FIELD_MSGTYPE]
if response_type == MSG_REPLY:
return self.decode(response[FIELD_CONTENT])
elif response_type == MSG_EXCEPTION:
raise load_exception(self._datatransferer, response[FIELD_CONTENT])
elif response_type == MSG_INTERNAL_ERROR:
raise RuntimeError(
"Error in the server runtime:\n\n===== SERVER TRACEBACK =====\n%s"
% response[FIELD_CONTENT])
def encode(self, obj):
# This encodes an object to transfer back out
# Basic data types will be sent over directly.
# In this direction (client -> server), we error on non-basic
# types. This could be changed by modifying the pickle_object method
# here.
return self._datatransferer.dump(obj)
def decode(self, json_obj):
# This decodes an object that was transferred in. This will call
# unpickle_object where needed. Any remote object that is handled by
# this connection will be converted to a local stub.
return self._datatransferer.load(json_obj)
def get_local_class(self, name, obj_id=None):
# Gets (and creates if needed), the class mapping to the remote
# class of name 'name'.
name = self._get_canonical_name(name)
if name == 'function':
# Special handling of pickled functions. We create a new class that
# simply has a __call__ method that will forward things back to
# the server side.
if obj_id is None:
raise RuntimeError(
"Local function unpickling without an object ID")
if obj_id not in self._proxied_standalone_functions:
self._proxied_standalone_functions[obj_id] = create_class(
self, '__function_%s' % obj_id, {}, {}, {},
{'__call__': ''})
return self._proxied_standalone_functions[obj_id]
if name not in self._proxied_classes:
raise ValueError("Class '%s' is not known" % name)
local_class = self._proxied_classes[name]
if local_class is None:
# We need to build up this class. To do so, we take everything that the
# remote class has and remove UNSUPPORTED things and overriden things
remote_methods = self.stub_request(None, OP_GETMETHODS, name)
local_class = create_class(self, name,
self._overrides.get(name, {}),
self._getattr_overrides.get(name, {}),
self._setattr_overrides.get(name, {}),
remote_methods)
self._proxied_classes[name] = local_class
return local_class
def can_pickle(self, obj):
return getattr(obj, "___connection___", None) == self
def pickle_object(self, obj):
# This function is called to pickle obj to be transferred back to the
# server. In this direction, we only allow objects that already exist
# on the remote side so if this is not a stub, we do not allow it to be
# transferred
if getattr(obj, "___connection___", None) == self:
# This is something we can transfer over
return ObjReference(VALUE_LOCAL, obj.___remote_class_name___,
obj.___identifier___)
raise ValueError(
"Cannot send object of type %s from client to server" % type(obj))
def unpickle_object(self, obj):
# This function is called when the server sends a remote reference.
# We create a local stub for it locally
if (not isinstance(obj,
ObjReference)) or obj.value_type != VALUE_REMOTE:
raise ValueError("Invalid transferred object: %s" % str(obj))
remote_class_name = obj.class_name
obj_id = obj.identifier
local_instance = self._proxied_objects.get(obj_id)
if not local_instance:
local_class = self.get_local_class(remote_class_name, obj_id)
local_instance = local_class(self, remote_class_name, obj_id)
return local_instance
def _get_canonical_name(self, name):
# We look at the aliases looking for the most specific match first
base_name = self._aliases.get(name)
if base_name is not None:
return base_name
for idx in reversed(
[pos for pos, char in enumerate(name) if char == '.']):
base_name = self._aliases.get(name[:idx])
if base_name is not None:
return ".".join([base_name, name[idx + 1:]])
return name
def _communicate(self, msg):
self._channel.send(msg)
response_ready = False
while not response_ready:
evt_list = self._poller.poll()
for fd, _ in evt_list:
if fd == self._channel.fileno():
# We deal with this last as this basically gives us the
# response so we stop looking at things on stdout/stderr
response_ready = True
# Readlines will never block here because bufsize is set to 1
# (line buffering)
elif fd == self._server_process.stdout.fileno():
sys.stdout.write(self._server_process.stdout.readline())
elif fd == self._server_process.stderr.fileno():
sys.stderr.write(self._server_process.stderr.readline())
# We make sure there is nothing left to read. On the server side, a
# flush happens before we respond so we read until we get an exception;
# this is non blocking
while True:
try:
line = self._server_process.stdout.readline()
if not line:
break
sys.stdout.write(line)
except (OSError, TypeError):
break
while True:
try:
line = self._server_process.stderr.readline()
if not line:
break
sys.stderr.write(line)
except (OSError, TypeError):
break
sys.stdout.flush()
sys.stderr.flush()
return self._channel.recv()
class VLC(object):
def __init__(self, host="127.0.0.1", port=8080):
# connection attrs
self.host = host
self.port = port
# private playlist var, stores list of file paths
# mirrors the list in the player at all times
self._playlist = []
# used to determine if a
self.supported = ["mp4", "avi", "mkv", "flv", ".aac",
"3gp"] # add more later
# creating an instance of the vlc window
# local socket connection to the vlc player
@property
def playlist(self):
'''returns list of file paths'''
return self._playlist
@property
def connection_open(self):
try:
resp = requests.get("http://%s:%s/requests/status.json" %
(self.host, self.port),
auth=("", "1234"))
if resp.status_code == 200:
return True
else:
return False
except Exception as e:
return False
@playlist.setter
def playlist(self, arg):
"""Takes a string, tuple or a list as an argument and
updates the player's playlist and the local_playlist variable
enqueues the vlc object with a playlist of all the files stored in it
can only add files to the playlist"""
if isinstance(arg, (list, tuple)):
for path in arg:
self.check_path(path)
if not path in self._playlist:
data = self._enqueue(path)
elif isinstance(arg, str):
self.check_path(arg)
if not arg in self._playlist:
data = self._enqueue(arg)
@playlist.deleter
def playlist(self):
'''clears the local playlist var and the remote one'''
self._playlist = []
self.clear()
@property
def current(self):
resp = requests.get("http://%s:%s/requests/status.json" %
(self.host, self.port),
auth=("", "1234"))
js = json.loads(resp.content)
return js["information"]["category"]["meta"]["filename"]
@property
def player_status(self):
resp = requests.get("http://%s:%s/requests/status.json" %
(self.host, self.port),
auth=("", "1234"))
js = json.loads(resp.content)
return js["state"]
def create_player(self):
if self.connection_open:
return
self.conn = Popen(["vlc", "-I", "http"],
stdout=PIPE,
stdin=PIPE,
stderr=PIPE)
def check_path(self, path):
'''Ensures all files added to the application are
valid paths.'''
if not os.path.isfile(path):
print "file exception %s" % path
raise FileNotFoundException()
path, file = os.path.split(path)
name, ext = file.split(".")
if ext not in self.supported:
print " supported exception %s" % path
raise UnsupportedFileTypeException()
def execute(self, cmd):
'''Prepare a command and send it to VLC'''
request_string = \
"http://%s:%s/requests/status.xml?command=%s" % (
self.host, self.port, cmd)
resp = requests.get(request_string, auth=("", "1234"))
def toggle_fullscreen(self):
'''puts the windows in full screen'''
resp = requests.get("http://%s:%s/requests/status.json" %
(self.host, self.port),
auth=("", "1234"))
js = json.loads(resp.content)
if js["fullscreen"] == 0:
print "executed fullscreen"
self.execute('fullscreen')
def toggle_loop(self):
'''makes the vlc player loop the current playlist'''
resp = requests.get("http://%s:%s/requests/status.json" %
(self.host, self.port),
auth=("", "1234"))
js = json.loads(resp.content)
if not js["loop"]:
self.execute("pl_loop")
def pause(self):
"""Checks the current state to make sure the player is playing something"""
self.execute('pl_pause')
def play(self):
"""First checks if a valid file is currently loaded."""
self.execute('pl_play')
time.sleep(5)
self.toggle_fullscreen()
self.toggle_loop()
def stop(self):
"""checks first if there is something to stop"""
self.execute('pl_stop')
def _enqueue(self, path):
'''adds a file to the playlist'''
data = self.execute('in_enqueue&input=%s' % (os.path.abspath(path), ))
self._playlist.append(path)
def clear(self):
'''clears all files from the playlist'''
self.execute('pl_empty')
self._playlist = []
def shutdown(self):
self.stop()
self.conn.kill()
class Proxy(object):
"""Create a proxy object as intermediary between client code and remote functionality.
This class is a context manager, so when used in a ``with`` statement,
it ensures the remote proxy server is stopped and disposed correctly.
However, if the proxy server is left open, it can be re-used for a follow-up connection,
saving start up time.
Parameters
----------
package : string, optional
The base package for the requested functionality.
Default is `None`, in which case a full path to function calls should be provided.
python : string, optional
The python executable that should be used to execute the code.
Default is ``'pythonw'``.
url : string, optional
The server address.
Default is ``'http://127.0.0.1'``.
port : int, optional
The port number on the remote server.
Default is ``1753``.
Notes
-----
If the server is your *localhost*, which will often be the case, it is better
to specify the address explicitly (``'http://127.0.0.1'``) because resolving
*localhost* takes a surprisingly significant amount of time.
The service will make the correct (version of the requested) functionality available
even if that functionality is part of a virtual environment. This is because it
will use the specific python interpreter for which the functionality is installed to
start the server.
If possible, the proxy will try to reconnect to an already existing service
Examples
--------
Minimal example showing connection to the proxy server, and ensuring the
server is disposed after using it:
.. code-block:: python
from compas.rpc import Proxy
with Proxy('compas.numerical') as numerical:
pass
"""
def __init__(self,
package=None,
python=None,
url='http://127.0.0.1',
port=1753,
service=None):
self._package = None
self._python = compas._os.select_python(python)
self._url = url
self._port = port
self._service = None
self._process = None
self._function = None
self._profile = None
self.service = service
self.package = package
self._implicitely_started_server = False
self._server = self._try_reconnect()
if self._server is None:
self._server = self.start_server()
self._implicitely_started_server = True
def __enter__(self):
return self
def __exit__(self, *args):
# If we started the RPC server, we will try to clean up and stop it
# otherwise we just disconnect from it
if self._implicitely_started_server:
self.stop_server()
else:
self._server.__close()
@property
def address(self):
return "{}:{}".format(self._url, self._port)
@property
def profile(self):
"""A profile of the executed code."""
return self._profile
@profile.setter
def profile(self, profile):
self._profile = profile
@property
def package(self):
"""The base package from which functionality will be called."""
return self._package
@package.setter
def package(self, package):
self._package = package
@property
def service(self):
return self._service
@service.setter
def service(self, service):
if not service:
self._service = 'compas.rpc.services.default'
else:
self._service = service
@property
def python(self):
return self._python
@python.setter
def python(self, python):
self._python = python
def _try_reconnect(self):
"""Try and reconnect to an existing proxy server.
Returns
-------
ServerProxy
Instance of the proxy if reconnection succeeded, otherwise ``None``.
"""
server = ServerProxy(self.address)
try:
server.ping()
except Exception:
return None
else:
print("Reconnecting to an existing server proxy.")
return server
def start_server(self):
"""Start the remote server.
Returns
-------
ServerProxy
Instance of the proxy, if the connection was successful.
Raises
------
RPCServerError
If the server providing the requested service cannot be reached after
100 contact attempts (*pings*).
Examples
--------
>>> p = Proxy()
>>> p.stop_server()
>>> p.start_server()
"""
env = compas._os.prepare_environment()
# this part starts the server side of the RPC setup
# it basically launches a subprocess
# to start the default service
# the default service creates a server
# and registers a dispatcher for custom functionality
try:
Popen
except NameError:
self._process = Process()
for name in env:
if self._process.StartInfo.EnvironmentVariables.ContainsKey(
name):
self._process.StartInfo.EnvironmentVariables[name] = env[
name]
else:
self._process.StartInfo.EnvironmentVariables.Add(
name, env[name])
self._process.StartInfo.UseShellExecute = False
self._process.StartInfo.RedirectStandardOutput = True
self._process.StartInfo.RedirectStandardError = True
self._process.StartInfo.FileName = self.python
self._process.StartInfo.Arguments = '-m {0} {1}'.format(
self.service, str(self._port))
self._process.Start()
else:
args = [self.python, '-m', self.service, str(self._port)]
self._process = Popen(args, stdout=PIPE, stderr=PIPE, env=env)
# this starts the client side
# it creates a proxy for the server
# and tries to connect the proxy to the actual server
server = ServerProxy(self.address)
print("Starting a new proxy server...")
success = False
count = 100
while count:
try:
server.ping()
except Exception:
time.sleep(0.1)
count -= 1
print(" {} attempts left.".format(count))
else:
success = True
break
if not success:
raise RPCServerError("The server is not available.")
else:
print("New proxy server started.")
return server
def stop_server(self):
"""Stop the remote server and terminate/kill the python process that was used to start it.
Examples
--------
>>> p = Proxy()
>>> p.stop_server()
>>> p.start_server()
"""
print("Stopping the server proxy.")
try:
self._server.remote_shutdown()
except Exception:
pass
self._terminate_process()
def _terminate_process(self):
"""Attempts to terminate the python process hosting the proxy server.
The process reference might not be present, e.g. in the case
of reusing an existing connection. In that case, this is a no-op.
"""
if not self._process:
return
try:
self._process.terminate()
except Exception:
pass
try:
self._process.kill()
except Exception:
pass
def __getattr__(self, name):
if self.package:
name = "{}.{}".format(self.package, name)
try:
self._function = getattr(self._server, name)
except Exception:
raise RPCServerError()
return self._proxy
def _proxy(self, *args, **kwargs):
"""Callable replacement for the requested functionality.
Parameters
----------
args : list
Positional arguments to be passed to the remote function.
kwargs : dict
Named arguments to be passed to the remote function.
Returns
-------
object
The result returned by the remote function.
Warning
-------
The `args` and `kwargs` have to be JSON-serialisable.
This means that, currently, only native Python objects are supported.
The returned results will also always be in the form of built-in Python objects.
"""
idict = {'args': args, 'kwargs': kwargs}
istring = json.dumps(idict, cls=DataEncoder)
# it makes sense that there is a broken pipe error
# because the process is not the one receiving the feedback
# when there is a print statement on the server side
# this counts as output
# it should be sent as part of RPC communication
try:
ostring = self._function(istring)
except Exception:
# not clear what the point of this is
# self.stop_server()
# if this goes wrong, it means a Fault error was generated by the server
# no need to stop the server for this
raise
if not ostring:
raise RPCServerError("No output was generated.")
result = json.loads(ostring)
if result['error']:
raise RPCServerError(result['error'])
self.profile = result['profile']
return result['data']
