#!/usr/bin/env python
baseDirs=[]
import os
from subprocess import Popen,PIPE
for p in ['py2-pippkgs','py2-pippkgs_depscipy']:
comm="scram tool info "+p+" | grep PYTHONPATH | cut -f2 -d="
p=Popen(comm,stdout=PIPE,shell=True)
baseDir=p.stdout.read().strip()
baseDirs.append(baseDir)
l=[]
for baseDir in baseDirs:
for root, dirs, files in os.walk(baseDir):
for file in files:
if file.endswith('.py'):
l.append(file[:-3])
for file in dirs:
if 'egg-info' in file: continue
if 'dist-info' in file: continue
l.append(file)
break
print l
skipIt=['pytest','climate','xrootdpyfs','theanets','hyperopt','thriftpy']
# climate misses plac
# pytest misses py
# theanets misses plac
# xrootdpyfs misses from fs.errors import DestinationExistsError, DirectoryNotEmptyError, \
def create_visdom_connections(self):
"""If the program could not connect to Visdom server, this function will start a new server at port < self.port > """
cmd = sys.executable + ' -m visdom.server -p %d &>/dev/null &' % self.port
print('\n\nCould not connect to Visdom server. \n Trying to start a server....')
print('Command: %s' % cmd)
Popen(cmd, shell=True, stdout=PIPE, stderr=PIPE)
def start(self) -> None:
self.outfile = tempfile.TemporaryFile()
self.start_time = time.perf_counter()
self.process = Popen(self.args, cwd=self.cwd, env=self.env,
stdout=self.outfile, stderr=STDOUT)
self.pid = self.process.pid
def run_renumber_by_editconf(pdbfile, gmx_path):
program = os.path.join(gmx_path, "editconf")
process = Popen([program, '-f', pdbfile, '-o', pdbfile, '-resnr', '1'],
stdout=PIPE,
stderr=PIPE)
stdout, stderr = process.communicate()
def tracert(self):
self.s = Popen(['traceroute', self.ip], stdout=PIPE)
while True:
self.line = self.s.stdout.readline()
if not self.line:
break
def prepare_rip(self, idx, track):
args = self.args
# reset progress
self.progress.prepare_track(track)
if self.progress.total_tracks > 1:
print(Fore.GREEN + "[ " + str(idx + 1) + " / " +
str(self.progress.total_tracks +
self.progress.skipped_tracks) + " ] Ripping " +
track.link.uri + Fore.WHITE +
"\t(ESC to skip)" + Fore.RESET)
else:
print(Fore.GREEN + "Ripping " + track.link.uri + Fore.RESET)
print(Fore.CYAN + self.audio_file + Fore.RESET)
file_size = calc_file_size(track)
print("Track Download Size: " + format_size(file_size))
audio_file_enc = enc_str(self.audio_file)
if args.output_type == "wav":
self.wav_file = wave.open(audio_file_enc, "wb")
self.wav_file.setparams((2, 2, 44100, 0, 'NONE', 'not compressed'))
elif args.output_type == "pcm":
self.pcm_file = open(audio_file_enc, 'wb')
elif args.output_type == "flac":
self.rip_proc = Popen(
["flac", "-f", ("-" + str(args.comp)), "--silent", "--endian",
"little", "--channels", "2", "--bps", "16", "--sample-rate",
"44100", "--sign", "signed", "-o", audio_file_enc, "-"],
stdin=PIPE)
elif args.output_type == "alac.m4a":
self.rip_proc = Popen(
["avconv", "-nostats", "-loglevel", "0", "-f", "s16le", "-ar",
"44100", "-ac", "2", "-channel_layout", "stereo", "-i", "-",
"-acodec", "alac", audio_file_enc],
stdin=PIPE)
elif args.output_type == "ogg":
if args.cbr:
self.rip_proc = Popen(
["oggenc", "--quiet", "--raw", "-b", args.bitrate, "-o",
audio_file_enc, "-"], stdin=PIPE)
else:
self.rip_proc = Popen(
["oggenc", "--quiet", "--raw", "-q", args.vbr, "-o",
audio_file_enc, "-"], stdin=PIPE)
elif args.output_type == "opus":
if args.cbr:
self.rip_proc = Popen(
["opusenc", "--quiet", "--comp", args.comp, "--cvbr",
"--bitrate", str(int(args.bitrate) / 2), "--raw",
"--raw-rate", "44100", "-", audio_file_enc], stdin=PIPE)
else:
self.rip_proc = Popen(
["opusenc", "--quiet", "--comp", args.comp, "--vbr",
"--bitrate", args.vbr, "--raw", "--raw-rate", "44100",
"-", audio_file_enc], stdin=PIPE)
elif args.output_type == "aac":
if self.dev_null is None:
self.dev_null = open(os.devnull, 'wb')
if args.cbr:
self.rip_proc = Popen(
["faac", "-P", "-X", "-b", args.bitrate, "-o",
audio_file_enc, "-"], stdin=PIPE,
stdout=self.dev_null, stderr=self.dev_null)
else:
self.rip_proc = Popen(
["faac", "-P", "-X", "-q", args.vbr, "-o",
audio_file_enc, "-"], stdin=PIPE,
stdout=self.dev_null, stderr=self.dev_null)
elif args.output_type == "m4a":
if args.cbr:
self.rip_proc = Popen(
["fdkaac", "-S", "-R", "-b",
args.bitrate, "-o", audio_file_enc, "-"], stdin=PIPE)
else:
self.rip_proc = Popen(
["fdkaac", "-S", "-R", "-m", args.vbr,
"-o", audio_file_enc, "-"], stdin=PIPE)
elif args.output_type == "mp3":
lame_args = ["lame", "--silent"]
if args.stereo_mode is not None:
lame_args.extend(["-m", args.stereo_mode])
if args.cbr:
lame_args.extend(["-cbr", "-b", args.bitrate])
else:
lame_args.extend(["-V", args.vbr])
lame_args.extend(["-h", "-r", "-", audio_file_enc])
self.rip_proc = Popen(lame_args, stdin=PIPE)
if self.rip_proc is not None:
self.pipe = self.rip_proc.stdin
self.ripping.set()
log_file_name = base_name + ".log"
if os.path.exists(log_file_name):
os.remove(log_file_name)
logfile = open(log_file_name, 'w')
# remove old output files, if any
# use regular expression module since differentiating
# between gold files and old output files can be tricky
files_to_remove = base_name + ".e"
for file in os.listdir(os.getcwd()):
if file in files_to_remove:
os.remove(file)
# run Peridigm
command = ["../../../../src/Peridigm", "../"+base_name+".xml"]
p = Popen(command, stdout=logfile, stderr=logfile)
return_code = p.wait()
if return_code != 0:
result = return_code
# compare output files against gold files
command = ["../../../../scripts/exodiff", \
"-stat", \
"-f", \
"../"+base_name+".comp", \
base_name+".e", \
"../"+base_name+"_gold.e"]
p = Popen(command, stdout=logfile, stderr=logfile)
return_code = p.wait()
if return_code != 0:
result = return_code
def zzztest_sdl2_audio_beeps(self):
open(os.path.join(self.get_dir(), 'sdl2_audio_beep.cpp'), 'w').write(self.with_report_result(open(path_from_root('tests', 'sdl2_audio_beep.cpp')).read()))
# use closure to check for a possible bug with closure minifying away newer Audio() attributes
Popen([EMCC, '-O2', '--closure', '1', '--minify', '0', os.path.join(self.get_dir(), 'sdl2_audio_beep.cpp'), '-s', 'DISABLE_EXCEPTION_CATCHING=0', '-s', 'USE_SDL=2', '-o', 'page.html']).communicate()
self.run_browser('page.html', '', '/report_result?1')
def test_freealut(self):
Popen([EMCC, '-O2'] + self.get_freealut_library() + ['-o', 'page.html']).communicate()
self.run_browser('page.html', 'You should hear "Hello World!"')
def create_entitystore(self,
cert_path,
key_path,
node_cert_path='',
chain=False,
store_name='entitystore',
store_password='******'):
"""
Generate `jks` entitystore to be used by a specific entity (server or
client).
Args:
cert_path: Path to the certificate or certificate chain to be put
in the keystore.
key_path: Path to the key corresponding to the certificate in
cert_path.
node_cert_path: the node directory for which to store trust databases
chain: Specifies if cert_path points to a certificate chain or not.
Default: False
store_name: Name of the keystore without the '.jks' extension.
Default: `entitystore`.
store_password: Password to be used for the keystore. Default:
`not-relevant-for-security`.
Returns:
Path to the created keystore.
"""
java_store_file = store_name + '.jks'
base_path = Path(self.artifact_dir) / node_cert_path
java_store_path = base_path / java_store_file
pkcs12_store_path = Path(str(java_store_path) + '.p12') # trust.jks.p12
certificate_alias = cert_path.name.split('.')[0]
pkcs12_cmd = [
'openssl', 'pkcs12', '-export', '-in',
str(cert_path), '-inkey',
str(key_path), '-out',
str(pkcs12_store_path), '-name', certificate_alias, '-passout',
'pass:'******'-CAfile', str(cert_path), '-chain'])
log.info('Creating pkcs12 {} keystore: {}'.format(
store_name, ' '.join(pkcs12_cmd)))
proc = Popen(pkcs12_cmd, shell=False, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.wait() != 0:
raise Exception('{}{}'.format(stdout.decode(), stderr.decode()))
keystore_cmd = [
'keytool', '-importkeystore', '-destkeystore',
str(java_store_path), '-srckeystore',
str(pkcs12_store_path), '-srcstoretype', 'pkcs12', '-alias',
certificate_alias, '-srcstorepass', store_password,
'-deststorepass', store_password, '-noprompt'
]
log.info('Creating jks {} keystore: {}'.format(store_name,
' '.join(pkcs12_cmd)))
proc = Popen(keystore_cmd, shell=False, stdout=PIPE, stderr=PIPE)
stdout, stderr = proc.communicate()
if proc.wait() != 0:
raise Exception('{}{}'.format(stdout.decode(), stderr.decode()))
try:
os.remove(pkcs12_store_path)
except OSError:
log.error('Could not remove pcks12 {} keystore: {}'.format(
store_name, ' '.join(pkcs12_cmd)))
java_store_path.chmod(0o600)
return java_store_path
from __future__ import print_function
import sys
import os
from common import db_database, db_username, db_password
from subprocess import Popen, check_call, PIPE
import getpass
from psycopg2 import IntegrityError
from datetime import datetime
if len(sys.argv) != 2:
print("Usage: %s <COMPRESSED-BACKUP>" % (sys.argv[0], ), file=sys.stderr)
sys.exit(1)
filename = sys.argv[1]
# You must specify your password in ~/.pgpass, as described here:
# http://www.postgresql.org/docs/current/static/libpq-pgpass.html
cat_command = ['bzcat', filename]
restore_command = ['psql', '-U', db_username, '--no-password', db_database]
p1 = Popen(cat_command, stdout=PIPE)
p2 = Popen(restore_command, stdin=p1.stdout)
p1.stdout.close()
p1.wait()
p2.wait()
def execute_test(gtest_binary, test_name):
process = Popen([ gtest_binary, "--gtest_color=yes", "--gtest_filter=%s"%(test_name) ], stdout=PIPE, stderr=PIPE)
(output, err) = process.communicate()
rv=process.wait()
return (rv, output, err)
def execute_process(self, command_line):
process = Popen(command_line, env=self.my_environment, shell=True, stdout=PIPE, stderr=STDOUT,
universal_newlines=True)
out, _ = process.communicate()
out = out.split('\n')
return out
from rosunit.junitxml import test_failure_junit_xml
if __name__ == '__main__':
parser = argparse.ArgumentParser()
parser.add_argument('command', help='command to run')
parser.add_argument('test_name', help='test name')
parser.add_argument('result_file', help='test result file')
parser.add_argument('--working-directory', help='working directory')
args = parser.parse_args()
cmd = args.command
test_name = args.test_name
result_file = args.result_file
cwd = args.working_directory
p = Popen(cmd, stdout=PIPE, stderr=STDOUT, shell=True, cwd=cwd)
stdout, _ = p.communicate()
ret_code = p.returncode
passed = ret_code == 0
print('...writing test results to', result_file)
if passed:
print('passed')
with open(result_file, 'w') as f:
f.write(test_success_junit_xml(test_name).decode())
else:
print('FAILURE:\n{0}'.format(stdout), file=sys.stderr)
with open(result_file, 'w') as f:
message = 'shell test with command [{0}] failed'.format(cmd)
f.write(test_failure_junit_xml(test_name, message, stdout=stdout))
# Thanks for the code to start Mike
#!/bin/python3.4
import json, os
from influxdb import InfluxDBClient
from subprocess import Popen, PIPE
client = InfluxDBClient(host=os.environ['influxdb_host'], port=os.environ['influxdb_port'], database='weather')
client.create_database('weather')
with Popen(['/usr/local/bin/rtl_433', '-F', 'json', '-C', 'customary'], stdout=PIPE, bufsize=1, universal_newlines=True) as data:
for line in data.stdout:
try:
tjson = json.loads(line)
except ValueError:
print("Error parsing data from: " + line)
else:
if 'id' in tjson or 'sensor_id' in tjson:
odd_data = True
temperature_F = wind_speed_mph = humidity = None
json_body = []
id = tjson['id'] if 'id' in tjson else tjson['sensor_id']
if 'temperature_F' in tjson:
temp = "{:03.1f}".format(tjson['temperature_F'])
json_body.append({'measurement': 'temperature', 'tags': {'id': id }, 'fields': { 'value': float(temp) } })
odd_data = False
if 'wind_speed_mph' in tjson:
wind_speed = tjson['wind_speed_mph']
json_body.append({'measurement': 'wind_speed', 'tags': {'id': id }, 'fields': { 'value': wind_speed } })
odd_data = False
if 'wind_avg_mi_h' in tjson:
wind_speed = tjson['wind_avg_mi_h']
def test_sdl_audio_mix_channels(self):
shutil.copyfile(path_from_root('tests', 'sounds', 'noise.ogg'), os.path.join(self.get_dir(), 'sound.ogg'))
open(os.path.join(self.get_dir(), 'sdl_audio_mix_channels.c'), 'w').write(self.with_report_result(open(path_from_root('tests', 'sdl_audio_mix_channels.c')).read()))
Popen([EMCC, '-O2', '--minify', '0', os.path.join(self.get_dir(), 'sdl_audio_mix_channels.c'), '--preload-file', 'sound.ogg', '-o', 'page.html']).communicate()
self.run_browser('page.html', '', '/report_result?1')
def _generate():
linux = b""
# machine-id is stable across boots, boot_id is not.
for filename in "/etc/machine-id", "/proc/sys/kernel/random/boot_id":
try:
with open(filename, "rb") as f:
value = f.readline().strip()
except IOError:
continue
if value:
linux += value
break
# Containers share the same machine id, add some cgroup
# information. This is used outside containers too but should be
# relatively stable across boots.
try:
with open("/proc/self/cgroup", "rb") as f:
linux += f.readline().strip().rpartition(b"/")[2]
except IOError:
pass
if linux:
return linux
# On OS X, use ioreg to get the computer's serial number.
try:
# subprocess may not be available, e.g. Google App Engine
# https://github.com/pallets/werkzeug/issues/925
from subprocess import Popen, PIPE
dump = Popen(["ioreg", "-c", "IOPlatformExpertDevice", "-d", "2"],
stdout=PIPE).communicate()[0]
match = re.search(b'"serial-number" = <([^>]+)', dump)
if match is not None:
return match.group(1)
except (OSError, ImportError):
pass
# On Windows, use winreg to get the machine guid.
try:
import winreg as wr
except ImportError:
try:
import _winreg as wr
except ImportError:
wr = None
if wr is not None:
try:
with wr.OpenKey(
wr.HKEY_LOCAL_MACHINE,
"SOFTWARE\\Microsoft\\Cryptography",
0,
wr.KEY_READ | wr.KEY_WOW64_64KEY,
) as rk:
guid, guid_type = wr.QueryValueEx(rk, "MachineGuid")
if guid_type == wr.REG_SZ:
return guid.encode("utf-8")
return guid
except WindowsError:
pass
from subprocess import Popen, PIPE, STDOUT, DEVNULL
from threading import Thread
import re
process = Popen("multimon-ng -a DTMF", stdout=PIPE, stderr=STDOUT, shell=True)
def speak(text):
Popen(f'sleep 1 && espeak "{text}"',
stdout=DEVNULL,
stderr=STDOUT,
shell=True)
def sendImage():
Thread(target=speak,
args=("sending image Sending image in Robot 24. Please wait.",
)).start()
Popen(
"raspistill -vf -hf -w 320 -h 240 -o /home/pi/out.jpg && python3 -m pysstv --vox /home/pi/out.jpg /home/pi/output.wav --mode Robot24BW --chan 1 && espeak \"sending image\" && mplayer /home/pi/output.wav",
stdout=DEVNULL,
stderr=STDOUT,
shell=True)
shellDTMFPattern = re.compile(r"DTMF:\s(.)")
while True:
line = process.stdout.readline().decode("utf-8")
if not line:
break
def run_file(cfg, filename):
argv = cfg.get_run_argv(relative_path(cfg.ROOTDIR, filename))
logger.debug(quote_argv(argv))
p = Popen(argv)
if p.wait() == 0:
return True
def speak(text):
Popen(f'sleep 1 && espeak "{text}"',
stdout=DEVNULL,
stderr=STDOUT,
shell=True)
def make(options, env, target, sources, cython_src, cfiles):
from schafer import SED_CMD
if options.ioscodesign is None:
error("You have to supply a CodeSign Authority with --ios-codesign")
exit()
if options.bare:
error("We don't support bare mode projects for iOS yet")
exit()
# Copy/update the skeleton
platform_build = join(target.project, 'ios')
ios_project = join(platform_build, 'ios_project')
project_name = options.project.split('.')[-1]
# Strip whitespace
project_name = re.sub(r'\s', '', project_name)
local_cfiles = []
for cfile in cfiles:
local_cfiles.append(basename(cfile))
if isdir(ios_project):
cmd = 'rm -rf %s' % ios_project
Popen(shlex.split(cmd), cwd=target.dist).communicate()
cmd = 'rsync -auqPm --exclude .svn --exclude .hg %s/ %s' % (target.dist,
ios_project)
Popen(shlex.split(cmd), cwd=target.dist).communicate()
# Modify the skeleton to suit the project
cmd = SED_CMD + "'s|\[\[PROJECT_NAME\]\]|%s|g' %s" % (
project_name, join(ios_project, 'ios.xcodeproj', 'project.pbxproj'))
Popen(shlex.split(cmd)).communicate()
cmd = SED_CMD + "'s|\[\[PROJECT_ARCHS\]\]|%s|g' %s" % (
env['ARCHS'], join(ios_project, 'ios.xcodeproj', 'project.pbxproj'))
Popen(shlex.split(cmd)).communicate()
cmd = SED_CMD + "'s|\[\[PROJECT_NAME\]\]|%s|g' %s" % (
project_name,
join(ios_project, 'ios.xcodeproj', 'xcuserdata', 'user.xcuserdatad',
'xcschemes', 'ios.xcscheme'))
Popen(shlex.split(cmd)).communicate()
cmd = SED_CMD + "'s|\[\[PROJECT_NAME\]\]|%s|g' %s" % (
project_name,
join(ios_project, 'ios.xcodeproj', 'xcuserdata', 'user.xcuserdatad',
'xcschemes', 'xcschememanagement.plist'))
Popen(shlex.split(cmd)).communicate()
cmd = SED_CMD + "'s|\[\[CODESIGN_DEVELOPER\]\]|%s|g' %s" % (
options.ioscodesign,
join(ios_project, 'ios.xcodeproj', 'project.pbxproj'))
Popen(shlex.split(cmd)).communicate()
cmd = SED_CMD + "'s|\[\[DEPLOY_TARGET\]\]|%s|g' %s" % (
env['IPHONEOS_DEPLOYMENT_TARGET'],
join(ios_project, 'ios.xcodeproj', 'project.pbxproj'))
Popen(shlex.split(cmd)).communicate()
project = XcodeProject.Load(
join(ios_project, 'ios.xcodeproj', 'project.pbxproj'))
for cf in cfiles:
project.add_file(cf)
for asset in options.assets:
project.add_file(abspath(join(target.project_root, asset)))
project.add_header_search_paths(join(target.dist, 'include'), False)
project.add_header_search_paths(join(target.dist, 'include', 'SDL2'),
False)
project.add_header_search_paths(join(target.dist, 'include', 'python2.7'),
False)
project.add_library_search_paths(join(target.dist, 'lib'), False)
project.save()
shutil.move(
join(ios_project, 'ios.xcodeproj', 'xcuserdata', 'user.xcuserdatad',
'xcschemes', 'ios.xcscheme'),
join(ios_project, 'ios.xcodeproj', 'xcuserdata', 'user.xcuserdatad',
'xcschemes', project_name + '.xcscheme'))
shutil.move(
join(ios_project, 'ios.xcodeproj', 'xcuserdata', 'user.xcuserdatad'),
join(ios_project, 'ios.xcodeproj', 'xcuserdata',
env['USER'] + '.xcuserdatad'))
shutil.move(join(ios_project, 'ios.xcodeproj'),
join(ios_project, project_name + '.xcodeproj'))
shutil.move(join(ios_project, 'ios', 'ios-Info.plist'),
join(ios_project, 'ios', project_name + '-Info.plist'))
shutil.move(join(ios_project, 'ios', 'ios-Prefix.pch'),
join(ios_project, 'ios', project_name + '-Prefix.pch'))
shutil.move(join(ios_project, 'ios'), join(ios_project, project_name))
app = join(ios_project, 'build/Release-iphoneos', project_name + '.app')
ipa = join(target.project_root, project_name + '.ipa')
if isdir(app):
cmd = 'rm -rf %s' % app
Popen(shlex.split(cmd)).communicate()
if isfile(ipa):
os.unlink(ipa)
cmd = "/usr/bin/xcodebuild"
# We do not use env here as it causes problems with xcodebuild!
Popen(shlex.split(cmd), cwd=ios_project, env=os.environ).communicate()
if not isdir(app) or not isfile(join(app, 'embedded.mobileprovision')):
error('error building iOS app')
return False
log('iOS app built, now we will codesign it and build the IPA')
# Hacky way to make an ad hoc distributable IPA
cmd = "/usr/bin/xcrun -sdk iphoneos PackageApplication \"%s\" -o \"%s\" --sign \"%s\" --embed \"%s\"" % (
app, ipa, options.ioscodesign, join(app, 'embedded.mobileprovision'))
# Use env here, or codesign may fail for using the wrong codesign_allocate binary
Popen(shlex.split(cmd), cwd=ios_project, env=env).communicate()
print cmd
if isfile(ipa):
log('iOS app built succesfully')
else:
error('error building iOS app')
return False
return True
def startJob(path):
proc = Popen('./runHaplomat.sh', cwd=path, shell=True)
proc.wait()
if not os.path.exists(path + '/results/estimatedHaplotypeFrequencies.dat'):
print('Results file was not created')
def load_jupyter_server_extension(nbapp):
"""serve the flask-app directory with flask server"""
Popen(["python", "./flask-app/main.py"])
def _reprepro(self, args):
os.chdir(self.path)
p = Popen(['/usr/bin/reprepro', '-Vb.'] + args.split(' '), stdout=PIPE, stderr=PIPE)
return (p.communicate(), p.returncode)
def processdata_reg(tdir):
dates=os.listdir(tdir)
dates.sort()
shifteddates=[]
sortedregions=None
with open(zoetrendfn,'w') as fp:
writer=csv.writer(fp)
#writer.writerow(['Date']+locs)
for date in dates:
tot=defaultdict(float)
totlon=defaultdict(float)
totreg={}
with open(join(tdir,date),'r') as fp:
dd=json.load(fp)
for d in dd.values():
region0=d["region"]
co=country(d["lat"],d["long"])
if co=="England+Wales": region=region0
else: region=co
if region not in totreg: totreg[region]=defaultdict(float)
for x in keys:
tot[x]+=d[x]
totreg[region][x]+=d[x]
#shdate=daytodate(datetoday(date)-1)# Go back a day because Zoe values are reported (and timestamped) the day after they occur
shdate=date# Change to simple recording by publication date. Adjust for lag later in the pipeline.
sr=sorted(list(totreg))
if sortedregions==None:
sortedregions=sr
writer.writerow(['Date']+sortedregions)
data={reg:[] for reg in sortedregions}
else:
assert sortedregions==sr
row=[shdate]
shifteddates.append(shdate)
for reg in sortedregions:
src=totreg[reg]
v=src["corrected_covid_positive"]/src["population"]*1e3
row.append("%.4g"%v)
data[reg].append(v)
writer.writerow(row)
print("Written %s"%zoetrendfn)
# Use this to cater for earlier versions of Python whose Popen()s don't have the 'encoding' keyword
def write(*s): p.write((' '.join(map(str,s))+'\n').encode('utf-8'))
trendfn='zoeregions.png'
p=Popen("gnuplot",shell=True,stdin=PIPE).stdin
write('set terminal pngcairo font "sans,13" size 1920,1280')
write('set bmargin 5;set lmargin 15;set rmargin 15;set tmargin 5')
write('set output "%s"'%trendfn)
write('set for [i=9:16] linetype i dashtype (20,7)')
write('set key left')
#write('set logscale y')
title="Zoe-estimated active cases per 1000 people, against publication date"
write('set title "%s"'%title)
#write('set xlabel "Days since '+desc+perstring+' reached %g'%thr)
write('set xdata time')
write('set format x "%Y-%m-%d"')
write('set timefmt "%Y-%m-%d"')
write('set grid xtics ytics lc rgb "#dddddd" lt 1')
write('set tics scale 3,0.5')
write('set xtics nomirror')
write('set xtics "2020-08-31", 86400*7')
write('set xtics rotate by 45 right offset 0.5,0')
s='plot '
for reg in sortedregions:
if s!='plot ': s+=', '
s+='"-" using 1:2 with lines lw 2 title "%s"'%reg
write(s)
for reg in sortedregions:
for (d,v) in zip(shifteddates,data[reg]): write(d,v)
write("e")
p.close()
print("Written %s"%trendfn)
def _launch_gateway(self, class_path, popen_kwargs=None):
"""
launch jvm gateway
:param popen_kwargs: Dictionary of kwargs to pass to Popen when spawning
the py4j JVM. This is a developer feature intended for use in
customizing how pyspark interacts with the py4j JVM (e.g., capturing
stdout/stderr).
"""
command = ["java"]
command.append("-cp")
command.append(class_path)
command.append("org.apache.spark.deploy.raydp.AppMasterEntryPoint")
# Create a temporary directory where the gateway server should write the connection
# information.
conn_info_dir = tempfile.mkdtemp()
try:
fd, conn_info_file = tempfile.mkstemp(dir=conn_info_dir)
os.close(fd)
os.unlink(conn_info_file)
env = dict(os.environ)
env["_RAYDP_APPMASTER_CONN_INFO_PATH"] = conn_info_file
# Launch the Java gateway.
popen_kwargs = {} if popen_kwargs is None else popen_kwargs
# We open a pipe to stdin so that the Java gateway can die when the pipe is broken
popen_kwargs["stdin"] = PIPE
# We always set the necessary environment variables.
popen_kwargs["env"] = env
# Don't send ctrl-c / SIGINT to the Java gateway:
def preexec_func():
signal.signal(signal.SIGINT, signal.SIG_IGN)
popen_kwargs["preexec_fn"] = preexec_func
# pylint: disable=R1732
proc = Popen(command, **popen_kwargs)
# Wait for the file to appear, or for the process to exit, whichever happens first.
while not proc.poll() and not os.path.isfile(conn_info_file):
time.sleep(0.1)
if not os.path.isfile(conn_info_file):
raise Exception(
"Java gateway process exited before sending its port number"
)
with open(conn_info_file, "rb") as info:
length = info.read(4)
if not length:
raise EOFError
gateway_port = struct.unpack("!i", length)[0]
finally:
shutil.rmtree(conn_info_dir)
gateway = JavaGateway(gateway_parameters=GatewayParameters(
port=gateway_port, auto_convert=True))
# Store a reference to the Popen object for use by the caller
# (e.g., in reading stdout/stderr)
gateway.proc = proc
return gateway
def autoreconf(options, buildout, version):
from subprocess import Popen
patch_libevent_configure_in(options, buildout, version)
process = Popen(['autoreconf'])
assert process.wait() == 0
change_install_name(options, buildout, version)
def run(self, result):
"""Run the test case and fill in result."""
base = os.path.basename(self.filename)
dirname = os.path.dirname(self.filename)
cmd = [self.exe.filename, base] + self.args
time_start = datetime.now()
print('\n*** BEGIN *** %r ***' % (
cmd,), file=result.environment.log)
print('*** %s *** %s ***' % (
self.uid, self.description,), file=result.environment.log)
print('*** START TIME: %s ***' % (
time_start.strftime("%Y-%m-%d %H:%M:%S")), file=result.environment.log)
result.environment.log.flush()
# Protect wrapper from Ctrl-C as long as test case is running
def handle_int(_signal, _frame):
"""Handle Ctrl-C signal."""
result.result = TestCodes.RESULT_SKIP
result.reason = TestCodes.REASON_ABORT
old_sig_int = signal.signal(signal.SIGINT, handle_int)
old_sig_alrm = signal.getsignal(signal.SIGALRM)
def prepare_child():
"""Setup child process."""
os.setsid()
signal.signal(signal.SIGINT, signal.SIG_IGN)
try:
TestCase.logger.debug('Running %r using %s in %s', cmd, self.exe, dirname)
try:
if result.environment.interactive:
proc = Popen(
cmd, executable=self.exe.filename,
shell=False, stdout=PIPE, stderr=PIPE,
close_fds=True, cwd=dirname,
preexec_fn=os.setsid
)
to_stdout, to_stderr = sys.stdout, sys.stderr
else:
devnull = open(os.path.devnull, 'rb')
try:
proc = Popen(
cmd, executable=self.exe.filename,
shell=False, stdin=devnull,
stdout=PIPE, stderr=PIPE, close_fds=True,
cwd=dirname, preexec_fn=prepare_child
)
finally:
devnull.close()
to_stdout = to_stderr = result.environment.log
signal.signal(signal.SIGCHLD, self.handle_shutdown)
if self.timeout:
old_sig_alrm = signal.signal(signal.SIGALRM, self.handle_shutdown)
signal.alarm(self.timeout)
self._run_tee(proc, result, to_stdout, to_stderr)
result.result = proc.wait()
except OSError:
TestCase.logger.error('Failed to execute %r using %s in %s', cmd, self.exe, dirname)
raise
finally:
signal.alarm(0)
signal.signal(signal.SIGALRM, old_sig_alrm)
signal.signal(signal.SIGINT, old_sig_int)
if result.reason == TestCodes.REASON_ABORT:
raise KeyboardInterrupt() # pylint: disable-msg=W1010
time_end = datetime.now()
time_delta = time_end - time_start
print('*** END TIME: %s ***' % (
time_end.strftime("%Y-%m-%d %H:%M:%S")), file=result.environment.log)
print('*** TEST DURATION (H:MM:SS.ms): %s ***' % (
time_delta), file=result.environment.log)
print('*** END *** %d ***' % (
result.result,), file=result.environment.log)
result.environment.log.flush()
result.duration = time_delta.total_seconds() * 1000
TestCase.logger.info('Test %r using %s in %s returned %s in %s ms', cmd, self.exe, dirname, result.result, result.duration)
self._translate_result(result)
msgstr.append(line)
if in_msgstr:
messages.append((msgid, msgstr))
return messages
files = sys.argv[1:]
# xgettext -n --keyword=_ $FILES
XGETTEXT=os.getenv('XGETTEXT', 'xgettext')
if not XGETTEXT:
print('Cannot extract strings: xgettext utility is not installed or not configured.',file=sys.stderr)
print('Please install package "gettext" and re-run \'./configure\'.',file=sys.stderr)
exit(1)
child = Popen([XGETTEXT,'--output=-','-n','--keyword=_'] + files, stdout=PIPE)
(out, err) = child.communicate()
messages = parse_po(out.decode('utf-8'))
f = open(OUT_CPP, 'w')
f.write("""
#include <QtGlobal>
// Automatically generated by extract_strings.py
#ifdef __GNUC__
#define UNUSED __attribute__((unused))
#else
#define UNUSED
#endif
def countdown(
stdscr,
alt_format=False,
font=DEFAULT_FONT,
blink=False,
critical=3,
quit_after=None,
text=None,
timespec=None,
title=None,
voice=None,
outfile=None,
no_seconds=False,
no_text_magic=True,
no_figlet=False,
no_window_title=False,
**kwargs
):
try:
sync_start, target = parse_timestr(timespec)
except ValueError:
raise click.BadParameter("Unable to parse TIME value '{}'".format(timespec))
curses_lock, input_queue, quit_event = setup(stdscr)
figlet = Figlet(font=font)
if title and not no_figlet:
try:
title = figlet.renderText(title)
except CharNotPrinted:
title = ""
input_thread = Thread(
args=(stdscr, input_queue, quit_event, curses_lock),
target=input_thread_body,
)
input_thread.start()
seconds_total = seconds_left = int(ceil((target - datetime.now()).total_seconds()))
try:
while seconds_left > 0 or blink or text:
figlet.width = stdscr.getmaxyx()[1]
if alt_format:
countdown_text = format_seconds_alt(
seconds_left, seconds_total, hide_seconds=no_seconds)
else:
countdown_text = format_seconds(seconds_left, hide_seconds=no_seconds)
if seconds_left > 0:
with curses_lock:
if not no_window_title:
os.write(stdout.fileno(), "\033]2;{0}\007".format(countdown_text).encode())
if outfile:
with open(outfile, 'w') as f:
f.write("{}\n{}\n".format(countdown_text, seconds_left))
stdscr.erase()
try:
draw_text(
stdscr,
countdown_text if no_figlet else figlet.renderText(countdown_text),
color=1 if seconds_left <= critical else 0,
fallback=title + "\n" + countdown_text if title else countdown_text,
title=title,
)
except CharNotPrinted:
draw_text(stdscr, "E")
if seconds_left <= 10 and voice:
voice_exec = "echo"
if os.path.exists("/usr/bin/say"):
voice_exec = "/usr/bin/say"
elif os.path.exists("/usr/bin/espeak"):
voice_exec = "/usr/bin/espeak"
Popen([voice_exec, "-v", voice, str(seconds_left)], stdout=DEVNULL, stderr=STDOUT)
# We want to sleep until this point of time has been
# reached:
sleep_target = sync_start + timedelta(seconds=1)
# If sync_start has microsecond=0, it might happen that we
# need to skip one frame (the very first one). This occurs
# when the program has been startet at, say,
# "2014-05-29 20:27:57.930651". Now suppose rendering the
# frame took about 0.2 seconds. The real time now is
# "2014-05-29 20:27:58.130000" and sleep_target is
# "2014-05-29 20:27:58.000000" which is in the past! We're
# already too late. We could either skip that frame
# completely or we can draw it right now. I chose to do the
# latter: Only sleep if haven't already missed our target.
now = datetime.now()
if sleep_target > now and seconds_left > 0:
try:
input_action = input_queue.get(True, (sleep_target - now).total_seconds())
except Empty:
input_action = None
if input_action == INPUT_PAUSE:
pause_start = datetime.now()
with curses_lock:
stdscr.erase()
try:
draw_text(
stdscr,
countdown_text if no_figlet else figlet.renderText(countdown_text),
color=3,
fallback=countdown_text,
)
except CharNotPrinted:
draw_text(stdscr, "E")
input_action = input_queue.get()
if input_action == INPUT_PAUSE:
sync_start += (datetime.now() - pause_start)
target += (datetime.now() - pause_start)
if input_action == INPUT_EXIT: # no elif here! input_action may have changed
break
elif input_action == INPUT_RESET:
sync_start, target = parse_timestr(timespec)
seconds_left = int(ceil((target - datetime.now()).total_seconds()))
continue
elif input_action == INPUT_LAP:
continue
sync_start = sleep_target
seconds_left = int(ceil((target - datetime.now()).total_seconds()))
if seconds_left <= 0:
# we could write this entire block outside the parent while
# but that would leave us unable to reset everything
with curses_lock:
curses.beep()
if text and not no_text_magic:
text = normalize_text(text)
if outfile:
with open(outfile, 'w') as f:
f.write("{}\n{}\n".format(text if text else "DONE", 0))
rendered_text = text
if text and not no_figlet:
try:
rendered_text = figlet.renderText(text)
except CharNotPrinted:
rendered_text = ""
if blink or text:
base_color = 1 if blink else 0
blink_reset = False
flip = True
slept = 0
extra_sleep = 0
while True:
with curses_lock:
os.write(stdout.fileno(), "\033]2;{0}\007".format("/" if flip else "\\").encode())
if text:
draw_text(
stdscr,
rendered_text,
color=base_color if flip else 4,
fallback=text,
)
else:
draw_text(stdscr, "", color=base_color if flip else 4)
if blink:
flip = not flip
try:
sleep_start = datetime.now()
input_action = input_queue.get(True, 0.5 + extra_sleep)
except Empty:
input_action = None
finally:
extra_sleep = 0
sleep_end = datetime.now()
if input_action == INPUT_PAUSE:
pause_start = datetime.now()
input_action = input_queue.get()
extra_sleep = (sleep_end - sleep_start).total_seconds()
if input_action == INPUT_EXIT:
# no elif here! input_action may have changed
return
elif input_action == INPUT_RESET:
sync_start, target = parse_timestr(timespec)
seconds_left = int(ceil((target - datetime.now()).total_seconds()))
blink_reset = True
break
slept += (sleep_end - sleep_start).total_seconds()
if quit_after and slept >= float(quit_after):
return
if blink_reset:
continue
finally:
with curses_lock:
if not no_window_title:
os.write(stdout.fileno(), "\033]2;\007".encode())
if outfile:
os.remove(outfile)
quit_event.set()
input_thread.join()
