def test(cmd):
util.info("")
util.info("- Starting " + cmd)
util.info("")
util.run(cmd)
commands.getoutput("rm -rf " + file1)
util.info("")
util.info("- Sending ./testclient localhost 2010 /testdata/file1.txt")
res = commands.getoutput("./testclient localhost 2010 /testdata/file1.txt")
arrival = util.get_stat2(res, "Stat-req-arrival")
dispatch = util.get_stat2(res, "Stat-req-dispatch")
read = util.get_stat2(res, "Stat-req-read")
complete = util.get_stat2(res, "Stat-req-complete")
print ""
print "dispatch = " + str(dispatch)
print "read = " + str(read)
print "complete = " + str(complete)
if dispatch >= 0 and read >=0 and complete >= 0 and dispatch + read <= complete:
util.good("You passed this test")
else:
util.error("Expected dispatch >= 0 and read >=0 and complete >= 0 and"
" dispatch + read <= complete:")
def build_libuv_windows(arch):
args = ["cmd", "/c", "vcbuild.bat", "release", "vs2017"]
if arch == "-32":
args.append("x86")
elif arch == "-64":
args.append("x64")
run(args, cwd=LIB_UV_DIR)
def test(cmd):
util.info("")
util.info("- Starting " + cmd)
util.info("")
util.run(cmd)
start = time.time()
clientlist = []
for i in range(0, NUM_CLIENT):
client = testit("Client-" + str(i))
clientlist.append(client)
client.start()
for client in clientlist:
client.join()
end = time.time()
util.info("Elapsed time (in seconds): " + str(end-start))
if util.is_server_alive(cmd) == -1:
util.error("Ouch! Server is dead!"
" Your bounded buffered may not be well protected");
def run_on_kubernetes(args):
create_gcloud_secret()
context = util.get_cluster_prefix()
namespace = util.get_current_namespace()
if len(args.number) == 0:
# Figure out the nodes based on the names of persistent disks, or just node 0 if none.
args.number = range(max(1,len(get_persistent_disks(context, namespace))))
if 'storage-projects' not in util.get_services():
util.run(['kubectl', 'create', '-f', 'conf/service.yaml'])
args.local = False # so tag is for gcloud
tag = util.get_tag(args, NAME, build)
if not args.tag:
tag = tag[:tag.rfind('-')] # get rid of the final -[service] part of the tag.
t = open(join('conf', '{name}.template.yaml'.format(name=NAME))).read()
ensure_ssh()
for number in args.number:
deployment_name = "{name}{number}".format(name=NAME, number=number)
ensure_persistent_disk_exists(context, namespace, number, args.size, args.type)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as tmp:
tmp.write(t.format(image = tag,
number = number,
gcloud_bucket = gcloud_bucket(namespace=namespace),
pd_name = pd_name(context=context, namespace=namespace, number=number),
health_delay = args.health_delay,
pull_policy = util.pull_policy(args)))
tmp.flush()
util.update_deployment(tmp.name)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('--computer',
default='/Users/lnyang/lab/qd2/qd2/compute_basic_feature.py')
parser.add_argument('--processed_dir', required=True,
help='dir of processed sf1 data')
parser.add_argument('--ticker_file', required=True)
parser.add_argument('--feature_base_dir', required=True)
parser.add_argument('--info_dir', required=True)
parser.add_argument('--use_mrx', action='store_true')
args = parser.parse_args()
for indicator, dimensions in ITEMS:
for dimension in dimensions:
if args.use_mrx:
dimension = util.getMrx(dimension)
folder = '%s-%s' % (indicator, dimension)
feature_dir = '%s/%s' % (args.feature_base_dir, folder)
info_file = '%s/%s' % (args.info_dir, folder)
if not shouldRun(feature_dir, info_file):
continue
cmd = ('%s --processed_dir=%s --ticker_file=%s --dimension=%s '
'--header=%s --feature_dir=%s --info_file=%s' % (
args.computer, args.processed_dir, args.ticker_file, dimension,
indicator, feature_dir, info_file))
util.run(cmd)
def build(tag, rebuild):
for service in SERVICES:
v = ['sudo', 'docker', 'build', '-t', full_tag(tag, service)]
if rebuild:
v.append("--no-cache")
v.append('.')
util.run(v, path=join(SCRIPT_PATH, 'images', service))
def build(tag, rebuild):
# Next build smc-hub, which depends on smc-hub-base.
v = ['sudo', 'docker', 'build', '-t', tag]
if rebuild: # will cause a git pull to happen
v.append("--no-cache")
v.append('.')
util.run(v, path=join(SCRIPT_PATH, 'image'))
def __init__(self, vlab, stdin=sys.stdin):
"""Instantiates a CLI object
:param vlab: Vlab class to be run"""
self.stdin = stdin
self.vlab = vlab
Cmd.__init__(self)
print('Starting CLI:\n')
# Setup history if readline is available
try:
import readline
except ImportError:
pass
else:
history_path = os.path.expanduser('~/.vlab_history')
if os.path.isfile(history_path):
readline.read_history_file(history_path)
atexit.register(
lambda: readline.write_history_file(history_path))
while True:
try:
if self.isatty():
run('stty sane')
self.cmdloop()
break
except KeyboardInterrupt:
print('\nInterrupt\n')
def init_tri_from_mono(model, root_dir, mono_dir, tri_mlf, mono_list, tri_list):
"""
Convert a monophone model and triphone mlf to triphones
"""
## Create the xword directory and the current output directory
output_dir = '%s/HMM-0-0' %root_dir
util.create_new_dir(root_dir)
util.create_new_dir(output_dir)
mktri_hed = '%s/mktri.hed' %output_dir
hhed_log = '%s/hhed_clone_mono.log' %output_dir
## Create an HHEd script to clone monophones to triphones
fh = open(mktri_hed, 'w')
for line in open(mono_list):
mono = line.strip()
fh.write('TI T_%s {(%s).transP}\n' %(mono, mono))
fh.write('CL %s\n' %tri_list)
fh.close()
## Run HHEd to clone monophones and tie transition matricies
cmd = 'HHEd -A -T 1 -H %s/MMF' %mono_dir
cmd += ' -M %s' %output_dir
cmd += ' %s %s > %s' %(mktri_hed, mono_list, hhed_log)
if model.local: os.system(cmd)
else: util.run(cmd, output_dir)
return output_dir
def predict(experiment_dir, config_map, predict_meta_file):
result_dir = getResultDir(experiment_dir)
util.maybeMakeDir(result_dir)
result_file = getResultPath(result_dir)
data_dir = getDataDir(experiment_dir)
data_file = getDataPath(data_dir)
if config_map['use_classification']:
label_file = getLabelPath(data_dir)
else:
label_file = getRlabelPath(data_dir)
meta_file = getMetaPath(data_dir)
model_dir = getModelDir(experiment_dir)
imputer_dir = getImputerDir(experiment_dir)
model_prefix = '%s-' % getModelName(config_map)
model_suffix = '-%d' % config_map['train_window']
imputer_prefix = 'imputer-'
imputer_suffix = '-%d' % config_map['train_window']
cmd = ('%s/predict_all.py --data_file=%s --label_file=%s '
'--meta_file=%s --model_dir=%s --model_prefix="%s" '
'--model_suffix="%s" --imputer_dir=%s --imputer_prefix="%s" '
'--imputer_suffix="%s" --prediction_window=%d '
'--delay_window=%d --predict_meta_file=%s --result_file=%s' % (
CODE_DIR, data_file, label_file, meta_file,
model_dir, model_prefix, model_suffix,
imputer_dir, imputer_prefix, imputer_suffix,
config_map['predict_window'],
config_map['delay_window'], predict_meta_file,
result_file))
util.run(cmd)
def test(cmd):
print ""
print "Starting " + cmd
util.run(cmd)
clientlist = []
start = time.time()
for i in range(0, NUM_CLIENT):
client = testit("Client-" + str(i))
client.setDaemon(True)
clientlist.append(client)
client.start()
for client in clientlist:
client.join()
end = time.time()
if util.is_server_alive(cmd) == -1:
util.error("Ouch! Server is dead!"
" Your bounded buffered may not be well protected");
print "Elapsed time (in seconds): " + str(end-start)
if end - start > EXPECTED_TIME:
util.error("your server is not multithreaded")
def test(cmd):
global expected
global got
global count
util.info("")
util.info("- Starting " + cmd)
util.info("")
util.run(cmd)
start = time.time()
clientlist = []
expected = []
for i in range(1, NUM_CLIENT):
expected.append(commands.getoutput("cat ./testdata/file%s.txt" % str(i)))
commands.getoutput("rm -rf %s" % tmpfile)
for i in range(0, NUM_CLIENT):
client = testit("Client-" + str(i), i)
clientlist.append(client)
client.start()
time.sleep(0.3)
for client in clientlist:
client.join()
end = time.time()
util.info("Elapsed time (in seconds): " + str(end-start))
time.sleep(CGI_SPIN_TIME + 2)
res = commands.getoutput("cat %s" % tmpfile)
if util.is_server_alive(cmd) == -1:
util.error("Ouch! Server is dead!"
" Your bounded buffered may not be well protected");
pos0 = res.find(expected[0])
pos1 = res.find(expected[1])
pos2 = res.find(expected[2])
passed = pos0 > 0 and pos1 > 0 and pos2 > 0 and pos0 < pos1 and pos1 < pos2
util.info(res)
if passed:
print ""
print "#####################################"
print "GOOD! you implement SFF correctly"
print "#####################################"
print ""
count = count + 1
else:
print ""
print "#####################################"
print "Oh oh! ERROR ERROR!"
print "SFF is not implemented correctly"
print "#####################################"
print ""
sys.exit(-1)
def gn_gen(mode):
os.environ["DENO_BUILD_MODE"] = mode
# Rather than using gn gen --args we write directly to the args.gn file.
# This is to avoid quoting/escaping complications when passing overrides as
# command-line arguments.
args_filename = os.path.join(build_path(), "args.gn")
# Check if args.gn exists, and if it was auto-generated or handcrafted.
existing_gn_args, hand_edited = read_gn_args(args_filename)
# If args.gn wasn't handcrafted, regenerate it.
if hand_edited:
print "%s: Using gn options from hand edited '%s'." % (mode,
args_filename)
gn_args = existing_gn_args
else:
print "%s: Writing gn options to '%s'." % (mode, args_filename)
gn_args = generate_gn_args(mode)
if gn_args != existing_gn_args:
write_gn_args(args_filename, gn_args)
for line in gn_args:
print " " + line
run([third_party.gn_path, "gen", build_path()],
env=third_party.google_env())
def install_kubernetes(args):
version = args.version
if not version:
version = util.run("curl -s https://github.com/kubernetes/kubernetes/releases | grep kubernetes/tree",
get_output=True).splitlines()[0].split("tree/v")[1].split('"')[0]
print("using latest version '%s'"%version)
install_path = os.path.join(os.environ['HOME'], 'install')
link_path = os.path.join(os.environ['HOME'], 'kubernetes')
if not os.path.exists(install_path):
os.makedirs(install_path)
if os.path.exists(link_path) and not os.path.islink(link_path):
raise RuntimeError("Please manually remove '%s'"%link_path)
target_path = os.path.join(install_path, 'kubernetes-v%s'%version)
if not os.path.exists(target_path):
target = os.path.join(install_path, 'kubernetes.tar.gz')
if os.path.exists(target):
os.unlink(target)
util.run(['wget', 'https://github.com/kubernetes/kubernetes/releases/download/v%s/kubernetes.tar.gz'%version],
path = install_path)
util.run(['tar', 'zvxf', target], path=install_path)
os.unlink(target)
shutil.move(os.path.join(install_path, 'kubernetes'), target_path)
if os.path.exists(link_path):
os.unlink(link_path)
os.symlink(target_path, link_path)
def mixup(model, root_dir, prev_dir, model_list, mix_size, estimateVarFloor=0):
"""
Run HHEd to initialize a mixup to mix_size gaussians
"""
output_dir = '%s/HMM-%d-%d' %(root_dir, mix_size, 0)
util.create_new_dir(output_dir)
## Make the hed script
mix_hed = '%s/mix_%d.hed' %(output_dir, mix_size)
fh = open(mix_hed, 'w')
if estimateVarFloor:
fh.write('LS %s/stats\n' %prev_dir)
fh.write('FA 0.1\n')
fh.write('MU %d {(sil,sp).state[2-%d].mix}\n' %(2*mix_size,model.states-1))
fh.write('MU %d {*.state[2-%d].mix}\n' %(mix_size, model.states-1))
fh.close()
hhed_log = '%s/hhed_mix.log' %output_dir
cmd = 'HHEd -A -D -T 1 -H %s/MMF -M %s' %(prev_dir, output_dir)
cmd += ' %s %s > %s' %(mix_hed, model_list, hhed_log)
if model.local == 1: os.system(cmd)
else: util.run(cmd, output_dir)
return output_dir
def mixdown_mono(model, root_dir, prev_dir, phone_list):
"""
Run HHEd to mixdown monophones
"""
output_dir = '%s/HMM-1-0' %root_dir
util.create_new_dir(output_dir)
## Create the full list of possible triphones
phones = open(phone_list).read().splitlines()
non_sil_phones = [p for p in phones if p not in ['sp', 'sil']]
## Make the hed script
mixdown_hed = '%s/mix_down.hed' %output_dir
fh = open(mixdown_hed, 'w')
fh.write('MD 12 {(sil,sp).state[2-%d].mix}\n' %(model.states-1))
for phone in non_sil_phones:
fh.write('MD 1 {%s.state[2-%d].mix}\n' %(phone, model.states-1))
fh.close()
hhed_log = '%s/hhed_mixdown.log' %output_dir
cmd = 'HHEd -A -D -T 1 -H %s/MMF -M %s' %(prev_dir, output_dir)
cmd += ' %s %s > %s' %(mixdown_hed, phone_list, hhed_log)
if model.local == 1: os.system(cmd)
else: util.run(cmd, output_dir)
return output_dir
def make_sparse_file(size="1M"):
path = "/tmp/block"
try:
os.unlink(path)
except:
pass
run([ "/bin/dd", "if=/dev/zero", "of=%s" % path, "seek=1M", "count=0", "bs=1"])
return path
def add_file_to_itunes(filename, cue=None):
filename = os.path.abspath(filename)
print "Adding file %s to iTunes" % filename
run('osascript', ADD_SONG, filename)
if cue is not None:
name = os.path.splitext(os.path.basename(filename))[0]
run('osascript', SET_TAG, name, cue.artist, cue.album, getattr(cue, 'genre'), cue.getname(name),
cue.track_count, cue.track_index(name), cue.year)
def createSetupLock():
global locked
try:
run("mkdir %s" % env.SETUP_LOCK)
locked = True
except:
logger.error("!!! First-time setup is already in progress. Exiting...")
sys.exit(1)
def forward_test(args):
v = util.get_pods(run='rethinkdb-proxy')
v = [x for x in v if x['STATUS'] == 'Running']
if len(v) == 0:
print("No rethinkdb-proxy nodes available")
else:
print("\n\nYou may connect to rethinkdb-proxy on localhost:\n\n")
util.run(['kubectl', 'port-forward', v[0]['NAME'], '28015:28015'])
def maybeRun(args, input_dir, feature, windows):
if not shouldRun(args.feature_base_dir, args.info_dir, feature, windows):
return
cmd = ('%s --input_dir=%s --output_dir=%s --feature=%s --windows=%s '
'--ticker_file=%s --info_base_dir=%s' % (
args.computer, input_dir, args.feature_base_dir, feature, windows,
args.ticker_file, args.info_dir))
util.run(cmd)
def resize_cluster(args):
prefix = util.get_cluster_prefix()
if args.name:
group = '{prefix}-{name}-minion-group'.format(prefix=prefix, name=args.name)
else:
group = '{prefix}-minion-group'.format(prefix=prefix)
util.run(['gcloud', 'compute', 'instance-groups', 'managed', 'resize', group,
'--size', str(args.size)])
def run_all():
x = util.get_deployments()
for name in ['rethinkdb-proxy', 'smc-webapp-static', 'smc-hub', 'haproxy']:
if name not in x:
if name == 'rethinkdb0':
name = 'rethinkdb'
print('\n******\nRUNNING {name}\n******\n'.format(name=name))
util.run([join(SCRIPT_PATH,'..',name,'control.py'), 'run'])
def test(cmd):
global count
util.info("")
util.info("- Starting " + cmd)
util.info("")
util.run(cmd)
commands.getoutput("rm -rf " + file1)
commands.getoutput("rm -rf " + file2)
os.system("./testclient localhost 2010 /output.cgi >> /dev/null &");
time.sleep(0.3)
# os.system("./testclient localhost 2010 /testdata/file-sff-large.txt > /tmp/file1 &");
os.system("./testclient localhost 2010 /testdata/file-sff-large.txt > %s &" % file1);
# time.sleep(0.1)
os.system("./testclient localhost 2010 /output.cgi >> /dev/null &");
# time.sleep(0.1)
# os.system("./testclient localhost 2010 /testdata/file-sff-small.txt > /tmp/file2 &");
os.system("./testclient localhost 2010 /testdata/file-sff-small.txt > %s &" % file2);
time.sleep(3 * CGI_SPIN_TIME)
time.sleep(5) # this doesn't hurt
if util.is_server_alive(cmd) == -1:
util.error("Ouch! Server is dead!"
" Your bounded buffered may not be well protected");
# time1 = os.path.getmtime(file1)
# time2 = os.path.getmtime(file2)
time_decimal1 = str(os.path.getmtime(file1))
time_decimal2 = str(os.path.getmtime(file2))
# print time_decimal1
# print time_decimal2
passed = Decimal(time_decimal1).compare_total(Decimal(time_decimal2)) != -1
# passed = time1 > time2
if passed:
print ""
print "#####################################"
print "GOOD! you implement SFF correctly"
print "#####################################"
print ""
count = count + 1
else:
print ""
print "#####################################"
print "Oh oh! ERROR ERROR!"
print "SFF is not implemented correctly"
print "#####################################"
print ""
sys.exit(-1)
util.kill()
def maybe_download_sysroot():
if sys.platform.startswith('linux'):
run([
'python',
os.path.join(root_path,
'build/linux/sysroot_scripts/install-sysroot.py'),
'--arch=amd64'
],
env=google_env())
def checkout(self, url, depth="infinity"):
"""Check out a working copy from the given URL.
Args:
url: The Subversion repository URL to check out.
depth: The depth of the working copy root.
"""
assert not self.exists()
util.run(["svn", "checkout", "--depth=" + depth, url, self.path()])
def build(self, docker):
if not os.path.isdir('docker/%s' % docker):
return
# Copy our public key to docker directory to propegate inside our containers
shutil.copy(os.path.expanduser("~") + '/.ssh/id_rsa.pub', 'docker/%s/root' % docker)
run('vagrant ssh --command "cd %s/%s && docker build -t=\"%s\" ."' % (self.baseDir, docker, docker), cwd="docker/%s" % docker)
def commit(self, message, path=""):
"""Commit scheduled changes to the source repository.
Args:
message: The commit message to use.
path: The path to commit.
"""
assert self.exists()
util.run(["svn", "commit", "-m", message, self.path(path)])
def init_repo(cache_dir, remote, fetch=True):
repo = get_repo_path(cache_dir, hash_str(remote))
if not is_git_dir(repo):
run(["git", "clone", "--bare", "-q", "--", remote, repo],
stdin=DEVNULL, check=True)
elif fetch:
run(["git", "-C", repo, "fetch", "--all", "-p", "-q"],
stdin=DEVNULL, check=True)
return repo
def create_install_path():
"""
Create install-tmp/, which contains files that will get
installed into the k8s host node when the daemon starts.
"""
remove_install_path()
os.makedirs("install-tmp")
shutil.copyfile("../driver/smc-storage.py", 'install-tmp/smc-storage')
util.run(['git', 'clone', 'https://github.com/sagemathinc/gke-zfs'], path=join(SCRIPT_PATH, 'install-tmp'))
def make_snapshot(self, name):
# Delete all the snapshots for this VM
for snap in self.get_domain(name).listAllSnapshots():
show('Deleting snapshot %s' % snap.getName())
snap.delete()
show('Creating new snapshot..')
stdout, stderr, rc = util.run(
['virsh', 'snapshot-create', '--domain', name])
show('Created!')
if rc != 0:
raise RuntimeError("Could not create snapshot for %s" % name)
def create_vm(self, name, template=locals.TEMPLATE_NAME):
# TODO: check if the VM with the name of name exists
# Check whether template VM exists
show('Checking for existence of template')
template_domain = self.get_domain(template)
# TODO: check if it is running, if is, print down warning and shut
# it down
if template_domain:
show('Cloning..')
# Find out next available MAC address in the pool
new_mac = self.get_next_free_mac()
output, errors, rc = util.run([
'virt-clone',
'-o',
template,
'--auto-clone',
'-n',
name,
'-m',
new_mac,
])
if rc != 0:
raise RuntimeError("Could not clone VM %s" % template)
show('Cloning successful')
# TODO: check that it started, if not, wait
show('Starting..')
self.start(name)
sleep(10)
# Macs are tied to the IPs
last_mac_segment = new_mac.split(':')[-1]
ip = locals.IP_BASE + '%s' % int(last_mac_segment)
show('IP determined: %s' % ip)
hostname = util.normalize_hostname(ip)
return VM(name=name,
backend=self,
hostname=hostname,
domain=locals.DOMAIN,
ip=ip)
def download(ticker_file, download_dir, overwrite):
tickers = util.readTickers(ticker_file)
for ticker in tickers:
download_file = '%s/%s.csv' % (download_dir, ticker)
if os.path.isfile(download_file) and not overwrite:
continue
cmd = '%s --quiet "%s%s" -O %s' % (
WGET, BASE_URL, ticker, download_file)
for i in range(RETRIES):
result = util.run(cmd, check=False)
if result == 0:
break
if os.path.isfile(download_file):
os.remove(download_file)
def run_exec_time(deno_exe, build_dir):
hyperfine_exe = third_party.get_prebuilt_tool_path("hyperfine")
benchmark_file = os.path.join(build_dir, "hyperfine_results.json")
run([
hyperfine_exe, "--ignore-failure", "--export-json", benchmark_file,
"--warmup", "3"
] + [
deno_exe + " run " + " ".join(args)
for [_, args] in exec_time_benchmarks
])
hyperfine_results = read_json(benchmark_file)
results = {}
for [[name, _], data] in zip(exec_time_benchmarks,
hyperfine_results["results"]):
results[name] = {
"mean": data["mean"],
"stddev": data["stddev"],
"user": data["user"],
"system": data["system"],
"min": data["min"],
"max": data["max"]
}
return results
def update_slurm_node_addrs(compute):
for node_name, operation in operations.items():
try:
# Do this after the instances have been initialized and then wait
# for all operations to finish. Then updates their addrs.
wait_for_operation(compute, cfg.project, operation)
pid = util.get_pid(node_name)
my_fields = 'networkInterfaces(name,network,networkIP,subnetwork)'
instance_networks = compute.instances().get(
project=cfg.project,
zone=cfg.partitions[pid].zone,
instance=node_name,
fields=my_fields).execute()
instance_ip = instance_networks['networkInterfaces'][0][
'networkIP']
util.run(
f"{SCONTROL} update node={node_name} nodeaddr={instance_ip}")
log.info("Instance " + node_name + " is now up")
except Exception:
log.exception(f"Error in adding {node_name} to slurm")
def init_rawfile(volume_id, size):
import time
import rawfile_util
from pathlib import Path
from util import run
img_dir = rawfile_util.img_dir(volume_id)
img_dir.mkdir(exist_ok=True)
img_file = Path(f"{img_dir}/disk.img")
if img_file.exists():
return
rawfile_util.patch_metadata(
volume_id,
{
"volume_id": volume_id,
"created_at": time.time(),
"img_file": img_file.as_posix(),
"size": size,
},
)
run(f"truncate -s {size} {img_file}")
run(f"mkfs.ext4 {img_file}")
def fetch_favicon(link_dir, link, timeout=TIMEOUT):
"""download site favicon from google's favicon api"""
if os.path.exists(os.path.join(link_dir, 'favicon.ico')):
return {'output': 'favicon.ico', 'status': 'skipped'}
CMD = [
CURL_BINARY,
'--max-time',
str(timeout),
*(() if CHECK_SSL_VALIDITY else ('--insecure', )),
'https://www.google.com/s2/favicons?domain={}'.format(
domain(link['url'])),
]
fout = open('{}/favicon.ico'.format(link_dir), 'w')
end = progress(timeout, prefix=' ')
try:
run(CMD, stdout=fout, stderr=DEVNULL, cwd=link_dir,
timeout=timeout) # favicon.ico
fout.close()
end()
chmod_file('favicon.ico', cwd=link_dir)
output = 'favicon.ico'
except Exception as e:
fout.close()
end()
print(' {}Failed: {} {}{}'.format(ANSI['red'],
e.__class__.__name__, e,
ANSI['reset']))
print(' Run to see full output:')
print(' {}'.format(' '.join(CMD)))
output = e
return {
'cmd': CMD,
'output': output,
}
def main(argv):
if len(argv) == 2:
build_dir = sys.argv[1]
elif len(argv) == 1:
build_dir = build_path()
else:
print "Usage: tools/benchmark.py [build_dir]"
sys.exit(1)
deno_path = os.path.join(build_dir, "deno")
benchmark_file = os.path.join(build_dir, "benchmark.json")
os.chdir(root_path)
import_data_from_gh_pages()
# TODO: Use hyperfine in //third_party
run(["hyperfine", "--export-json", benchmark_file, "--warmup", "3"] +
[deno_path + " " + " ".join(args) for [_, args] in benchmarks])
all_data = read_json(data_file)
benchmark_data = read_json(benchmark_file)
sha1 = run_output(["git", "rev-parse", "HEAD"]).strip()
new_data = {
"created_at": time.strftime("%Y-%m-%dT%H:%M:%SZ"),
"sha1": sha1,
"binary_size": os.path.getsize(deno_path),
"benchmark": {}
}
for [[name, _], data] in zip(benchmarks, benchmark_data["results"]):
new_data["benchmark"][name] = {
"mean": data["mean"],
"stddev": data["stddev"],
"user": data["user"],
"system": data["system"],
"min": data["min"],
"max": data["max"]
}
all_data.append(new_data)
write_json(data_file, all_data)
def eslint():
script = os.path.join(third_party_path, "node_modules", "eslint", "bin",
"eslint")
# Find all *directories* in the main repo that contain .ts/.js files.
source_files = get_sources(root_path, [
"*.js",
"*.ts",
":!:cli/tests/swc_syntax_error.ts",
":!:std/**/testdata/*",
":!:std/**/node_modules/*",
":!:cli/compilers/wasm_wrap.js",
":!:cli/tests/error_syntax.js",
":!:cli/tests/lint/**",
":!:cli/tests/encoding/**",
":!:cli/dts/**",
":!:cli/tsc/*typescript.js",
":!:cli/bench/node*.js",
])
if source_files:
max_command_len = 30000
pre_command = ["node", script, "--max-warnings=0", "--"]
chunks = [[]]
cmd_len = len(" ".join(pre_command))
for f in source_files:
if cmd_len + len(f) > max_command_len:
chunks.append([f])
cmd_len = len(" ".join(pre_command))
else:
chunks[-1].append(f)
cmd_len = cmd_len + len(f) + 1
for c in chunks:
print_command("eslint", c)
# Set NODE_PATH so we don't have to maintain a symlink in root_path.
env = os.environ.copy()
env["NODE_PATH"] = os.path.join(root_path, "third_party",
"node_modules")
run(pre_command + c, shell=False, env=env, quiet=True)
def stage_package(self):
super().stage_package_before()
run("mkdir -p {}".format(self.stage_include_dir_path))
run("rm -rf {}/*".format(self.package_stage_include_dir_path))
run("cp -rv {}/{} {}".format(self.package_clone_dir_path,
header_cp_pattern,
self.package_stage_include_dir_path))
super().stage_package_after()
def run_on_kubernetes(args):
util.ensure_secret_exists('sendgrid-api-key', 'sendgrid')
util.ensure_secret_exists('zendesk-api-key', 'zendesk')
args.local = False # so tag is for gcloud
if args.replicas is None:
args.replicas = util.get_desired_replicas(NAME, 2)
tag = util.get_tag(args, NAME, build)
opts = {
'image_hub': tag,
'replicas': args.replicas,
'pull_policy': util.pull_policy(args),
'min_read_seconds': args.gentle,
'smc_db_hosts': args.database_nodes,
'smc_db_pool': args.database_pool_size,
'smc_db_concurrent_warn': args.database_concurrent_warn
}
if args.database_nodes == 'localhost':
from argparse import Namespace
ns = Namespace(tag=args.rethinkdb_proxy_tag, local=False)
opts['image_rethinkdb_proxy'] = util.get_tag(ns, 'rethinkdb-proxy',
build)
filename = 'smc-hub-rethinkdb-proxy.template.yaml'
else:
filename = '{name}.template.yaml'.format(name=NAME)
t = open(join('conf', filename)).read()
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as tmp:
r = t.format(**opts)
#print(r)
tmp.write(r)
tmp.flush()
util.update_deployment(tmp.name)
if NAME not in util.get_services():
util.run(['kubectl', 'expose', 'deployment', NAME])
def fetch_git(link_dir, link, timeout=TIMEOUT):
"""download full site using git"""
url_is_clonable = (domain(link['url']) in GIT_DOMAINS
or link['url'].endswith('.git')
or link['type'] == 'git')
if not url_is_clonable:
return {'output': None, 'status': 'skipped'}
git_dir = os.path.join(link_dir, 'git')
if os.path.exists(git_dir):
return {'output': 'git', 'status': 'skipped'}
os.makedirs(git_dir, exist_ok=True)
output = 'git'
CMD = [
GIT_BINARY,
'clone',
'--mirror',
'--recursive',
*(() if CHECK_SSL_VALIDITY else ('-c', 'http.sslVerify=false')),
without_query(without_fragment(link['url'])),
]
end = progress(timeout, prefix=' ')
try:
result = run(CMD,
stdout=PIPE,
stderr=PIPE,
cwd=git_dir,
timeout=timeout + 1)
end()
if result.returncode == 128:
# ignore failed re-download when the folder already exists
pass
elif result.returncode > 0:
print(' got git response code {}:'.format(
result.returncode))
raise Exception('Failed git download')
except Exception as e:
end()
output = e
print_error_hints(cmd=CMD, pwd=link_dir, err=e)
return {
'cmd': CMD,
'output': output,
}
def rio_bind_workload(stack, vname, wrklname):
fullVolName = (f"{stack}/{vname}")
fullWklName = (f"{stack}/{wrklname}")
util.wait_for_state(fullVolName, "active")
util.run(f"rio exec {fullVolName} touch /persistentvolumes/helloworld")
util.run(f"rio run -n {fullWklName} -v data-{vname}-0:/data nginx")
util.run(f"rio wait {fullWklName}")
output = util.run(f"rio exec {fullWklName} ls /data")
print(f'OUTPUT = {output}')
return output
def dlint():
executable_path = get_prebuilt_tool_path("dlint")
# Find all *directories* in the main repo that contain .ts/.js files.
source_files = get_sources(root_path, [
"*.js",
"*.ts",
":!:cli/tests/swc_syntax_error.ts",
":!:cli/tests/038_checkjs.js",
":!:cli/tests/error_008_checkjs.js",
":!:std/**/testdata/*",
":!:std/**/node_modules/*",
":!:cli/bench/node*.js",
":!:cli/compilers/wasm_wrap.js",
":!:cli/dts/**",
":!:cli/tests/encoding/**",
":!:cli/tests/error_syntax.js",
":!:cli/tests/lint/**",
":!:cli/tests/tsc/**",
":!:cli/tsc/*typescript.js",
])
if source_files:
max_command_len = 30000
pre_command = [executable_path, "run"]
chunks = [[]]
cmd_len = len(" ".join(pre_command))
for f in source_files:
if cmd_len + len(f) > max_command_len:
chunks.append([f])
cmd_len = len(" ".join(pre_command))
else:
chunks[-1].append(f)
cmd_len = cmd_len + len(f) + 1
for c in chunks:
print_command("dlint", c)
run(pre_command + c, shell=False, quiet=True)
def create_cluster(args):
if args.min_nodes > args.max_nodes:
args.max_nodes = args.min_nodes
if args.cost:
c = cost_of_cluster(node_size = args.node_size,
node_disk_type = 'pd-ssd' if args.node_ssd else 'pd-standard',
node_disk_size = args.node_disk_size,
min_nodes = args.min_nodes,
max_nodes = args.max_nodes,
master_size = args.master_size,
master_disk_type = 'pd-ssd', # forced by k8s
master_disk_size = args.master_disk_size,
preemptible = not args.non_preemptible)
print(c)
return
# see https://github.com/kubernetes/kubernetes/blob/master/cluster/gce/config-default.sh for env vars
env = {
'KUBE_ENABLE_CLUSTER_MONITORING' : 'google',
'KUBE_GCE_ZONE' : args.zone,
'NODE_SIZE' : args.node_size,
'NUM_NODES' : str(args.min_nodes),
'MASTER_SIZE' : args.master_size,
'MASTER_DISK_SIZE' : "%sGB"%args.master_disk_size,
'NODE_DISK_TYPE' : 'pd-ssd' if args.node_ssd else 'pd-standard',
'NODE_DISK_SIZE' : "%sGB"%args.node_disk_size,
'PREEMPTIBLE_NODE' : 'false' if args.non_preemptible else 'true',
'KUBE_GCE_INSTANCE_PREFIX' : 'k8s',
'KUBE_ENABLE_NODE_AUTOSCALER' : 'true' if args.min_nodes < args.max_nodes else 'false',
'KUBE_AUTOSCALER_MIN_NODES' : str(args.min_nodes),
'KUBE_AUTOSCALER_MAX_NODES' : str(args.max_nodes)
}
env.update(os.environ)
util.run(join(CLUSTER, 'kube-up.sh'), env=env)
update_firewall()
def container_registry_has_image(full_image_name, docker_path):
env = os.environ.copy()
env["DOCKER_CLI_EXPERIMENTAL"] = "enabled" # needed for "docker manifest"
proc = run(docker_path,
"manifest",
"inspect",
"--insecure",
full_image_name,
env=env,
check=False,
quiet=True)
image_found = proc.returncode == 0
log.debug(
"Image {} in registry".format("found" if image_found else "not found"))
return image_found
def available_cluster_ip_range():
# Determine available ip range. TODO: this is NOT rock solid -- it's just enough to
# prevent collisions with other clusters, which is all we need. However, be nervous.
routes = json.loads(util.run(['gcloud', '--format=json', 'compute', 'routes', 'list'], get_output=True))
n = 245
ranges = [route['destRange'] for route in routes]
ranges.sort()
print(ranges)
while True:
for route in ranges:
if route.startswith('10.%s'%n):
n += 1
continue
break
return '10.%s.0.0/16'%n
def postmsg(opts):
"""Helpful info to show at the end for release manager."""
cmd = ['git', 'rev-parse', 'remotes/origin/stable']
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
current_release = ret.stdout.strip()
cmd = [
'git', 'log', '--format=%h (%aN) %s', '--no-merges',
f'remotes/origin/stable..{opts.tag}'
]
ret = util.run(opts, cmd, encoding='utf-8', stdout=subprocess.PIPE)
shortlog = ret.stdout.strip()
print(f"""
Here's the short log since the last release.
{shortlog}
To push release to the public:
git push origin {opts.commit}:stable {opts.tag} -n
NB: People will start upgrading to this version immediately.
To roll back a release:
git push origin --force {current_release}:stable -n
""")
def testExecuteReorder(self):
GEN_FILE = TEST + "testlib/testlib_patched-reorder.dll"
changed_bytes = reorder.do_reordering(self.f.blocks, gen_patched=True)
# check the number of changed bytes
self.assertEqual(len(changed_bytes), 56)
# check the numner of generated files
self.assertEqual(len(glob.glob(GEN_FILE)), 1)
# check the output of the generated files
output = util.run("%s %s" % (self.PROG, GEN_FILE), 10)
self.assertTrue(self.EXPECT in output,
"%s: %s %s" % (GEN_FILE, self.EXPECT, output))
# remove the generated files
map(os.remove, glob.glob(GEN_FILE))
def main(args):
# Delete it if already there so we ensure we get the correct version if the
# version number in this script changes.
clean_dir("deps")
print("Cloning libuv...")
run([
"git", "clone", "--quiet", "--depth=1",
"https://github.com/libuv/libuv.git", LIB_UV_DIR
])
print("Getting tags...")
run(["git", "fetch", "--quiet", "--depth=1", "--tags"], cwd=LIB_UV_DIR)
print("Checking out libuv " + LIB_UV_VERSION + "...")
run(["git", "checkout", "--quiet", LIB_UV_VERSION], cwd=LIB_UV_DIR)
# TODO: Pin gyp to a known-good commit. Update a previously downloaded gyp
# if it doesn't match that commit.
print("Downloading gyp...")
run([
"git", "clone", "--quiet", "--depth=1",
"https://chromium.googlesource.com/external/gyp.git",
LIB_UV_DIR + "/build/gyp"
])
# We don't need all of libuv and gyp's various support files.
print("Deleting unneeded files...")
remove_dir("deps/libuv/build/gyp/buildbot")
remove_dir("deps/libuv/build/gyp/infra")
remove_dir("deps/libuv/build/gyp/samples")
remove_dir("deps/libuv/build/gyp/test")
remove_dir("deps/libuv/build/gyp/tools")
remove_dir("deps/libuv/docs")
remove_dir("deps/libuv/img")
remove_dir("deps/libuv/samples")
remove_dir("deps/libuv/test")
# We are going to commit libuv and GYP in the main Wren repo, so we don't
# want them to be their own repos.
remove_dir("deps/libuv/.git")
remove_dir("deps/libuv/build/gyp/.git")
# Libuv's .gitignore ignores GYP, but we want to commit it.
replace_in_file("deps/libuv/.gitignore", "/build/gyp",
"# /build/gyp (We do want to commit GYP in Wren's repo)")
def testExecuteEquiv(self):
GEN_FILE = TEST + "testlib/testlib_patched-equiv.dll"
changed_bytes = equiv.do_equiv_instrs(self.f.instrs, gen_patched=True)
# check the number of changed bytes
self.assertEqual(len(changed_bytes), 13)
# check the numner of generated files
self.assertEqual(len(glob.glob(GEN_FILE)), 1)
output = util.run("%s %s" % (self.PROG, GEN_FILE), 10)
# check the output of the generated files
self.assertTrue(self.EXPECT in output,
"%s: %s %s" % (GEN_FILE, self.EXPECT, output))
# remove the generated files
map(os.remove, glob.glob(GEN_FILE))
def create_kubectl_secret():
"""
Ensure that the kubectl secret needed for using kubectl instead of the pod to
use this cluster/namespace exists.
"""
if SECRET_NAME not in util.get_secrets():
with tempfile.TemporaryDirectory() as tmp:
target = join(tmp, 'config')
config = json.loads(util.run(['kubectl', 'config', 'view', '--raw', '-o=json'], get_output=True, verbose=False))
prefix = util.get_cluster_prefix()
# Include only secret info that is relevant to this cluster (a mild security measure -- we can't restrict namespace btw).
for k in ['contexts', 'clusters', 'users']:
config[k] = [x for x in config[k] if x['name'].endswith(prefix)]
open(join(tmp, 'config'), 'w').write(yaml.dump(config))
util.create_secret(SECRET_NAME, tmp)
def update_stats(self): # todo: split into separate stats/svd updates
"""Updates all covariance/SVD info of correctable factors."""
s = self
ops = []
# update covariances
# s.grad.update() # TODO: not needed
# s.grad2.update()
for var in s:
ops.append(s[var].A.cov_update_op)
ops.append(s[var].B2.cov_update_op)
with u.timeit("covariances"):
u.run(ops)
# update SVDs
corrected_vars = list(s)
with u.timeit("svd"):
with s.write_lock():
for var in s:
if not dont_update_first_layer or s[var].A.svd.update_counter==0:
s[var].A.svd.update()
s[var].B2.svd.update()
def test(cmd):
util.info("")
util.info("- Starting " + cmd)
util.info("")
util.run(cmd)
start = time.time()
clientlist = []
for i in range(0, NUM_CLIENT):
client = testit("Client-" + str(i))
clientlist.append(client)
client.start()
for client in clientlist:
client.join()
end = time.time()
util.info("Elapsed time (in seconds): " + str(end - start))
if util.is_server_alive(cmd) == -1:
util.error("Ouch! Server is dead!"
" Your bounded buffered may not be well protected")
def run_on_kubernetes(args):
create_gcloud_secret()
context = util.get_cluster_prefix()
namespace = util.get_current_namespace()
if len(args.number) == 0:
# Figure out the nodes based on the names of persistent disks, or just node 0 if none.
args.number = range(
max(1, len(get_persistent_disks(context, namespace))))
if 'storage-projects' not in util.get_services():
util.run(['kubectl', 'create', '-f', 'conf/service.yaml'])
args.local = False # so tag is for gcloud
tag = util.get_tag(args, NAME, build)
if not args.tag:
tag = tag[:tag.rfind(
'-')] # get rid of the final -[service] part of the tag.
t = open(join('conf', '{name}.template.yaml'.format(name=NAME))).read()
ensure_ssh()
for number in args.number:
deployment_name = "{name}{number}".format(name=NAME, number=number)
ensure_persistent_disk_exists(context, namespace, number, args.size,
args.type)
with tempfile.NamedTemporaryFile(suffix='.yaml', mode='w') as tmp:
tmp.write(
t.format(image=tag,
number=number,
gcloud_bucket=gcloud_bucket(namespace=namespace),
pd_name=pd_name(context=context,
namespace=namespace,
number=number),
health_delay=args.health_delay,
pull_policy=util.pull_policy(args)))
tmp.flush()
util.update_deployment(tmp.name)
def collectData(experiment_dir, config_map):
data_dir = getDataDir(experiment_dir)
util.maybeMakeDir(data_dir)
gain_dir = getLabelDir(config_map['label'])
feature_list = getFeatureListPath(experiment_dir)
data_file = getDataPath(data_dir)
label_file = getLabelPath(data_dir)
rlabel_file = getRlabelPath(data_dir)
meta_file = getMetaPath(data_dir)
weight_file = getWeightPath(data_dir)
cmd = (
'%s/collect_data.py --gain_dir=%s --max_neg=%f --min_pos=%f '
'--feature_base_dir=%s --feature_list=%s --feature_stats=%s '
'--min_date=%s --max_date=%s --window=%d --min_feature_perc=%f '
'--data_file=%s --label_file=%s --rlabel_file=%s --meta_file=%s '
'--weight_power=%f --weight_file=%s' %
(CODE_DIR, gain_dir, config_map['max_neg'], config_map['min_pos'],
FEATURE_DIR, feature_list, FEATURE_STATS_FILE, config_map['min_date'],
config_map['max_date'], config_map['feature_window'],
config_map['min_feature_perc'], data_file, label_file, rlabel_file,
meta_file, config_map['weight_power'], weight_file))
util.run(cmd)
def eslint():
script = os.path.join(third_party_path, "node_modules", "eslint", "bin",
"eslint")
# Find all *directories* in the main repo that contain .ts/.js files.
source_files = get_sources(root_path, [
"*.js",
"*.ts",
":!:cli/tests/swc_syntax_error.ts",
":!:std/**/testdata/*",
":!:std/**/node_modules/*",
":!:cli/compilers/wasm_wrap.js",
":!:cli/tests/error_syntax.js",
":!:cli/tests/lint/**",
])
if source_files:
print_command("eslint", source_files)
# Set NODE_PATH so we don't have to maintain a symlink in root_path.
env = os.environ.copy()
env["NODE_PATH"] = os.path.join(root_path, "third_party",
"node_modules")
run(["node", script, "--max-warnings=0", "--"] + source_files,
shell=False,
env=env,
quiet=True)
def test_rio_app_endpoint(nspc):
image = "ibuildthecloud/demo:v1"
image2 = "ibuildthecloud/demo:v3"
srv = create_service(nspc, image)
fullName = (f"{nspc}/{srv}")
print(fullName)
stage_service(image2, fullName, "v3")
appEndpoint = get_app_info(fullName, "status.endpoints[0]")
print(f"{appEndpoint}")
results = util.run(f"curl -s {appEndpoint}")
print(f"{results}")
assert results == 'Hello World'
def up(self):
self.vagrant('up')
# Wait for VirtualBox tools to come online
if not os.environ.get('VAGRANT_DEFAULT_PROVIDER') or os.environ.get(
'VAGRANT_DEFAULT_PROVIDER') == 'virtualbox':
# Wait for VirtualBox to restart
time.sleep(30)
while run('vagrant ssh --command "pgrep -f VBoxService"',
returncode=True,
echo=False):
sys.stdout.write(".")
print "."
self.vagrant('reload')
def main():
args = parse_args()
log.debug("Dockerfile: {}, context: {}, docker build args: '{}'".format(
args.dockerfile, args.context, args.docker_build_args))
use_container_registry = args.container_registry is not None
if not use_container_registry:
log.info("No container registry will be used")
tag = generate_tag(args.dockerfile, args.context, args.docker_build_args)
full_image_name = \
"{}.azurecr.io/{}:{}".format(args.container_registry, args.repository, tag) \
if use_container_registry else \
"{}:{}".format(args.repository, tag)
log.info("Image: {}".format(full_image_name))
if use_container_registry and container_registry_has_image(
full_image_name, args.docker_path):
log.info("Pulling image...")
run(args.docker_path, "pull", full_image_name)
else:
log.info("Building image...")
run(args.docker_path, "build", "--pull",
*shlex.split(args.docker_build_args),
*shlex.split(args.docker_build_args_not_affecting_image_content),
"--tag", full_image_name, "--file", args.dockerfile, args.context)
if use_container_registry:
# avoid pushing if an identically tagged image has been pushed since the last check
# there is still a race condition, but this reduces the chance of a redundant push
if not container_registry_has_image(full_image_name,
args.docker_path):
log.info("Pushing image...")
run(args.docker_path, "push", full_image_name)
else:
log.info("Image now found, skipping push")
# tag so we can refer to the image by repository name
run(args.docker_path, "tag", full_image_name, args.repository)
return 0
