def run_lstm(model_directory, X, Y, X_test, Y_test):
model = get_model.rnn_lstm(nclass=5, dense_layers=[64, 16, 8])
file_name = "mitbih_rnn_lstm"
file_path = os.path.join(model_directory, file_name + ".h5")
model = helpers.run(model, file_path, X, Y)
model.load_weights(file_path)
# Save the entire model as a SavedModel.
model.save(os.path.join(model_directory, file_name))
# Make predictions on test set
pred_test = model.predict(X_test)
# Evaluate predictions
helpers.test_nonbinary(Y_test, pred_test, verbose=True)
def run_gru_bidir(model_directory, X, Y, X_test, Y_test):
model = get_model.rnn_gru_bidir(nclass=1, dense_layers=[64, 16, 8], binary=True)
file_name = "ptbdb_rnn_gru_bidir"
file_path = os.path.join(model_directory, file_name + ".h5")
model = helpers.run(model, file_path, X, Y)
model.load_weights(file_path)
# Save the entire model as a SavedModel.
model.save(os.path.join(model_directory, file_name))
# Make predictions on test set
pred_test = model.predict(X_test)
# Evaluate predictions
helpers.test_binary(Y_test, pred_test)
def run_transfer_learning(base_model, model_directory, X, Y, X_test, Y_test):
model = get_model.transfer_learning(
nclass=1, base_model=base_model, loss=losses.binary_crossentropy
)
file_name = "ptbdb_rnn_gru_bidir_transfer"
file_path = file_name + ".h5"
model = helpers.run(model, file_path, X, Y)
model.load_weights(file_path)
# Save the entire model as a SavedModel.
model.save(os.path.join(model_directory, file_name))
# Make predictions on test set
pred_test = model.predict(X_test)
# Evaluate predictions
helpers.test_binary(Y_test, pred_test)
def fpga(self, **kwargs):
# download installation packages from Chameleon object storage
region = kwargs["region"]
ip = kwargs["ip"]
rc = kwargs["rc"]
ssh_key_file = kwargs["ssh_key_file"]
ssh_args = kwargs["ssh_args"]
tmp_fpga_dir = '/tmp/fpga'
session = helpers.get_auth_session_from_rc(rc)
if region == 'CHI@TACC':
objects = ['aocl-rte-16.0.0-1.x86_64.rpm', 'nalla_pcie_16.0.2.tgz']
elif region == 'CHI@UC':
objects = [
'aocl-pro-rte-17.1.0-240.x86_64.rpm',
'QuartusProProgrammerSetup-17.1.0.240-linux.run',
'de5a_net_e1.tar.gz'
]
else:
raise RuntimeError('Region incorrect!')
helpers.run('mkdir -p {}'.format(tmp_fpga_dir))
helpers.remote_run(ip=ip,
command='sudo mkdir -p {}'.format(tmp_fpga_dir))
swift_connection = helpers.connect_to_swift_with_admin(session, region)
for obj in objects:
print('downloading {}'.format(obj))
resp_headers, obj_contents = swift_connection.get_object(
'FPGA', obj)
with open('{}/{}'.format(tmp_fpga_dir, obj), 'wb') as local:
local.write(obj_contents)
if ssh_key_file:
proc = helpers.run('scp -i {} {} {}/{} cc@{}:'.format(
ssh_key_file, ' '.join(ssh_args), tmp_fpga_dir, obj, ip))
else:
proc = helpers.run('scp {} {}/{} cc@{}:'.format(
' '.join(ssh_args), tmp_fpga_dir, obj, ip))
print(' - stdout:\n{}\n - stderr:\n{}\n--------'.format(
proc.stdout, proc.stderr))
if proc.returncode != 0:
raise RuntimeError('scp to remote failed!')
else:
helpers.remote_run(ip=ip,
command='sudo mv ~/{} {}/'.format(
obj, tmp_fpga_dir))
helpers.remote_run(
ip=ip, command='sudo chmod -R 755 {}'.format(tmp_fpga_dir))
# clean up
helpers.run('rm -rf {}'.format(tmp_fpga_dir))
helpers.remote_run(ip=ip, command='sudo ls -la /tmp/fpga')
def do_build(ip, rc, repodir, commit, metadata, variant):
chi.server.wait_for_tcp(ip, port=22)
print('remote contactable!')
ssh_key_file = os.environ.get('SSH_KEY_FILE', None)
region = os.environ['OS_REGION_NAME']
ssh_args = [
'-o UserKnownHostsFile=/dev/null', '-o StrictHostKeyChecking=no'
]
extra_steps = getattr(ExtraSteps(), variant, None)
if extra_steps:
kwargs = {
"region": region,
"ip": ip,
"rc": rc,
"ssh_key_file": ssh_key_file,
"ssh_args": ssh_args,
}
extra_steps(**kwargs)
# init remote repo
helpers.remote_run(ip=ip, command='rm -rf ~/build.git')
out = helpers.remote_run(ip=ip, command='git init --bare build.git')
print(out)
print('- pushing repo to remote')
# GIT_SSH_COMMAND setup (requires Git 2.3.0+, CentOS repos have ~1.8)
git_ssh_args = ssh_args
if ssh_key_file:
print(' - using ssh keyfile at: {}'.format(ssh_key_file))
git_ssh_args.append('-i {}'.format(ssh_key_file))
proc = helpers.run(
'git push --all ssh://cc@{ip}/~/build.git'.format(ip=ip),
cwd=repodir,
env={
'GIT_SSH_COMMAND': 'ssh {}'.format(' '.join(git_ssh_args)),
})
print(' - stdout:\n{}\n - stderr:\n{}\n--------'.format(
proc.stdout, proc.stderr))
if proc.returncode != 0:
raise RuntimeError('repo push to remote failed')
# checkout local rev on remote
helpers.remote_run(ip=ip, command='rm -rf ~/build')
helpers.remote_run(ip=ip, command='git clone ~/build.git ~/build')
helpers.remote_run(
ip=ip,
command=
'cd /home/cc/build && git -c advice.detachedHead=false checkout {head}'
.format(head=commit))
helpers.remote_run(ip=ip, command='ls -a')
out = io.StringIO()
# install build reqs
helpers.remote_run(ip=ip,
command='sudo bash ~/build/install-reqs.sh',
pty=True,
out_stream=out)
# there's a lot of output and it can do strange things if we don't
# use a buffer or file or whatever
cmd = ('export DIB_CC_PROVENANCE={provenance}; '
'cd /home/cc/build/ && '
'python3 create-image.py --release {release} '
'--variant {variant} --region {region}').format(
provenance=base64.b64encode(
json.dumps(metadata).encode('ascii')).decode('ascii'),
release=metadata["build-release"],
variant=variant,
region=region,
)
# DO THE THING
helpers.remote_run(ip=ip, command=cmd, pty=True, out_stream=out)
with open('build.log', 'w') as f:
print(f.write(out.getvalue()))
out.seek(0)
ibi = 'Image built in '.format(ip=ip)
for line in out:
if not line.startswith(ibi):
continue
output_file = line[len(ibi):].strip()
break
else:
raise RuntimeError("didn't find output file in logs.")
print(output_file)
checksum_result = helpers.remote_run(ip=ip,
command=f"md5sum {output_file}")
checksum = checksum_result.stdout.split()[0].strip()
return {
'image_loc': output_file,
'checksum': checksum,
}