def prepare(id_f, master_dns, credential_f, key_id, secret_key):
try:
for f in [credential_f, 'ec2/ec2.bashrc']:
# TODO: ec2.bashrc is not sourced
scpScript = _CMD['scp'].format(id=id_f, f=f, dns=master_dns)
stdout, stderr = runScript(scpScript, output_opt='display', input_opt='display')
app_root = _APP_INFO['repo_url'].split('/')[-1].split('.git')[0]
combineCmd = []
combineCmd += [_CMD['source_rc'].format(rc='ec2.bashrc')]
combineCmd += [_CMD['key_id_export'].format(key_id=key_id, secret_key=secret_key)]
# TODO: hdfs set up aws credential for cp from S3
combineCmd += [_CMD['hdfs_conf']\
.format(hdfs=_AWS_DIR_INFO['hdfs'], key_id=key_id, secret=secret_key)]
combineCmd += [_CMD['hdfs_cp'].format(f=_AWS_DIR_INFO['data'])]
combineCmd += [_CMD['dir_create'].format(dir='/tmp/spark-events/')]
combineCmd += [_CMD['dir_clone'].format(dir=app_root, dir_git=_APP_INFO['repo_url'])]
combineCmd += [_CMD['py3_check']]
combineCmd = '\n'.join(combineCmd)
remoteScript = _CMD['pipe_remote']
printf(remoteScript)
stdout, stderr = runScript(remoteScript, output_opt='display', input_opt='pipe', input_pipe=[combineCmd, '.quit'])
except ScriptException as se:
printf(se, type='ERROR')
exit()
def print_to_file(f_name, msg, *reflex, type='INFO', separator='default',
log_dir=_LOG_DIR_DEFAULT, mode='a', perm='default'):
"""
write message into file
* create dir / file if not exist
* read-only policy
argument:
f_name name of the log file
msg,
reflex,
type,
separator see logf.stringf
log_dir the directory to be appended in front of f_name
mode 'w' for truncate; 'a' for append
perm permission of f_name after write to it
Policy of perm='default':
if f_name exists before, then don't change permission
if f_name doesn't exist, then set permission to '0444'
Policy of perm='xxxx' (string consisting of 4 digits):
set permission to 'xxxx' no matter f_name exists or not
"""
msg = str(msg)
f_full = '{}/{}'.format(log_dir, f_name)
if perm == 'default':
perm = (os.path.exists(f_full)) and os.stat(f_full).st_mode or '0444'
# make sure directory exists
mkdir_r(os.path.dirname(f_full))
if os.path.exists(f_full):
set_f_perm(f_full, '0222')
f = open(f_full, mode)
print(stringf(msg, *reflex, type=type, separator=separator), file=f)
printf('write to file: {}', f_full, separator=None)
f.close()
set_f_perm(f_full, perm)
def _perm_to_int(perm):
"""
convert perm (which can be a str or int) to int (understandable by os module)
e.g.: perm='0444' if for read-only policy
However, I won't process the first char for now
"""
if type(perm) == type(0):
return perm
ERROR_PERM_FORMAT = 'format of perm is wrong!'
try:
assert len(perm) == 4
except AssertionError:
printf(ERROR_PERM_FORMAT, type='ERROR')
exit()
p_pos = ['','USR', 'GRP', 'OTH'] # don't care, owner, group, others
p_ret = 0
eval_str = 'stat.S_I{}{}'
for n in range(1,4):
p_int = int(perm[n])
try:
assert p_int <= 7 and p_int >= 0
except AssertionError:
printf(ERROR_PERM_FORMAT, type='ERROR')
exit()
if p_int >= 4:
p_ret |= eval(eval_str.format('R', p_pos[n]))
if p_int in [2,3,6,7]:
p_ret |= eval(eval_str.format('W', p_pos[n]))
if p_int%2 == 1:
p_ret |= eval(eval_str.format('X', p_pos[n]))
return p_ret
def setup_env(cred_f):
try:
stdout, stderr = runScript(_CMD['get_env'].format(cred_f=cred_f))
aws_access_key_id = stdout.decode('utf-8').split("\n")[0]
aws_secret_access_key = stdout.decode('utf-8').split("\n")[1]
os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key
os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id
printf('environment var set: {}, {}', aws_access_key_id, aws_secret_access_key)
return aws_access_key_id, aws_secret_access_key
except ScriptException as se:
printf(se, type='ERROR')
def setup_spark_ec2_flag(args):
second_lvl_arg_dict = {k:conf.DEFAULT_EC2_ARGS[k] for k in mode_keys}
second_lvl_arg_list = map(lambda i: '{} {}'.format(list(second_lvl_arg_dict.keys())[i],
list(second_lvl_arg_dict.values())[i]), range(len(second_lvl_arg_dict)))
# NOTE: don't use '+=', cuz if the same arg is overwritten in cmd line,
# you need to put the default value prior to the overwritten one
spark_ec2_flag = ' {} {}'.format(' '.join(second_lvl_arg_list), args.spark_ec2_flag)
spark_ec2_flag += ' -i {} -k {}' \
.format(args.identity_file, args.identity_file.split('.pem')[0].split('/')[-1])
printf('args to spark-ec2 script: \n\t{}',spark_ec2_flag)
return spark_ec2_flag
def submit_application(master_dns):
try:
submit_main = '/root/{}/{}'.format(_APP_INFO['name'], _APP_INFO['submit_main'])
log_dir = _AWS_DIR_INFO['log'].format(train_name=_APP_INFO['name'])
spark_dir = _AWS_DIR_INFO['spark']
shot = '/root/{name}/ec2/fire_and_leave {dns} {main} {args}'\
.format(name=_APP_INFO['name'], dns=master_dns, main=_APP_INFO['submit_main'], args='')
remoteScript = _CMD['pipe_remote']
stdout, stderr = runScript(remoteScript, output_opt='display', input_opt='pipe', input_pipe=[shot, '.quit'])
except ScriptException as se:
printf(se, type='ERROR')
exit()
def get_master_DNS(cluster_name):
"""
get the ID and public-DNS of master node
"""
master_dns = ''
try:
stdout, stderr = runScript('aws ec2 describe-instances', output_opt='pipe')
printf('instance info got, target: {}-master', cluster_name)
master_id_regex = '{}-master-{}'.format(cluster_name, '\S*')
master_id = re.search(master_id_regex, stdout.decode('utf-8'))
if not master_id:
printf('failed to get master-id:\n check your cluster name / region ...', type='ERROR')
exit()
master_id = master_id.group().split('master-')[-1][:-1]
master_id = master_id.split('"')[0]
stdout, stderr = runScript('aws ec2 describe-instances --instance-ids {}'.format(master_id), output_opt='pipe')
master_dns_regex = '"{}": "{}",'.format('PublicDnsName', '\S*')
master_dns = re.search(master_dns_regex, stdout.decode('utf-8'))\
.group().split("\"")[-2]
printf("Get {}-master public DNS:\n {}", cluster_name, master_dns)
return master_dns
except ScriptException as se:
printf(se, type='ERROR')
exit()
def mkdir_r(dir_r):
"""
recursively mkdir if not exist
dir_r of 'a/b/c' or 'a/b/c/' will both create directory a, b and c
WARNING:
no explicit error checking:
e.g.: if there is a file (not dir) called 'a/b', then this function will fail
"""
dir_parent = os.path.dirname(dir_r)
dir_parent = (dir_parent != '') and dir_parent or '.'
if not os.path.exists(dir_parent):
mkdir_r(dir_parent)
if not os.path.exists(dir_r):
os.mkdir(dir_r)
printf("created dir: {}", dir_r, separator=None)
def conf_AWS_CLI(credential_f, region):
"""
setup default values for aws command line interface.
"""
try:
scriptGetKey = _CMD['key_id_parse'].format(credential_f=credential_f)
stdout, stderr = runScript(scriptGetKey, output_opt='pipe')
key_id, secret_key= stdout.decode('utf-8').split('\n')[:-1]
os.environ['AWS_SECRET_ACCESS_KEY'] = secret_key
os.environ['AWS_ACCESS_KEY_ID'] = key_id
stdout, stderr = runScript('aws configure', output_opt='display',
input_opt='pipe', input_pipe=[key_id, secret_key, region, _OUTPUT_FORMAT])
print()
printf('AWS-CLI conf done')
return key_id, secret_key
except ScriptException as se:
printf(se, type='ERROR')
exit()
def log_streaming_data_SPN(N, p, input_data, output_data, resources):
row_len = 0
raw_sequence = {'0: INPUT': input_data, '1: OUTPUT': output_data}
strf = N // 10 and '{:2s}' or '{:1s}'
def tostr_tuple(e):
return (strf + ',' + strf).format(str(e[0]), str(e[1]))
v_tostr_tuple = np.vectorize(tostr_tuple)
for k in sorted(raw_sequence.keys()):
printf('{}', k)
for tup in raw_sequence[k]:
row_len += len(tup)
str_tup = ''
for start_i in range(0, len(tup), N):
if start_i > 0:
str_tup += '\n' * 2
str_tup += ' '.join(v_tostr_tuple(tup[start_i:start_i + N]))
print(str_tup + ' ', end='')
if row_len >= N:
row_len = 0
print('\n')
for k in resources.keys():
printf('{}: {}', k, resources[k])
def login(wrap_script, wrap_args, name, input_opt):
"""
input_opt should be either 'pipe' or 'cmd'.
"""
try:
input_pipe = []
if input_opt == 'pipe':
printf('enter cmds you want to send to ec2 cluster. type \'.quit\' to finish up.')
while True:
new_ip = input('>> ')
print(new_ip)
if new_ip != '.quit':
input_pipe += [new_ip]
else:
break
stdout, stderr = runScript(_CMD['login'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name),
output_opt='display', input_opt=input_opt, input_pipe=input_pipe)
printf('finish interaction with master node.')
except ScriptException as se:
printf(se, type='ERROR')
def save(self):
if self.img is None:
printf('img has not been drawn yet')
exit()
else:
self.img.save(self.op_path, 'PNG')
def save(self):
if self.img is None:
printf('img has not been drawn yet')
exit()
else:
self.img.save(self.output_file, 'PNG')
def destroy(wrap_script, wrap_args, name):
try:
stdout, stderr = runScript(_CMD['destroy'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name), output_opt='display', input_opt='cmd')
printf('cluster successfully destroyed.')
except ScriptException as se:
printf(se, type='ERROR')
def launch(wrap_script, wrap_args, name):
try:
stdout, stderr = runScript(_CMD['launch'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name), output_opt='display')
printf('cluster successfully launched.')
except ScriptException as se:
printf(se, type='ERROR')
printf('cluster successfully destroyed.')
except ScriptException as se:
printf(se, type='ERROR')
if __name__=='__main__':
args = conf.parse_args()
wrap_script = '{}/ec2/spark-ec2'.format(args.spark_dir)
if args.spark_ec2_help:
try:
printf('\nhelp msg from spark-ec2 script:\n')
stdout, stderr = runScript('{} -h'.format(wrap_script), output_opt='display')
except ScriptException as se:
printf(se, type='ERROR')
exit()
if args.launch:
mode_keys=_DEF_LAUNCH_ARGS
elif args.login:
mode_keys=_DEF_LOGIN_ARGS
elif args.destroy:
mode_keys=_DEF_DESTROY_ARGS
else:
printf('unknown launch mode', type='ERROR')
exit()
def save(self):
if self.img is None:
printf('img has not been drawn yet')
exit()
else:
self.img.save(self.output_file, 'PNG')
"""
This script is to manually test if a trained model feels good.
"""
from net_structure import Net_structure
import argparse
import numpy as np
import util.convert_ndarr_img as img_cvt
from logf.printf import printf
def parse_args():
parser = argparse.ArgumentParser('evaluate the quality of trained model')
parser.add_argument('checkpoint', type=str, help='path to the checkpoint file of trained net')
parser.add_argument('test_img', type=str, help='path to the image to be tested')
return parser.parse_args()
if __name__ == '__main__':
args = parse_args()
net = Net_structure(None)
net.import_(args.checkpoint)
ip_arr = img_cvt.img_to_array(args.test_img)
op = net.net_act_forward(ip_arr)
printf('predicted category: {}', op.argmax(axis=1))
