def prepare(id_f, master_dns, credential_f, key_id, secret_key): try: for f in [credential_f, 'ec2/ec2.bashrc']: # TODO: ec2.bashrc is not sourced scpScript = _CMD['scp'].format(id=id_f, f=f, dns=master_dns) stdout, stderr = runScript(scpScript, output_opt='display', input_opt='display') app_root = _APP_INFO['repo_url'].split('/')[-1].split('.git')[0] combineCmd = [] combineCmd += [_CMD['source_rc'].format(rc='ec2.bashrc')] combineCmd += [_CMD['key_id_export'].format(key_id=key_id, secret_key=secret_key)] # TODO: hdfs set up aws credential for cp from S3 combineCmd += [_CMD['hdfs_conf']\ .format(hdfs=_AWS_DIR_INFO['hdfs'], key_id=key_id, secret=secret_key)] combineCmd += [_CMD['hdfs_cp'].format(f=_AWS_DIR_INFO['data'])] combineCmd += [_CMD['dir_create'].format(dir='/tmp/spark-events/')] combineCmd += [_CMD['dir_clone'].format(dir=app_root, dir_git=_APP_INFO['repo_url'])] combineCmd += [_CMD['py3_check']] combineCmd = '\n'.join(combineCmd) remoteScript = _CMD['pipe_remote'] printf(remoteScript) stdout, stderr = runScript(remoteScript, output_opt='display', input_opt='pipe', input_pipe=[combineCmd, '.quit']) except ScriptException as se: printf(se, type='ERROR') exit()
def print_to_file(f_name, msg, *reflex, type='INFO', separator='default', log_dir=_LOG_DIR_DEFAULT, mode='a', perm='default'): """ write message into file * create dir / file if not exist * read-only policy argument: f_name name of the log file msg, reflex, type, separator see logf.stringf log_dir the directory to be appended in front of f_name mode 'w' for truncate; 'a' for append perm permission of f_name after write to it Policy of perm='default': if f_name exists before, then don't change permission if f_name doesn't exist, then set permission to '0444' Policy of perm='xxxx' (string consisting of 4 digits): set permission to 'xxxx' no matter f_name exists or not """ msg = str(msg) f_full = '{}/{}'.format(log_dir, f_name) if perm == 'default': perm = (os.path.exists(f_full)) and os.stat(f_full).st_mode or '0444' # make sure directory exists mkdir_r(os.path.dirname(f_full)) if os.path.exists(f_full): set_f_perm(f_full, '0222') f = open(f_full, mode) print(stringf(msg, *reflex, type=type, separator=separator), file=f) printf('write to file: {}', f_full, separator=None) f.close() set_f_perm(f_full, perm)
def _perm_to_int(perm): """ convert perm (which can be a str or int) to int (understandable by os module) e.g.: perm='0444' if for read-only policy However, I won't process the first char for now """ if type(perm) == type(0): return perm ERROR_PERM_FORMAT = 'format of perm is wrong!' try: assert len(perm) == 4 except AssertionError: printf(ERROR_PERM_FORMAT, type='ERROR') exit() p_pos = ['','USR', 'GRP', 'OTH'] # don't care, owner, group, others p_ret = 0 eval_str = 'stat.S_I{}{}' for n in range(1,4): p_int = int(perm[n]) try: assert p_int <= 7 and p_int >= 0 except AssertionError: printf(ERROR_PERM_FORMAT, type='ERROR') exit() if p_int >= 4: p_ret |= eval(eval_str.format('R', p_pos[n])) if p_int in [2,3,6,7]: p_ret |= eval(eval_str.format('W', p_pos[n])) if p_int%2 == 1: p_ret |= eval(eval_str.format('X', p_pos[n])) return p_ret
def setup_env(cred_f): try: stdout, stderr = runScript(_CMD['get_env'].format(cred_f=cred_f)) aws_access_key_id = stdout.decode('utf-8').split("\n")[0] aws_secret_access_key = stdout.decode('utf-8').split("\n")[1] os.environ['AWS_SECRET_ACCESS_KEY'] = aws_secret_access_key os.environ['AWS_ACCESS_KEY_ID'] = aws_access_key_id printf('environment var set: {}, {}', aws_access_key_id, aws_secret_access_key) return aws_access_key_id, aws_secret_access_key except ScriptException as se: printf(se, type='ERROR')
def setup_spark_ec2_flag(args): second_lvl_arg_dict = {k:conf.DEFAULT_EC2_ARGS[k] for k in mode_keys} second_lvl_arg_list = map(lambda i: '{} {}'.format(list(second_lvl_arg_dict.keys())[i], list(second_lvl_arg_dict.values())[i]), range(len(second_lvl_arg_dict))) # NOTE: don't use '+=', cuz if the same arg is overwritten in cmd line, # you need to put the default value prior to the overwritten one spark_ec2_flag = ' {} {}'.format(' '.join(second_lvl_arg_list), args.spark_ec2_flag) spark_ec2_flag += ' -i {} -k {}' \ .format(args.identity_file, args.identity_file.split('.pem')[0].split('/')[-1]) printf('args to spark-ec2 script: \n\t{}',spark_ec2_flag) return spark_ec2_flag
def submit_application(master_dns): try: submit_main = '/root/{}/{}'.format(_APP_INFO['name'], _APP_INFO['submit_main']) log_dir = _AWS_DIR_INFO['log'].format(train_name=_APP_INFO['name']) spark_dir = _AWS_DIR_INFO['spark'] shot = '/root/{name}/ec2/fire_and_leave {dns} {main} {args}'\ .format(name=_APP_INFO['name'], dns=master_dns, main=_APP_INFO['submit_main'], args='') remoteScript = _CMD['pipe_remote'] stdout, stderr = runScript(remoteScript, output_opt='display', input_opt='pipe', input_pipe=[shot, '.quit']) except ScriptException as se: printf(se, type='ERROR') exit()
def get_master_DNS(cluster_name): """ get the ID and public-DNS of master node """ master_dns = '' try: stdout, stderr = runScript('aws ec2 describe-instances', output_opt='pipe') printf('instance info got, target: {}-master', cluster_name) master_id_regex = '{}-master-{}'.format(cluster_name, '\S*') master_id = re.search(master_id_regex, stdout.decode('utf-8')) if not master_id: printf('failed to get master-id:\n check your cluster name / region ...', type='ERROR') exit() master_id = master_id.group().split('master-')[-1][:-1] master_id = master_id.split('"')[0] stdout, stderr = runScript('aws ec2 describe-instances --instance-ids {}'.format(master_id), output_opt='pipe') master_dns_regex = '"{}": "{}",'.format('PublicDnsName', '\S*') master_dns = re.search(master_dns_regex, stdout.decode('utf-8'))\ .group().split("\"")[-2] printf("Get {}-master public DNS:\n {}", cluster_name, master_dns) return master_dns except ScriptException as se: printf(se, type='ERROR') exit()
def mkdir_r(dir_r): """ recursively mkdir if not exist dir_r of 'a/b/c' or 'a/b/c/' will both create directory a, b and c WARNING: no explicit error checking: e.g.: if there is a file (not dir) called 'a/b', then this function will fail """ dir_parent = os.path.dirname(dir_r) dir_parent = (dir_parent != '') and dir_parent or '.' if not os.path.exists(dir_parent): mkdir_r(dir_parent) if not os.path.exists(dir_r): os.mkdir(dir_r) printf("created dir: {}", dir_r, separator=None)
def conf_AWS_CLI(credential_f, region): """ setup default values for aws command line interface. """ try: scriptGetKey = _CMD['key_id_parse'].format(credential_f=credential_f) stdout, stderr = runScript(scriptGetKey, output_opt='pipe') key_id, secret_key= stdout.decode('utf-8').split('\n')[:-1] os.environ['AWS_SECRET_ACCESS_KEY'] = secret_key os.environ['AWS_ACCESS_KEY_ID'] = key_id stdout, stderr = runScript('aws configure', output_opt='display', input_opt='pipe', input_pipe=[key_id, secret_key, region, _OUTPUT_FORMAT]) print() printf('AWS-CLI conf done') return key_id, secret_key except ScriptException as se: printf(se, type='ERROR') exit()
def log_streaming_data_SPN(N, p, input_data, output_data, resources): row_len = 0 raw_sequence = {'0: INPUT': input_data, '1: OUTPUT': output_data} strf = N // 10 and '{:2s}' or '{:1s}' def tostr_tuple(e): return (strf + ',' + strf).format(str(e[0]), str(e[1])) v_tostr_tuple = np.vectorize(tostr_tuple) for k in sorted(raw_sequence.keys()): printf('{}', k) for tup in raw_sequence[k]: row_len += len(tup) str_tup = '' for start_i in range(0, len(tup), N): if start_i > 0: str_tup += '\n' * 2 str_tup += ' '.join(v_tostr_tuple(tup[start_i:start_i + N])) print(str_tup + ' ', end='') if row_len >= N: row_len = 0 print('\n') for k in resources.keys(): printf('{}: {}', k, resources[k])
def login(wrap_script, wrap_args, name, input_opt): """ input_opt should be either 'pipe' or 'cmd'. """ try: input_pipe = [] if input_opt == 'pipe': printf('enter cmds you want to send to ec2 cluster. type \'.quit\' to finish up.') while True: new_ip = input('>> ') print(new_ip) if new_ip != '.quit': input_pipe += [new_ip] else: break stdout, stderr = runScript(_CMD['login'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name), output_opt='display', input_opt=input_opt, input_pipe=input_pipe) printf('finish interaction with master node.') except ScriptException as se: printf(se, type='ERROR')
def save(self): if self.img is None: printf('img has not been drawn yet') exit() else: self.img.save(self.op_path, 'PNG')
def save(self): if self.img is None: printf('img has not been drawn yet') exit() else: self.img.save(self.output_file, 'PNG')
def destroy(wrap_script, wrap_args, name): try: stdout, stderr = runScript(_CMD['destroy'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name), output_opt='display', input_opt='cmd') printf('cluster successfully destroyed.') except ScriptException as se: printf(se, type='ERROR')
def launch(wrap_script, wrap_args, name): try: stdout, stderr = runScript(_CMD['launch'].format(wrap_script=wrap_script, wrap_args=wrap_args, name=name), output_opt='display') printf('cluster successfully launched.') except ScriptException as se: printf(se, type='ERROR')
printf('cluster successfully destroyed.') except ScriptException as se: printf(se, type='ERROR') if __name__=='__main__': args = conf.parse_args() wrap_script = '{}/ec2/spark-ec2'.format(args.spark_dir) if args.spark_ec2_help: try: printf('\nhelp msg from spark-ec2 script:\n') stdout, stderr = runScript('{} -h'.format(wrap_script), output_opt='display') except ScriptException as se: printf(se, type='ERROR') exit() if args.launch: mode_keys=_DEF_LAUNCH_ARGS elif args.login: mode_keys=_DEF_LOGIN_ARGS elif args.destroy: mode_keys=_DEF_DESTROY_ARGS else: printf('unknown launch mode', type='ERROR') exit()
""" This script is to manually test if a trained model feels good. """ from net_structure import Net_structure import argparse import numpy as np import util.convert_ndarr_img as img_cvt from logf.printf import printf def parse_args(): parser = argparse.ArgumentParser('evaluate the quality of trained model') parser.add_argument('checkpoint', type=str, help='path to the checkpoint file of trained net') parser.add_argument('test_img', type=str, help='path to the image to be tested') return parser.parse_args() if __name__ == '__main__': args = parse_args() net = Net_structure(None) net.import_(args.checkpoint) ip_arr = img_cvt.img_to_array(args.test_img) op = net.net_act_forward(ip_arr) printf('predicted category: {}', op.argmax(axis=1))