def link_dependencies(proj_base_path, dep_path, verbosity="VERB_HIGH"): # import ipdb as pdb; pdb.set_trace() proj_rtl_dir = proj_base_path + "/rtl/" command = "cd " + proj_rtl_dir util.run_command(command, verbosity=verbosity) for ip_path in dep_path: ip_path = ip_path.replace("\n", "") util.check_for_file(ip_path, exit=False) command = "ln -s " + ip_path + " " + proj_rtl_dir util.run_command(command, verbosity=verbosity)
def publish(streams_path, publish_script, dry_run=False, verbose=False): juju_dist_path = os.path.join(streams_path, 'juju-dist') command = [publish_script, 'weekly', juju_dist_path, 'cpc'] for attempt in range(3): try: run_command(command, dry_run=dry_run, verbose=verbose) break except subprocess.CalledProcessError: # Raise an error when the third attempt fails; the cloud is ill. if attempt == 2: raise
def create_f_file(proj_base_path, deps, verbosity="VERB_HIGH"): # import ipdb as pdb; pdb.set_trace() path_to_ffile = "{0}/scripts/files.f".format(proj_dir) command = "touch {0}".format(path_to_ffile) util.run_command(command, verbosity=verbosity) ips = [] with open(path_to_ffile, "w") as f: for dep in deps: ip = dep.split("/")[-1].split("\n")[0] f.write("../rtl/" + ip + "\n") f.close()
def main(argv=None): args = parse_args(argv) juju_home = args.juju_home win_host = args.win_host win_path = ( 'C:\\\\Users\\\\Administrator\\\\juju-ci-tools\\\\download_juju.py') osx_path = '$HOME/juju-ci-tools/download_juju.py' osx_host = args.osx_host rev = get_revisions(os.environ['HOME']) with NamedTemporaryFile() as temp_file: for path, host in [[win_path, win_host], [osx_path, osx_host]]: create_workspace_yaml(juju_home, path, temp_file, rev) run_command(['workspace-run', temp_file.name, host])
def generate(model, batch_size, sub_model): if model == "LSTM": command = "fairseq-generate data/fairseq_binaries \ --path checkpoints/lstm/checkpoint_{}.pt \ --batch-size {} --beam 3".format(sub_model, batch_size) elif model == "CNN": command = "fairseq-generate data/fairseq_binaries \ --path checkpoints/conv/checkpoint_{}.pt \ --batch-size {} --beam 3".format(sub_model, batch_size) else: raise NotImplementedError run_command(command)
def generate(model, batch_size): if model == "LSTM": command = "fairseq-generate data/fairseq_binaries \ --path checkpoints/lstm/checkpoint_last.pt \ --batch-size 64 --beam 3" elif model == "CNN": command = "fairseq-generate data/fairseq_binaries \ --path checkpoints/conv/checkpoint_last.pt \ --batch-size 64 --beam 3" else: raise NotImplementedError run_command(command)
def main(argv=None): args = parse_args(argv) configure_logging(args.verbose) start_juju_path = None if args.upgrade else args.juju_bin client = client_from_config(args.env, start_juju_path, debug=args.debug, soft_deadline=args.deadline) with boot_context(args.temp_env_name, client, None, [], args.series, args.agent_url, args.agent_stream, args.logs, args.keep_env, upload_tools=args.upload_tools, region=args.region): assess_deployer( args, client, args.agent_timeout, args.workload_timeout) if args.bundle_verification_script: client_ser = pickle.dumps(client) logging.info('Calling bundle verification script {}'.format( args.bundle_verification_script)) run_command([args.bundle_verification_script, client_ser]) return 0
def optimize_model(expr_name, onnx_file, model_xml, weight_bin): run_opt = False if (model_xml == None): util.print_log("Could not find xml model", id_str="warning") run_opt = True if (weight_bin == None): util.print_log("Could not find binary weights", id_str="warning") run_opt = True if run_opt: util.print_banner( "Running OpenVino optimizer on {0}".format(onnx_file), color='green', verbosity="VERB_LOW") cmd = "python {0}/deployment_tools/model_optimizer/mo.py --input_model={1} --model_name {2}".format( openvino_inst_path, onnx_file, expr_name) util.run_command(cmd, verbosity="VERB_LOW") model_xml, weight_bin = expr_name + ".xml", expr_name + ".bin" # load model # import ipdb as pdb; pdb.set_trace() return model_xml, weight_bin
def publish_candidates(path, streams_path, juju_release_tools=None, dry_run=False, verbose=False): """Assemble and publish weekly streams from the candidates.""" timestamp = datetime.datetime.utcnow().strftime('%Y_%m_%dT%H_%M_%S') with temp_dir() as debs_path: for dir_name in get_artifact_dirs(path): artifacts_path = os.path.join(path, dir_name) branch_name = dir_name.split('-')[0] for deb_name in os.listdir(artifacts_path): deb_path = os.path.join(artifacts_path, deb_name) if verbose: print('Copying %s' % deb_path) new_path = os.path.join(debs_path, deb_name) shutil.copyfile(deb_path, new_path) if deb_name == 'buildvars.json': # buildvars.json is also in the artifacts_path; copied by # download_candidate_files(). Set it aside so it can be # sync'd to S3 as a record of what was published. buildvar_dir = '{}/weekly/{}/{}'.format( path, timestamp, branch_name) if not os.path.isdir(buildvar_dir): os.makedirs(buildvar_dir) buildvar_path = '{}/{}'.format(buildvar_dir, deb_name) shutil.copyfile(deb_path, buildvar_path) assemble_script, publish_script = get_scripts(juju_release_tools) # XXX sinzui 2014-12-01: IGNORE uses the local juju, but when # testing juju's that change generate-tools, we may need to use # the highest version. command = [ assemble_script, '-t', debs_path, 'weekly', 'IGNORE', streams_path] run_command(command, dry_run=dry_run, verbose=verbose) publish(streams_path, publish_script, dry_run=dry_run, verbose=verbose) # Sync buildvars.json files out to s3. url = 's3://juju-qa-data/juju-releases/weekly/' s3_path = '{}/weekly/{}'.format(path, timestamp) if verbose: print('Calling s3cmd to sync %s out to %s' % (s3_path, url)) if not dry_run: s3_cmd(['sync', s3_path, url]) extract_candidates(path, dry_run=dry_run, verbose=verbose)
def optimize_model(expr_name, onnx_file, model_xml, weight_bin, opt_ops=""): # import ipdb as pdb; pdb.set_trace() run_opt = False options = "" if (model_xml == None): util.print_log("No xml model was provided", id_str="warning") run_opt = True if (weight_bin == None): util.print_log("No binary weight file was provided", id_str="warning") run_opt = True if run_opt: util.print_banner("Running OpenVino optimizer on {0}".format(onnx_file), color='green', verbosity="VERB_LOW") options += "--input_model={0} ".format(onnx_file) options += "--model_name {0} ".format(expr_name) options += (" --"+opt_ops[0]) if len(opt_ops)==1 else "".join(" --"+e for e in opt_ops) cmd = "python {0}/deployment_tools/model_optimizer/mo_onnx.py {1}".format(openvino_inst_path, options) util.run_command(cmd, verbosity="VERB_LOW") model_xml, weight_bin = expr_name+".xml", expr_name+".bin" # load model # import ipdb as pdb; pdb.set_trace() return model_xml, weight_bin
def train(model_name, epochs, batch_size): if model_name == "LSTM": command = "CUDA_VISIBLE_DEVICES=0 fairseq-train {} --clip-norm 5 --batch-size {} \ --save-dir checkpoints/lstm --arch lstm --max-epoch {} --encoder-hidden-size 258 \ --encoder-layers 2 --decoder-hidden-size 258 --decoder-layers 2 --optimizer adam --lr 0.001 \ --dropout 0.3 --encoder-embed-path glove.6B.300d.txt --encoder-bidirectional --encoder-embed-dim 300 \ --decoder-embed-dim 300 --no-epoch-checkpoints --decoder-embed-path glove.6B.300d.txt --decoder-out-embed-dim 300 \ --num-workers 3".format( FAIRSEQ_PREPROCESSED_DATASET, batch_size, epochs ) elif model_name == "CNN": command = "CUDA_VISIBLE_DEVICES=0 fairseq-train {} --batch-size {} \ --save-dir checkpoints/conv --arch fconv_iwslt_de_en --max-epoch {} \ --optimizer adam --lr 0.001 \ --dropout 0.3 --encoder-embed-path glove.6B.300d.txt --encoder-embed-dim 300 \ --decoder-embed-dim 300 --no-epoch-checkpoints --decoder-embed-path glove.6B.300d.txt --decoder-out-embed-dim 300 \ --num-workers 3".format( FAIRSEQ_PREPROCESSED_DATASET, batch_size, epochs ) else: raise NotImplementedError run_command(command)
def test_run_command_dry_run(self): with patch('subprocess.check_output') as co_mock: run_command(['foo', 'bar'], dry_run=True) self.assertEqual(0, co_mock.call_count)
verbosity="VERB_LOW") sys.exit() util.print_banner("Compiling input files", verbosity=verbosity) if files == None: util.print_log("You need to provide f-file", "ERROR", verbosity="VERB_LOW") sys.exit() sv_rtl, v_rtl, vhdl_rtl = get_rtl_files(files) # import ipdb as pdb; pdb.set_trace() if sv_rtl != "": cmd_to_run = "xvlog --sv {0} ".format(sv_rtl) if silence: cmd_to_run += "> /dev/null" util.run_command(cmd_to_run, split=False, verbosity=verbosity) if v_rtl != "": cmd_to_run = "xvlog {0} ".format(v_rtl) if silence: cmd_to_run += "> /dev/null" util.run_command(cmd_to_run, split=False, verbosity=verbosity) if vhdl_rtl != "": cmd_to_run = "xvhdl {0} ".format(vhdl_rtl) if silence: cmd_to_run += "> /dev/null" util.run_command(cmd_to_run, split=False, verbosity=verbosity) util.print_banner("Creating snapshot", verbosity=verbosity) cmd_to_run = "xelab {0} ".format(top_level) if waveform: cmd_to_run += "--debug all "
def preprocess(self): """Preprocesses the dataset""" # shutil.copyfile(original, target) if not os.path.exists( os.path.join(DATASET_FOLDER, PROCESSED_DATASET_FOLDER)): os.mkdir(os.path.join(DATASET_FOLDER, PROCESSED_DATASET_FOLDER)) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["train"][0]), PROCESSED_DATASET["train"] + ".sentence", ) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["train"][1]), PROCESSED_DATASET["train"] + ".question", ) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["valid"][0]), PROCESSED_DATASET["valid"] + ".sentence", ) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["valid"][1]), PROCESSED_DATASET["valid"] + ".question", ) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["test"][0]), PROCESSED_DATASET["test"] + ".sentence", ) shutil.copyfile( os.path.join(self.dataset_location, RAW_DATASET_FILENAMES["test"][1]), PROCESSED_DATASET["test"] + ".question", ) logger.info( "Running FairSeq Preprocessing to convert files into fairseq binaries" ) if os.path.exists(FAIRSEQ_PREPROCESSED_DATASET): logger.debug("Old Binaries present deleting them") shutil.rmtree(FAIRSEQ_PREPROCESSED_DATASET) logger.debug("Deleted old binaries now generating new one's") pre_process_command = "fairseq-preprocess --source-lang sentence --target-lang question \ --trainpref {} --testpref {} \ --validpref {} --destdir {} --seed {} \ --nwordssrc {} --nwordstgt {}".format( PROCESSED_DATASET["train"], PROCESSED_DATASET["test"], PROCESSED_DATASET["valid"], FAIRSEQ_PREPROCESSED_DATASET, SEED, SRC_WORDS, TRG_WORDS, ) run_command(pre_process_command)
def preprocess(self): """Preprocesses the dataset""" with open(RAW_DATASET) as csv_file: csv_reader = csv.DictReader(csv_file, delimiter="\t") for row in csv_reader: if "_" in row["Question"]: row["Question"] = re.sub(r"[_]{2,}", "_", row["Question"]) self.dataset["sentence"].append(row["answer"].lstrip().strip()) self.dataset["question"].append(row["Question"].strip()) logger.debug("DataSet Preprocessed Successfully!") self.dataset = pd.DataFrame.from_dict(self.dataset) # Test Split self.trainset, self.testset = train_test_split( self.dataset, test_size=0.15, random_state=SEED ) # Valid Split self.trainset, self.validset = train_test_split( self.trainset, test_size=0.10, random_state=SEED ) self.trainset["sentence"].to_csv( "{}.sentence".format(PROCESSED_DATASET["train"]), index=False, sep="\t", header=False, ) self.validset["sentence"].to_csv( "{}.sentence".format(PROCESSED_DATASET["valid"]), index=False, sep="\t", header=False, ) self.testset["sentence"].to_csv( "{}.sentence".format(PROCESSED_DATASET["test"]), index=False, sep="\t", header=False, ) self.trainset["question"].to_csv( "{}.question".format(PROCESSED_DATASET["train"]), index=False, sep="\t", header=False, ) self.validset["question"].to_csv( "{}.question".format(PROCESSED_DATASET["valid"]), index=False, sep="\t", header=False, ) self.testset["question"].to_csv( "{}.question".format(PROCESSED_DATASET["test"]), index=False, sep="\t", header=False, ) logger.debug( "Trainset Size: {}, Validset Size: {}, Tesetset Size: {}".format( self.trainset.shape, self.validset.shape, self.testset.shape ) ) logger.debug( "Saving the file preprocessed files to : {}".format( PROCESSED_DATASET_FOLDER ) ) logger.info( "Running FairSeq Preprocessing to convert files into fairseq binaries" ) if os.path.exists(FAIRSEQ_PREPROCESSED_DATASET): logger.debug("Old Binaries present deleting them") shutil.rmtree(FAIRSEQ_PREPROCESSED_DATASET) logger.debug("Deleted old binaries now generating new one's") pre_process_command = "fairseq-preprocess --source-lang sentence --target-lang question \ --trainpref {} --testpref {} \ --validpref {} --destdir {} --seed {} \ --nwordssrc 5000 --nwordstgt 5000".format( PROCESSED_DATASET["train"], PROCESSED_DATASET["test"], PROCESSED_DATASET["valid"], FAIRSEQ_PREPROCESSED_DATASET, SEED, ) run_command(pre_process_command)
def run_cmds(commands): for cmd in commands: run_command(cmd, verbose=True)
def test_run_command_args(self): with patch('subprocess.check_output') as co_mock: run_command(['foo', 'bar']) args, kwargs = co_mock.call_args self.assertEqual((['foo', 'bar'], ), args)
def test_run_command_verbose(self): with patch('subprocess.check_output'): with patch('utility.print_now') as p_mock: run_command(['foo', 'bar'], verbose=True) self.assertEqual(2, p_mock.call_count)
default_path_to_fpga_dir)) path_fpga = default_path_to_fpga_dir # Check if project has already been created if os.path.isdir("{0}{1}".format(path_for_proj, project_name)): util.print_log("Project path {0}{1} already exist!".format( path_for_proj, project_name), id_str="ERROR") sys.exit() if os.getcwd() not in path_for_proj: proj_dir = os.getcwd() + "/" + path_for_proj + project_name else: proj_dir = path_for_proj + project_name proj_dir = os.path.abspath(proj_dir) # import ipdb as pdb; pdb.set_trace() command = "mkdir {0}".format(project_name) util.run_command(command, verbosity=verbosity) command = "cd {0}".format(proj_dir) util.run_command(command, verbosity=verbosity) command = "mkdir {0}/docs {0}/results {0}/rtl {0}/scripts {0}/sw {0}/tb".format( proj_dir) util.run_command(command, verbosity=verbosity) command = "touch {0}/rtl/{1}.sv".format(proj_dir, project_name.lower()) util.run_command(command, verbosity=verbosity) command = "touch {0}/tb/{1}_tester.sv".format(proj_dir, project_name.lower()) util.run_command(command, verbosity=verbosity) if not (__args__['dependancy']) == None: deps = read_dep_file(__args__['dependancy']) link_dependencies(proj_dir, deps, verbosity=verbosity) create_f_file(proj_dir, deps, verbosity=verbosity)