def main(data_paths_fname, stats_list_fname, stats_key, stat_indices=None, verbose=True, check_if_file_exists=False): data_paths = util.read_yaml(data_paths_fname) stats_list = util.read_yaml(stats_list_fname) stats = stats_list[stats_key] db_keys = get_db_keys(stats) if stats_list.has_key('_config') == True: out_dir = stats_list['_config']['out_dir'] assert out_dir.endswith('/') else: out_dir = '' db = util.Database(data_paths_fname) db.load_db(db_keys, people=True) start_time = time.time() data = patient_stats(db, stats, stats_key, out_dir, stat_indices, verbose, check_if_file_exists) if verbose == True: print time.time() - start_time return data
def _configure_source_type(self, args: argparse.Namespace): self.source_dir = os.path.abspath(args.source_dir) source_yaml = util.read_yaml( os.path.join(self.source_dir, 'versions.yml')) self.source_type = SourceType(source_yaml['SourceType']) if self.source_type == SourceType.SOURCE_PACKAGE: self.revision = source_yaml['Revision']
def __init__(self, conffile): self.conf = read_yaml(conffile) self.col = Colours() self.req = {} self.req['headers'] = {'user-agent': self.conf['ua']} self.s_in = self.session_login() self.s_out = requests.Session()
def main(): """main function""" args = parse_args() # Clone pylero repo ret = clone_config_repo(git_repo=PYLERO_REPO, git_branch="main", repo_dir="pylero") if not ret: sys.exit(1) polarion_config_file = "polarion_config.yml" filename = SCRIPT_DIR + polarion_config_file config_data = read_yaml(filename) generate_polarion_config(filename, config_data, args.testrun_title) # Installs pylero cmd = "pip3 install pylero/." ret = subprocess.call(cmd, shell=True) if ret: print("Failed to install pylero") return False addPropertyScriptName = SCRIPT_DIR + "xunit_add_properties.py" testRunCreateScriptName = SCRIPT_DIR + "createPolarionTestRun.py" # Add property tag to the test report xml file" output_report_file = "polarion-report-" + str(uuid.uuid1()) + ".xml" cmd = ("python3 {} -c {} -i {} -x {} -o {}".format(addPropertyScriptName, polarion_config_file, args.robot_result_file, args.xunit_xml_file, output_report_file)) ret = subprocess.call(cmd, shell=True) if ret: print("Failed to add property to the test report xml file") return False # Create test plan and test runs in polarion cmd = ("python3 {} -s {} -t {} -n {} -u {} -p {}".format( testRunCreateScriptName, args.release_plan_name, args.test_plan_name, args.testrun_title, args.polarion_username, args.polarion_password)) ret = subprocess.call(cmd, shell=True) if ret: print("Failed to create test plan and test runs in polarion") return False # Update test results in polarion cmd = ("curl -k -X POST -u {}:{} -F file=@{} {}".format( args.polarion_username, args.polarion_password, output_report_file, POLARION_URL)) ret = subprocess.call(cmd, shell=True) if ret: print("Failed to update test results in polarion") return False
def get_all_versions(filename: str) -> Mapping[str, LLVMBMTC]: """Build the database containing all releases from a YAML file.""" versions = {} yml = util.read_yaml(filename) for value in yml['Revisions']: toolchain = LLVMBMTC(value) if toolchain.revision in versions: die('toolchain revision {} previously defined'.format( toolchain.revision)) versions[toolchain.revision] = toolchain return versions
def _read_configs(problem): # some defaults settings = { 'timelimit': 1, 'name': '', 'floatabs': None, 'floatrel': None, 'validation': 'default', 'case_sensitive': False, 'space_change_sensitive': False, 'validator_flags': None } # parse problem.yaml yamlpath = problem / 'problem.yaml' for k, v in util.read_yaml(problem / 'problem.yaml').items(): settings[k] = v # parse validator_flags if 'validator_flags' in settings and settings['validator_flags']: flags = settings['validator_flags'].split(' ') i = 0 while i < len(flags): if flags[i] in ['case_sensitive', 'space_change_sensitive']: settings[flags[i]] = True elif flags[i] == 'float_absolute_tolerance': settings['floatabs'] = float(flags[i + 1]) i += 1 elif flags[i] == 'float_relative_tolerance': settings['floatrel'] = float(flags[i + 1]) i += 1 elif flags[i] == 'float_tolerance': settings['floatabs'] = float(flags[i + 1]) settings['floatrel'] = float(flags[i + 1]) i += 1 i += 1 # TODO: Get rid of domjudge-problem.ini; it's only remaining use is # timelimits. # parse domjudge-problem.ini domjudge_path = problem / 'domjudge-problem.ini' if domjudge_path.is_file(): with domjudge_path.open() as f: for line in f.readlines(): key, var = line.strip().split('=') var = var[1:-1] settings[key] = float(var) if key == 'timelimit' else var return settings
def read_classes(): """ Reads yaml with classes That .yaml file is also used for training YOLO network :return classes_names - list with class names (names of detected gestures) meaning_id_dict - meanings with their ids """ data_dict = read_yaml(CLASSES_YAML) classes_names = data_dict['names'] classes_count = data_dict['nc'] assert (classes_count == len(classes_names)) return classes_names, classes_count
def __init__(self, pod_name, verbose=0): """POD initializer. Given a POD name, we attempt to read a settings.yml file in a subdirectory of ``/diagnostics`` by that name and parse the contents. Args: pod_name (:obj:`str`): Name of the POD to initialize. verbose (:obj:`int`, optional): Logging verbosity level. Default 0. """ paths = util.PathManager() self.name = pod_name self.__dict__.update(paths.podPaths(self)) file_contents = util.read_yaml( os.path.join(self.POD_CODE_DIR, 'settings.yml')) config = self._parse_pod_settings(file_contents['settings'], verbose) self.__dict__.update(config) config = self._parse_pod_varlist(file_contents['varlist'], verbose) self.varlist = config
def main(): """main function""" args = parse_args() if not args.skip_clone: ret = clone_config_repo(git_repo=args.git_repo, git_branch=args.git_repo_branch, repo_dir=args.repo_dir, git_username=args.git_username, git_password=args.git_password) if not ret: sys.exit(1) else: print("Skipping cloning of config gitlab repo") config_file = args.repo_dir + "/test-variables.yml" config_data = read_yaml(config_file) # Generate test config file generate_test_config_file(args.config_template, config_data, args.test_cluster, args.set_prometheus_config)
def read_weights(): """ Reads yaml with weights and meanings :return gesture_weight_dict - weights list for meanings in each gesture (by gesture's id) meaning_id_dict - meanings with their ids """ data_dict = read_yaml(WEIGHTS_YAML) gesture_weights_dict = {} for gesture in data_dict['weights']: value = next(iter(gesture.values())) key = next(iter(gesture.keys())) assert (len(value) == len(data_dict['meanings']) ) # vector of meanings don't match value = np.array(value) gesture_weights_dict[key] = value meanings_id_dict = (pd.Categorical(data_dict['meanings'])) return gesture_weights_dict, meanings_id_dict
def argparse_wrapper(): cwd = os.path.dirname(os.path.realpath(__file__)) # gets dir of currently executing script code_root = os.path.realpath(os.path.join(cwd, '..')) # parent dir of that parser = argparse.ArgumentParser() parser.add_argument("-v", "--verbosity", action="count", help="Increase output verbosity") # default paths set in config.yml/paths parser.add_argument('--CODE_ROOT', nargs='?', type=str, default=code_root, help="Code installation directory.") parser.add_argument('--MODEL_DATA_ROOT', nargs='?', type=str, help="Parent directory containing results from different models.") parser.add_argument('--OBS_DATA_ROOT', nargs='?', type=str, help="Parent directory containing observational data used by individual PODs.") parser.add_argument('--WORKING_DIR', nargs='?', type=str, help="Working directory.") parser.add_argument('--OUTPUT_DIR', nargs='?', type=str, help="Directory to write output files. Defaults to working directory.") # defaults set in config.yml/settings parser.add_argument("--test_mode", action="store_const", const=True, help="Set flag to do a dry run, disabling calls to PODs") parser.add_argument('--data_manager', nargs='?', type=str, help="Method to fetch model data. Currently supported options are {'Localfile'}.") parser.add_argument('--environment_manager', nargs='?', type=str, help="Method to manage POD runtime dependencies. Currently supported options are {'None', 'Conda'}.") # non-flag arguments parser.add_argument('config_file', nargs='?', type=str, default=os.path.join(cwd, 'config.yml'), help="Configuration file.") args = parser.parse_args() if args.verbosity == None: verbose = 1 else: verbose = args.verbosity + 1 # fix for case verb = 0 config = util.read_yaml(args.config_file) config = util.parse_mdtf_args(args, config) config.verbose = verbose return config
logging.info('There are {} individauls being included.'.format( df_pheno.shape[1])) logging.info('Loading genotypes.') pr = genotypeio.PlinkReader(args.geno_bed_prefix, select_samples=df_pheno.columns, dtype=np.int8) variant_df = pr.bim.set_index('snp')[['chrom', 'pos']] genotype_df = pd.DataFrame(pr.load_genotypes(), index=pr.bim['snp'], columns=pr.fam['iid']) if args.map_trans_params is None: map_args = {} else: map_args = read_yaml(args.map_trans_params) logging.info('Arguments used for tensorqtl: ' + dict_to_str(map_args)) logging.info('Start mapping.') pairs_df = trans.map_trans(genotype_df, df_pheno, df_covar, logger=None, **map_args) logging.info('Finish mapping and writing to disk.') pheno_list = list(df_pheno.index) for pheno in tqdm(pheno_list): sub = pairs_df[pairs_df.phenotype_id == pheno].reset_index(drop=True) if sub.shape[0] > 0: sub.to_parquet('{}{}.parquet'.format(args.output_prefix, pheno),
def build_jar(source_jar_path, dest_jar_path, config, venv=None, definition=None, logdir=None): """Build a StormTopology .jar which encapsulates the topology defined in topology_dir. Optionally override the module and function names. This feature supports the definition of multiple topologies in a single directory.""" if definition is None: definition = 'create.create' # Prepare data we'll use later for configuring parallelism. config_yaml = read_yaml(config) parallelism = dict((k.split('.')[-1], v) for k, v in config_yaml.iteritems() if k.startswith('petrel.parallelism')) pip_options = config_yaml.get('petrel.pip_options', '') module_name, dummy, function_name = definition.rpartition('.') topology_dir = os.getcwd() # Make a copy of the input "jvmpetrel" jar. This jar acts as a generic # starting point for all Petrel topologies. source_jar_path = os.path.abspath(source_jar_path) dest_jar_path = os.path.abspath(dest_jar_path) if source_jar_path == dest_jar_path: raise ValueError("Error: Destination and source path are the same.") shutil.copy(source_jar_path, dest_jar_path) jar = zipfile.ZipFile(dest_jar_path, 'a', compression=zipfile.ZIP_DEFLATED) added_path_entry = False try: # Add the files listed in manifest.txt to the jar. with open(os.path.join(topology_dir, MANIFEST), 'r') as f: for fn in f.readlines(): # Ignore blank and comment lines. fn = fn.strip() if len(fn) and not fn.startswith('#'): add_file_to_jar(jar, os.path.expandvars(fn.strip())) # Add user and machine information to the jar. add_to_jar( jar, '__submitter__.yaml', ''' petrel.user: %s petrel.host: %s ''' % (getpass.getuser(), socket.gethostname())) # Also add the topology configuration to the jar. with open(config, 'r') as f: config_text = f.read() add_to_jar(jar, '__topology__.yaml', config_text) # Call module_name/function_name to populate a Thrift topology object. builder = TopologyBuilder() module_dir = os.path.abspath(topology_dir) if module_dir not in sys.path: sys.path[:0] = [module_dir] added_path_entry = True module = __import__(module_name) getattr(module, function_name)(builder) # Add the spout and bolt Python scripts to the jar. Create a # setup_<script>.sh for each Python script. # Add Python scripts and any other per-script resources. for k, v in chain(builder._spouts.iteritems(), builder._bolts.iteritems()): # components inherited from petrel.storm does not have script property # and should be added via manifest.txt if hasattr(v, 'script'): add_file_to_jar(jar, topology_dir, v.script) # Create a bootstrap script. if venv is not None: # Allow overriding the execution command from the "petrel" # command line. This is handy if the server already has a # virtualenv set up with the necessary libraries. v.execution_command = os.path.join(venv, 'bin/python') # If a parallelism value was specified in the configuration YAML, # override any setting provided in the topology definition script. if k in parallelism: builder._commons[k].parallelism_hint = int(parallelism.pop(k)) emitter = builder._spouts.get(k, None) or builder._bolts.get( k, None) instance_of_emitter_base = isinstance(emitter, EmitterBase) if instance_of_emitter_base: v.execution_command, v.script = \ intercept_petrel_emitter(venv, v.execution_command, \ os.path.splitext(v.script)[0], jar, pip_options, logdir) else: import base64, pickle, re from operator import add module_name = '%s' % emitter.__module__ class_name = '%s' % emitter.__class__.__name__ v.script = '%s@%s.%s' % (k, module_name, class_name) v.script = re.sub('[\W]+', '_', k) v.execution_command = EmitterBase.DEFAULT_PYTHON emitter = base64.b64encode(pickle.dumps(emitter)) bootstrap = [] for bootstrap_cmpnt in builder._bootstrap: bootstrap.append(bootstrap_cmpnt) bootstrap = base64.b64encode(pickle.dumps(bootstrap)) v.execution_command, v.script = \ intercept_storm_emitter(venv, v.execution_command, v.script, \ jar, pip_options, logdir, \ module_name, class_name, emitter, bootstrap) if len(parallelism): raise ValueError( 'Parallelism settings error: There are no components named: %s' % ','.join(parallelism.keys())) # Build the Thrift topology object and serialize it to the .jar. Must do # this *after* the intercept step above since that step may modify the # topology definition. io = StringIO() topology = builder.write(io) add_to_jar(jar, 'topology.ser', io.getvalue()) finally: jar.close() if added_path_entry: # Undo our sys.path change. sys.path[:] = sys.path[1:]
def run(out_dir, config_fname, data_paths_fname, stats_list_fname, split_fname=None, check_if_file_exists=False, verbose=True): data_paths = util.read_yaml(data_paths_fname) config = util.read_yaml(config_fname) stats_key = config['stats_key'] outcome_stat_name = config['outcome_stat_name'] cohort_stat_name = config.get('cohort_stat_name', None) lab_lower_bound = config.get('lab_lower_bound', None) lab_upper_bound = config.get('lab_upper_bound', None) gap_days = config.get('gap_days', None) training_window_days = config['training_window_days'] buffer_window_days = config['buffer_window_days'] outcome_window_days = config['outcome_window_days'] time_period_days = config['time_period_days'] time_scale_days = config['time_scale_days'] use_just_labs = config['use_just_labs'] feature_loincs_fname = config['feature_loincs_fname'] add_age_sex = config['add_age_sex'] calc_gfr = config['calc_gfr'] regularizations = config.get('regularizations', [1]) lin_n_cv_iters = config.get('lin_n_cv_iters', -1) n_cv_iters = config.get('n_cv_iters', -1) progression = config['progression'] progression_lab_lower_bound = config.get('progression_lab_lower_bound', None) progression_lab_upper_bound = config.get('progression_lab_upper_bound', None) progression_gap_days = config.get('progression_gap_days', None) progression_stages = config.get('progression_stages', None) progression_init_stages = config.get('progression_init_stages', None) evaluate_nn = config.get('evaluate_nn', True) outcome_fname = out_dir + stats_key + '_' + outcome_stat_name + '.txt' if cohort_stat_name is None: cohort_fname = data_paths['demographics_fname'] else: cohort_fname = out_dir + stats_key + '_' + cohort_stat_name + '.txt' gfr_loincs = util.read_list_files('data/gfr_loincs.txt') training_data_fname = out_dir + stats_key + '_training_data.txt' feature_loincs = util.read_list_files(feature_loincs_fname) if use_just_labs == False: feature_diseases = [[ icd9 ] for icd9 in util.read_list_files('data/kidney_disease_mi_icd9s.txt')] feature_drugs = [ util.read_list_files('data/drug_class_' + dc.lower().replace( '-', '_').replace(',', '_').replace(' ', '_') + '_ndcs.txt') for dc in util.read_list_files( 'data/kidney_disease_drug_classes.txt') ] else: feature_diseases = [] feature_drugs = [] n_labs = len(feature_loincs) if add_age_sex: age_index = len(feature_loincs) + len(feature_diseases) + len( feature_drugs) gender_index = len(feature_loincs) + len(feature_diseases) + len( feature_drugs) + 1 else: age_index = None gender_index = None features_fname = out_dir + stats_key + '_features.h5' features_split_fname = out_dir + stats_key + '_features_split.h5' predict_fname = out_dir + stats_key + '_prediction_results.yaml' if evaluate_nn: nn_predict_fname = out_dir + stats_key + '_nn_prediction_results.yaml' else: nn_predict_fname = None if verbose: print "Loading data" db = util.Database(data_paths_fname) db.load_people() db.load_db(['loinc', 'loinc_vals', 'cpt', 'icd9_proc', 'icd9', 'ndc']) stats = util.read_yaml(stats_list_fname)[stats_key] if verbose: print "Calculating patient stats" data = ps.patient_stats(db, stats, stats_key, out_dir, stat_indices=None, verbose=verbose, check_if_file_exists=check_if_file_exists, save_files=True) if verbose: print "Building training data" outcome_data = btd.build_outcome_data(out_dir, outcome_fname) cohort_data = btd.setup(data_paths['demographics_fname'], outcome_fname, cohort_fname) # calc_gfr = True here because it's required to define the condition training_data = btd.build_training_data(db, cohort_data, gfr_loincs, lab_lower_bound, lab_upper_bound, \ training_window_days, buffer_window_days, outcome_window_days, time_period_days, time_scale_days, gap_days, calc_gfr=True, verbose=verbose, \ progression=progression, progression_lab_lower_bound=progression_lab_lower_bound, progression_lab_upper_bound=progression_lab_upper_bound, \ progression_gap_days=progression_gap_days, progression_init_stages=progression_init_stages, progression_stages=progression_stages) training_data.to_csv(training_data_fname, index=False, sep='\t') if verbose: print "Building features" features.features(db, training_data, feature_loincs, feature_diseases, feature_drugs, time_scale_days, features_fname, calc_gfr, verbose, add_age_sex) if split_fname is None: split_fname = out_dir + stats_key + '_split.txt' features.train_validation_test_split(training_data['person'].unique(), split_fname, verbose=verbose) features.split(features_fname, features_split_fname, split_fname, verbose) if verbose: print "Training, validating and testing models" predict.predict(features_split_fname, lin_n_cv_iters, n_cv_iters, regularizations, n_labs, age_index, gender_index, predict_fname, nn_predict_fname)
def build_contest_pdf(contest, problems, solutions=False, web=False): builddir = config.tmpdir / contest builddir.mkdir(parents=True, exist_ok=True) build_type = 'solution' if solutions else 'problem' main_file = 'solutions' if solutions else 'contest' main_file += '-web.tex' if web else '.tex' if solutions: ensure_symlink(builddir / 'solutions-base.tex', config.tools_root / 'latex/solutions-base.tex') ensure_symlink(builddir / 'bapc.cls', config.tools_root / 'latex/bapc.cls') ensure_symlink(builddir / 'images', config.tools_root / 'latex/images') ensure_symlink(builddir / main_file, config.tools_root / 'latex' / main_file) config_data = util.read_yaml(Path('contest.yaml')) config_data['testsession'] = '\\testsession' if config_data.get( 'testsession') else '' util.copy_and_substitute(config.tools_root / 'latex/contest-data.tex', builddir / 'contest_data.tex', config_data) ensure_symlink(builddir / 'logo.pdf', find_logo()) problems_data = '' # Link the solve stats directory if it exists. solve_stats = Path('solve_stats') if solve_stats.exists(): ensure_symlink(builddir / 'solve_stats', solve_stats) # include a header slide in the solutions PDF headertex = Path('solution_header.tex') if headertex.exists(): ensure_symlink(builddir / 'solution_header.tex', headertex) if solutions and headertex.exists(): problems_data += f'\\input{{{headertex}}}\n' per_problem_data = (config.tools_root / 'latex' / f'contest-{build_type}.tex').read_text() # Some logic to prevent duplicate problem IDs. for problem in problems: prepare_problem(problem) id_ok = True problems_data += util.substitute( per_problem_data, { 'problemlabel': problem.label, 'problemyamlname': problem.config['name'], 'problemauthor': problem.config.get('author'), 'timelimit': get_tl(problem.config), 'problemdir': config.tmpdir / problem.id, }) # include a statistics slide in the solutions PDF footer_tex = Path('solution_footer.tex') if footer_tex.exists(): ensure_symlink(builddir / 'solution_footer.tex', footer_tex) if solutions and footer_tex.exists(): problems_data += f'\\input{{{footer_tex}}}\n' (builddir / f'contest-{build_type}s.tex').write_text(problems_data) for i in range(3): ok, err, out = util.exec_command(PDFLATEX + [ '-output-directory', builddir, (builddir / main_file).with_suffix('.tex') ], 0, False, cwd=builddir, stdout=subprocess.PIPE) if ok is not True: print(f'{_c.red}Failure compiling pdf:{_c.reset}\n{out}') return False # link the output pdf output_pdf = Path(main_file).with_suffix('.pdf') ensure_symlink(output_pdf, builddir / output_pdf, True) print(f'{_c.green}Pdf written to {output_pdf}{_c.reset}') return True
def build_jar(source_jar_path, dest_jar_path, config, venv=None, definition=None): """Build a StormTopology .jar which encapsulates the topology defined in topology_dir. Optionally override the module and function names. This feature supports the definition of multiple topologies in a single directory.""" if definition is None: definition = 'create.create' # Prepare data we'll use later for configuring parallelism. config_yaml = read_yaml(config) parallelism = dict((k.split('.')[-1], v) for k, v in config_yaml.iteritems() if k.startswith('petrel.parallelism')) pip_options = config_yaml.get('petrel.pip_options', '') module_name, dummy, function_name = definition.rpartition('.') topology_dir = os.getcwd() # Make a copy of the input "jvmpetrel" jar. This jar acts as a generic # starting point for all Petrel topologies. source_jar_path = os.path.abspath(source_jar_path) dest_jar_path = os.path.abspath(dest_jar_path) if source_jar_path == dest_jar_path: raise ValueError("Error: Destination and source path are the same.") shutil.copy(source_jar_path, dest_jar_path) jar = zipfile.ZipFile(dest_jar_path, 'a', compression=zipfile.ZIP_DEFLATED) added_path_entry = False try: # Add the files listed in manifest.txt to the jar. with open(os.path.join(topology_dir, MANIFEST), 'r') as f: for fn in f.readlines(): # Ignore blank and comment lines. fn = fn.strip() if len(fn) and not fn.startswith('#'): add_file_to_jar(jar, fn.strip()) # Add user and machine information to the jar. add_to_jar(jar, '__submitter__.yaml', ''' petrel.user: %s petrel.host: %s ''' % (getpass.getuser(),socket.gethostname())) # Also add the topology configuration to the jar. with open(config, 'r') as f: config_text = f.read() add_to_jar(jar, '__topology__.yaml', config_text) # Call module_name/function_name to populate a Thrift topology object. builder = TopologyBuilder() module_dir = os.path.abspath(topology_dir) if module_dir not in sys.path: sys.path[:0] = [ module_dir ] added_path_entry = True module = __import__(module_name) getattr(module, function_name)(builder) # Add the spout and bolt Python scripts to the jar. Create a # setup_<script>.sh for each Python script. # Add Python scripts and any other per-script resources. for k, v in chain(builder._spouts.iteritems(), builder._bolts.iteritems()): add_file_to_jar(jar, topology_dir, v.script) # Create a bootstrap script. if venv is not None: # Allow overriding the execution command from the "petrel" # command line. This is handy if the server already has a # virtualenv set up with the necessary libraries. v.execution_command = os.path.join(venv, 'bin/python') # If a parallelism value was specified in the configuration YAML, # override any setting provided in the topology definition script. if k in parallelism: builder._commons[k].parallelism_hint = int(parallelism.pop(k)) v.execution_command, v.script = \ intercept(venv, v.execution_command, os.path.splitext(v.script)[0], jar, pip_options) if len(parallelism): raise ValueError( 'Parallelism settings error: There are no components named: %s' % ','.join(parallelism.keys())) # Build the Thrift topology object and serialize it to the .jar. Must do # this *after* the intercept step above since that step may modify the # topology definition. io = StringIO() topology = builder.write(io) add_to_jar(jar, 'topology.ser', io.getvalue()) finally: jar.close() if added_path_entry: # Undo our sys.path change. sys.path[:] = sys.path[1:]
import data import neural_network import support_vector_machine import predict import test_accuracy import util if __name__ == "__main__": config = util.read_yaml() all_students_data_path = config["all_students_data_path"] target_cap = config["target_cap"] # Build Neural Network print("---------------------------------------------------------") print("Generating Neural Network:\n") try: model_NN = neural_network.create_model() except: model_NN = neural_network.retrieve_model() print("\n") # Build Support Vector Machine print("---------------------------------------------------------") print("Generating Support Vector Machine:\n") try: model_SVM = support_vector_machine.create_model() except: model_SVM = support_vector_machine.retrieve_model() print("\n") # Evaluate Neural Network
def build_contest_pdf(contest, problems, tmpdir, solutions=False, web=False): builddir = tmpdir / contest builddir.mkdir(parents=True, exist_ok=True) build_type = 'solution' if solutions else 'problem' main_file = 'solutions' if solutions else 'contest' main_file += '-web.tex' if web else '.tex' config_data = util.read_yaml(Path('contest.yaml')) config_data['testsession'] = '\\testsession' if config_data.get('testsession') else '' config_data['logofile'] = find_logo().as_posix() util.copy_and_substitute(config.tools_root / 'latex/contest-data.tex', builddir / 'contest_data.tex', config_data) problems_data = '' if solutions: # Link the solve stats directory if it exists. solve_stats = Path('solve_stats') # include a header slide in the solutions PDF headertex = Path('solution_header.tex') if headertex.exists(): problems_data += f'\\input{{{headertex}}}\n' per_problem_data = (config.tools_root / 'latex' / f'contest-{build_type}.tex').read_text() # Some logic to prevent duplicate problem IDs. for problem in problems: if build_type == 'problem': prepare_problem(problem) problems_data += util.substitute( per_problem_data, { 'problemlabel': problem.label, 'problemyamlname': problem.settings.name.replace('_', ' '), 'problemauthor': problem.settings.author, 'timelimit': problem.settings.timelimit, 'problemdir': problem.path.absolute().as_posix(), 'builddir': problem.tmpdir.as_posix(), }) if solutions: # include a statistics slide in the solutions PDF footer_tex = Path('solution_footer.tex') if footer_tex.exists(): problems_data += f'\\input{{{footer_tex}}}\n' (builddir / f'contest-{build_type}s.tex').write_text(problems_data) env = get_environment() for i in range(3): ret = util.exec_command( PDFLATEX + ['-output-directory', builddir, config.tools_root / 'latex' / main_file], 0, False, cwd=builddir, stdout=subprocess.PIPE, env=env, ) if ret.ok is not True: print(f'{Fore.RED}Failure compiling pdf:{Style.RESET_ALL}\n{ret.out}') return False # link the output pdf output_pdf = Path(main_file).with_suffix('.pdf') ensure_symlink(output_pdf, builddir / output_pdf, True) print(f'{Fore.GREEN}Pdf written to {output_pdf}{Style.RESET_ALL}') return True