def main(data_paths_fname,
stats_list_fname,
stats_key,
stat_indices=None,
verbose=True,
check_if_file_exists=False):
data_paths = util.read_yaml(data_paths_fname)
stats_list = util.read_yaml(stats_list_fname)
stats = stats_list[stats_key]
db_keys = get_db_keys(stats)
if stats_list.has_key('_config') == True:
out_dir = stats_list['_config']['out_dir']
assert out_dir.endswith('/')
else:
out_dir = ''
db = util.Database(data_paths_fname)
db.load_db(db_keys, people=True)
start_time = time.time()
data = patient_stats(db, stats, stats_key, out_dir, stat_indices, verbose,
check_if_file_exists)
if verbose == True:
print time.time() - start_time
return data
def _configure_source_type(self, args: argparse.Namespace):
self.source_dir = os.path.abspath(args.source_dir)
source_yaml = util.read_yaml(
os.path.join(self.source_dir, 'versions.yml'))
self.source_type = SourceType(source_yaml['SourceType'])
if self.source_type == SourceType.SOURCE_PACKAGE:
self.revision = source_yaml['Revision']
def __init__(self, conffile):
self.conf = read_yaml(conffile)
self.col = Colours()
self.req = {}
self.req['headers'] = {'user-agent': self.conf['ua']}
self.s_in = self.session_login()
self.s_out = requests.Session()
def main():
"""main function"""
args = parse_args()
# Clone pylero repo
ret = clone_config_repo(git_repo=PYLERO_REPO,
git_branch="main",
repo_dir="pylero")
if not ret:
sys.exit(1)
polarion_config_file = "polarion_config.yml"
filename = SCRIPT_DIR + polarion_config_file
config_data = read_yaml(filename)
generate_polarion_config(filename, config_data, args.testrun_title)
# Installs pylero
cmd = "pip3 install pylero/."
ret = subprocess.call(cmd, shell=True)
if ret:
print("Failed to install pylero")
return False
addPropertyScriptName = SCRIPT_DIR + "xunit_add_properties.py"
testRunCreateScriptName = SCRIPT_DIR + "createPolarionTestRun.py"
# Add property tag to the test report xml file"
output_report_file = "polarion-report-" + str(uuid.uuid1()) + ".xml"
cmd = ("python3 {} -c {} -i {} -x {} -o {}".format(addPropertyScriptName,
polarion_config_file,
args.robot_result_file,
args.xunit_xml_file,
output_report_file))
ret = subprocess.call(cmd, shell=True)
if ret:
print("Failed to add property to the test report xml file")
return False
# Create test plan and test runs in polarion
cmd = ("python3 {} -s {} -t {} -n {} -u {} -p {}".format(
testRunCreateScriptName, args.release_plan_name, args.test_plan_name,
args.testrun_title, args.polarion_username, args.polarion_password))
ret = subprocess.call(cmd, shell=True)
if ret:
print("Failed to create test plan and test runs in polarion")
return False
# Update test results in polarion
cmd = ("curl -k -X POST -u {}:{} -F file=@{} {}".format(
args.polarion_username, args.polarion_password, output_report_file,
POLARION_URL))
ret = subprocess.call(cmd, shell=True)
if ret:
print("Failed to update test results in polarion")
return False
def get_all_versions(filename: str) -> Mapping[str, LLVMBMTC]:
"""Build the database containing all releases from a YAML file."""
versions = {}
yml = util.read_yaml(filename)
for value in yml['Revisions']:
toolchain = LLVMBMTC(value)
if toolchain.revision in versions:
die('toolchain revision {} previously defined'.format(
toolchain.revision))
versions[toolchain.revision] = toolchain
return versions
def _read_configs(problem):
# some defaults
settings = {
'timelimit': 1,
'name': '',
'floatabs': None,
'floatrel': None,
'validation': 'default',
'case_sensitive': False,
'space_change_sensitive': False,
'validator_flags': None
}
# parse problem.yaml
yamlpath = problem / 'problem.yaml'
for k, v in util.read_yaml(problem / 'problem.yaml').items():
settings[k] = v
# parse validator_flags
if 'validator_flags' in settings and settings['validator_flags']:
flags = settings['validator_flags'].split(' ')
i = 0
while i < len(flags):
if flags[i] in ['case_sensitive', 'space_change_sensitive']:
settings[flags[i]] = True
elif flags[i] == 'float_absolute_tolerance':
settings['floatabs'] = float(flags[i + 1])
i += 1
elif flags[i] == 'float_relative_tolerance':
settings['floatrel'] = float(flags[i + 1])
i += 1
elif flags[i] == 'float_tolerance':
settings['floatabs'] = float(flags[i + 1])
settings['floatrel'] = float(flags[i + 1])
i += 1
i += 1
# TODO: Get rid of domjudge-problem.ini; it's only remaining use is
# timelimits.
# parse domjudge-problem.ini
domjudge_path = problem / 'domjudge-problem.ini'
if domjudge_path.is_file():
with domjudge_path.open() as f:
for line in f.readlines():
key, var = line.strip().split('=')
var = var[1:-1]
settings[key] = float(var) if key == 'timelimit' else var
return settings
def read_classes():
"""
Reads yaml with classes
That .yaml file is also used for training YOLO network
:return classes_names - list with class names (names of detected gestures)
meaning_id_dict - meanings with their ids
"""
data_dict = read_yaml(CLASSES_YAML)
classes_names = data_dict['names']
classes_count = data_dict['nc']
assert (classes_count == len(classes_names))
return classes_names, classes_count
def __init__(self, pod_name, verbose=0):
"""POD initializer. Given a POD name, we attempt to read a settings.yml
file in a subdirectory of ``/diagnostics`` by that name and parse the
contents.
Args:
pod_name (:obj:`str`): Name of the POD to initialize.
verbose (:obj:`int`, optional): Logging verbosity level. Default 0.
"""
paths = util.PathManager()
self.name = pod_name
self.__dict__.update(paths.podPaths(self))
file_contents = util.read_yaml(
os.path.join(self.POD_CODE_DIR, 'settings.yml'))
config = self._parse_pod_settings(file_contents['settings'], verbose)
self.__dict__.update(config)
config = self._parse_pod_varlist(file_contents['varlist'], verbose)
self.varlist = config
def main():
"""main function"""
args = parse_args()
if not args.skip_clone:
ret = clone_config_repo(git_repo=args.git_repo,
git_branch=args.git_repo_branch,
repo_dir=args.repo_dir,
git_username=args.git_username,
git_password=args.git_password)
if not ret:
sys.exit(1)
else:
print("Skipping cloning of config gitlab repo")
config_file = args.repo_dir + "/test-variables.yml"
config_data = read_yaml(config_file)
# Generate test config file
generate_test_config_file(args.config_template, config_data,
args.test_cluster, args.set_prometheus_config)
def read_weights():
"""
Reads yaml with weights and meanings
:return gesture_weight_dict - weights list for meanings in each gesture (by gesture's id)
meaning_id_dict - meanings with their ids
"""
data_dict = read_yaml(WEIGHTS_YAML)
gesture_weights_dict = {}
for gesture in data_dict['weights']:
value = next(iter(gesture.values()))
key = next(iter(gesture.keys()))
assert (len(value) == len(data_dict['meanings'])
) # vector of meanings don't match
value = np.array(value)
gesture_weights_dict[key] = value
meanings_id_dict = (pd.Categorical(data_dict['meanings']))
return gesture_weights_dict, meanings_id_dict
def argparse_wrapper():
cwd = os.path.dirname(os.path.realpath(__file__)) # gets dir of currently executing script
code_root = os.path.realpath(os.path.join(cwd, '..')) # parent dir of that
parser = argparse.ArgumentParser()
parser.add_argument("-v", "--verbosity", action="count",
help="Increase output verbosity")
# default paths set in config.yml/paths
parser.add_argument('--CODE_ROOT', nargs='?', type=str,
default=code_root,
help="Code installation directory.")
parser.add_argument('--MODEL_DATA_ROOT', nargs='?', type=str,
help="Parent directory containing results from different models.")
parser.add_argument('--OBS_DATA_ROOT', nargs='?', type=str,
help="Parent directory containing observational data used by individual PODs.")
parser.add_argument('--WORKING_DIR', nargs='?', type=str,
help="Working directory.")
parser.add_argument('--OUTPUT_DIR', nargs='?', type=str,
help="Directory to write output files. Defaults to working directory.")
# defaults set in config.yml/settings
parser.add_argument("--test_mode", action="store_const", const=True,
help="Set flag to do a dry run, disabling calls to PODs")
parser.add_argument('--data_manager', nargs='?', type=str,
help="Method to fetch model data. Currently supported options are {'Localfile'}.")
parser.add_argument('--environment_manager', nargs='?', type=str,
help="Method to manage POD runtime dependencies. Currently supported options are {'None', 'Conda'}.")
# non-flag arguments
parser.add_argument('config_file', nargs='?', type=str,
default=os.path.join(cwd, 'config.yml'),
help="Configuration file.")
args = parser.parse_args()
if args.verbosity == None:
verbose = 1
else:
verbose = args.verbosity + 1 # fix for case verb = 0
config = util.read_yaml(args.config_file)
config = util.parse_mdtf_args(args, config)
config.verbose = verbose
return config
logging.info('There are {} individauls being included.'.format(
df_pheno.shape[1]))
logging.info('Loading genotypes.')
pr = genotypeio.PlinkReader(args.geno_bed_prefix,
select_samples=df_pheno.columns,
dtype=np.int8)
variant_df = pr.bim.set_index('snp')[['chrom', 'pos']]
genotype_df = pd.DataFrame(pr.load_genotypes(),
index=pr.bim['snp'],
columns=pr.fam['iid'])
if args.map_trans_params is None:
map_args = {}
else:
map_args = read_yaml(args.map_trans_params)
logging.info('Arguments used for tensorqtl: ' + dict_to_str(map_args))
logging.info('Start mapping.')
pairs_df = trans.map_trans(genotype_df,
df_pheno,
df_covar,
logger=None,
**map_args)
logging.info('Finish mapping and writing to disk.')
pheno_list = list(df_pheno.index)
for pheno in tqdm(pheno_list):
sub = pairs_df[pairs_df.phenotype_id == pheno].reset_index(drop=True)
if sub.shape[0] > 0:
sub.to_parquet('{}{}.parquet'.format(args.output_prefix, pheno),
def build_jar(source_jar_path,
dest_jar_path,
config,
venv=None,
definition=None,
logdir=None):
"""Build a StormTopology .jar which encapsulates the topology defined in
topology_dir. Optionally override the module and function names. This
feature supports the definition of multiple topologies in a single
directory."""
if definition is None:
definition = 'create.create'
# Prepare data we'll use later for configuring parallelism.
config_yaml = read_yaml(config)
parallelism = dict((k.split('.')[-1], v)
for k, v in config_yaml.iteritems()
if k.startswith('petrel.parallelism'))
pip_options = config_yaml.get('petrel.pip_options', '')
module_name, dummy, function_name = definition.rpartition('.')
topology_dir = os.getcwd()
# Make a copy of the input "jvmpetrel" jar. This jar acts as a generic
# starting point for all Petrel topologies.
source_jar_path = os.path.abspath(source_jar_path)
dest_jar_path = os.path.abspath(dest_jar_path)
if source_jar_path == dest_jar_path:
raise ValueError("Error: Destination and source path are the same.")
shutil.copy(source_jar_path, dest_jar_path)
jar = zipfile.ZipFile(dest_jar_path, 'a', compression=zipfile.ZIP_DEFLATED)
added_path_entry = False
try:
# Add the files listed in manifest.txt to the jar.
with open(os.path.join(topology_dir, MANIFEST), 'r') as f:
for fn in f.readlines():
# Ignore blank and comment lines.
fn = fn.strip()
if len(fn) and not fn.startswith('#'):
add_file_to_jar(jar, os.path.expandvars(fn.strip()))
# Add user and machine information to the jar.
add_to_jar(
jar, '__submitter__.yaml', '''
petrel.user: %s
petrel.host: %s
''' % (getpass.getuser(), socket.gethostname()))
# Also add the topology configuration to the jar.
with open(config, 'r') as f:
config_text = f.read()
add_to_jar(jar, '__topology__.yaml', config_text)
# Call module_name/function_name to populate a Thrift topology object.
builder = TopologyBuilder()
module_dir = os.path.abspath(topology_dir)
if module_dir not in sys.path:
sys.path[:0] = [module_dir]
added_path_entry = True
module = __import__(module_name)
getattr(module, function_name)(builder)
# Add the spout and bolt Python scripts to the jar. Create a
# setup_<script>.sh for each Python script.
# Add Python scripts and any other per-script resources.
for k, v in chain(builder._spouts.iteritems(),
builder._bolts.iteritems()):
# components inherited from petrel.storm does not have script property
# and should be added via manifest.txt
if hasattr(v, 'script'):
add_file_to_jar(jar, topology_dir, v.script)
# Create a bootstrap script.
if venv is not None:
# Allow overriding the execution command from the "petrel"
# command line. This is handy if the server already has a
# virtualenv set up with the necessary libraries.
v.execution_command = os.path.join(venv, 'bin/python')
# If a parallelism value was specified in the configuration YAML,
# override any setting provided in the topology definition script.
if k in parallelism:
builder._commons[k].parallelism_hint = int(parallelism.pop(k))
emitter = builder._spouts.get(k, None) or builder._bolts.get(
k, None)
instance_of_emitter_base = isinstance(emitter, EmitterBase)
if instance_of_emitter_base:
v.execution_command, v.script = \
intercept_petrel_emitter(venv, v.execution_command, \
os.path.splitext(v.script)[0], jar, pip_options, logdir)
else:
import base64, pickle, re
from operator import add
module_name = '%s' % emitter.__module__
class_name = '%s' % emitter.__class__.__name__
v.script = '%s@%s.%s' % (k, module_name, class_name)
v.script = re.sub('[\W]+', '_', k)
v.execution_command = EmitterBase.DEFAULT_PYTHON
emitter = base64.b64encode(pickle.dumps(emitter))
bootstrap = []
for bootstrap_cmpnt in builder._bootstrap:
bootstrap.append(bootstrap_cmpnt)
bootstrap = base64.b64encode(pickle.dumps(bootstrap))
v.execution_command, v.script = \
intercept_storm_emitter(venv, v.execution_command, v.script, \
jar, pip_options, logdir, \
module_name, class_name, emitter, bootstrap)
if len(parallelism):
raise ValueError(
'Parallelism settings error: There are no components named: %s'
% ','.join(parallelism.keys()))
# Build the Thrift topology object and serialize it to the .jar. Must do
# this *after* the intercept step above since that step may modify the
# topology definition.
io = StringIO()
topology = builder.write(io)
add_to_jar(jar, 'topology.ser', io.getvalue())
finally:
jar.close()
if added_path_entry:
# Undo our sys.path change.
sys.path[:] = sys.path[1:]
def run(out_dir,
config_fname,
data_paths_fname,
stats_list_fname,
split_fname=None,
check_if_file_exists=False,
verbose=True):
data_paths = util.read_yaml(data_paths_fname)
config = util.read_yaml(config_fname)
stats_key = config['stats_key']
outcome_stat_name = config['outcome_stat_name']
cohort_stat_name = config.get('cohort_stat_name', None)
lab_lower_bound = config.get('lab_lower_bound', None)
lab_upper_bound = config.get('lab_upper_bound', None)
gap_days = config.get('gap_days', None)
training_window_days = config['training_window_days']
buffer_window_days = config['buffer_window_days']
outcome_window_days = config['outcome_window_days']
time_period_days = config['time_period_days']
time_scale_days = config['time_scale_days']
use_just_labs = config['use_just_labs']
feature_loincs_fname = config['feature_loincs_fname']
add_age_sex = config['add_age_sex']
calc_gfr = config['calc_gfr']
regularizations = config.get('regularizations', [1])
lin_n_cv_iters = config.get('lin_n_cv_iters', -1)
n_cv_iters = config.get('n_cv_iters', -1)
progression = config['progression']
progression_lab_lower_bound = config.get('progression_lab_lower_bound',
None)
progression_lab_upper_bound = config.get('progression_lab_upper_bound',
None)
progression_gap_days = config.get('progression_gap_days', None)
progression_stages = config.get('progression_stages', None)
progression_init_stages = config.get('progression_init_stages', None)
evaluate_nn = config.get('evaluate_nn', True)
outcome_fname = out_dir + stats_key + '_' + outcome_stat_name + '.txt'
if cohort_stat_name is None:
cohort_fname = data_paths['demographics_fname']
else:
cohort_fname = out_dir + stats_key + '_' + cohort_stat_name + '.txt'
gfr_loincs = util.read_list_files('data/gfr_loincs.txt')
training_data_fname = out_dir + stats_key + '_training_data.txt'
feature_loincs = util.read_list_files(feature_loincs_fname)
if use_just_labs == False:
feature_diseases = [[
icd9
] for icd9 in util.read_list_files('data/kidney_disease_mi_icd9s.txt')]
feature_drugs = [
util.read_list_files('data/drug_class_' + dc.lower().replace(
'-', '_').replace(',', '_').replace(' ', '_') + '_ndcs.txt')
for dc in util.read_list_files(
'data/kidney_disease_drug_classes.txt')
]
else:
feature_diseases = []
feature_drugs = []
n_labs = len(feature_loincs)
if add_age_sex:
age_index = len(feature_loincs) + len(feature_diseases) + len(
feature_drugs)
gender_index = len(feature_loincs) + len(feature_diseases) + len(
feature_drugs) + 1
else:
age_index = None
gender_index = None
features_fname = out_dir + stats_key + '_features.h5'
features_split_fname = out_dir + stats_key + '_features_split.h5'
predict_fname = out_dir + stats_key + '_prediction_results.yaml'
if evaluate_nn:
nn_predict_fname = out_dir + stats_key + '_nn_prediction_results.yaml'
else:
nn_predict_fname = None
if verbose:
print "Loading data"
db = util.Database(data_paths_fname)
db.load_people()
db.load_db(['loinc', 'loinc_vals', 'cpt', 'icd9_proc', 'icd9', 'ndc'])
stats = util.read_yaml(stats_list_fname)[stats_key]
if verbose:
print "Calculating patient stats"
data = ps.patient_stats(db,
stats,
stats_key,
out_dir,
stat_indices=None,
verbose=verbose,
check_if_file_exists=check_if_file_exists,
save_files=True)
if verbose:
print "Building training data"
outcome_data = btd.build_outcome_data(out_dir, outcome_fname)
cohort_data = btd.setup(data_paths['demographics_fname'], outcome_fname,
cohort_fname)
# calc_gfr = True here because it's required to define the condition
training_data = btd.build_training_data(db, cohort_data, gfr_loincs, lab_lower_bound, lab_upper_bound, \
training_window_days, buffer_window_days, outcome_window_days, time_period_days, time_scale_days, gap_days, calc_gfr=True, verbose=verbose, \
progression=progression, progression_lab_lower_bound=progression_lab_lower_bound, progression_lab_upper_bound=progression_lab_upper_bound, \
progression_gap_days=progression_gap_days, progression_init_stages=progression_init_stages, progression_stages=progression_stages)
training_data.to_csv(training_data_fname, index=False, sep='\t')
if verbose:
print "Building features"
features.features(db, training_data, feature_loincs, feature_diseases,
feature_drugs, time_scale_days, features_fname, calc_gfr,
verbose, add_age_sex)
if split_fname is None:
split_fname = out_dir + stats_key + '_split.txt'
features.train_validation_test_split(training_data['person'].unique(),
split_fname,
verbose=verbose)
features.split(features_fname, features_split_fname, split_fname, verbose)
if verbose:
print "Training, validating and testing models"
predict.predict(features_split_fname, lin_n_cv_iters, n_cv_iters,
regularizations, n_labs, age_index, gender_index,
predict_fname, nn_predict_fname)
def build_contest_pdf(contest, problems, solutions=False, web=False):
builddir = config.tmpdir / contest
builddir.mkdir(parents=True, exist_ok=True)
build_type = 'solution' if solutions else 'problem'
main_file = 'solutions' if solutions else 'contest'
main_file += '-web.tex' if web else '.tex'
if solutions:
ensure_symlink(builddir / 'solutions-base.tex',
config.tools_root / 'latex/solutions-base.tex')
ensure_symlink(builddir / 'bapc.cls', config.tools_root / 'latex/bapc.cls')
ensure_symlink(builddir / 'images', config.tools_root / 'latex/images')
ensure_symlink(builddir / main_file,
config.tools_root / 'latex' / main_file)
config_data = util.read_yaml(Path('contest.yaml'))
config_data['testsession'] = '\\testsession' if config_data.get(
'testsession') else ''
util.copy_and_substitute(config.tools_root / 'latex/contest-data.tex',
builddir / 'contest_data.tex', config_data)
ensure_symlink(builddir / 'logo.pdf', find_logo())
problems_data = ''
# Link the solve stats directory if it exists.
solve_stats = Path('solve_stats')
if solve_stats.exists():
ensure_symlink(builddir / 'solve_stats', solve_stats)
# include a header slide in the solutions PDF
headertex = Path('solution_header.tex')
if headertex.exists():
ensure_symlink(builddir / 'solution_header.tex', headertex)
if solutions and headertex.exists():
problems_data += f'\\input{{{headertex}}}\n'
per_problem_data = (config.tools_root / 'latex' /
f'contest-{build_type}.tex').read_text()
# Some logic to prevent duplicate problem IDs.
for problem in problems:
prepare_problem(problem)
id_ok = True
problems_data += util.substitute(
per_problem_data, {
'problemlabel': problem.label,
'problemyamlname': problem.config['name'],
'problemauthor': problem.config.get('author'),
'timelimit': get_tl(problem.config),
'problemdir': config.tmpdir / problem.id,
})
# include a statistics slide in the solutions PDF
footer_tex = Path('solution_footer.tex')
if footer_tex.exists():
ensure_symlink(builddir / 'solution_footer.tex', footer_tex)
if solutions and footer_tex.exists():
problems_data += f'\\input{{{footer_tex}}}\n'
(builddir / f'contest-{build_type}s.tex').write_text(problems_data)
for i in range(3):
ok, err, out = util.exec_command(PDFLATEX + [
'-output-directory', builddir,
(builddir / main_file).with_suffix('.tex')
],
0,
False,
cwd=builddir,
stdout=subprocess.PIPE)
if ok is not True:
print(f'{_c.red}Failure compiling pdf:{_c.reset}\n{out}')
return False
# link the output pdf
output_pdf = Path(main_file).with_suffix('.pdf')
ensure_symlink(output_pdf, builddir / output_pdf, True)
print(f'{_c.green}Pdf written to {output_pdf}{_c.reset}')
return True
def build_jar(source_jar_path, dest_jar_path, config, venv=None, definition=None):
"""Build a StormTopology .jar which encapsulates the topology defined in
topology_dir. Optionally override the module and function names. This
feature supports the definition of multiple topologies in a single
directory."""
if definition is None:
definition = 'create.create'
# Prepare data we'll use later for configuring parallelism.
config_yaml = read_yaml(config)
parallelism = dict((k.split('.')[-1], v) for k, v in config_yaml.iteritems()
if k.startswith('petrel.parallelism'))
pip_options = config_yaml.get('petrel.pip_options', '')
module_name, dummy, function_name = definition.rpartition('.')
topology_dir = os.getcwd()
# Make a copy of the input "jvmpetrel" jar. This jar acts as a generic
# starting point for all Petrel topologies.
source_jar_path = os.path.abspath(source_jar_path)
dest_jar_path = os.path.abspath(dest_jar_path)
if source_jar_path == dest_jar_path:
raise ValueError("Error: Destination and source path are the same.")
shutil.copy(source_jar_path, dest_jar_path)
jar = zipfile.ZipFile(dest_jar_path, 'a', compression=zipfile.ZIP_DEFLATED)
added_path_entry = False
try:
# Add the files listed in manifest.txt to the jar.
with open(os.path.join(topology_dir, MANIFEST), 'r') as f:
for fn in f.readlines():
# Ignore blank and comment lines.
fn = fn.strip()
if len(fn) and not fn.startswith('#'):
add_file_to_jar(jar, fn.strip())
# Add user and machine information to the jar.
add_to_jar(jar, '__submitter__.yaml', '''
petrel.user: %s
petrel.host: %s
''' % (getpass.getuser(),socket.gethostname()))
# Also add the topology configuration to the jar.
with open(config, 'r') as f:
config_text = f.read()
add_to_jar(jar, '__topology__.yaml', config_text)
# Call module_name/function_name to populate a Thrift topology object.
builder = TopologyBuilder()
module_dir = os.path.abspath(topology_dir)
if module_dir not in sys.path:
sys.path[:0] = [ module_dir ]
added_path_entry = True
module = __import__(module_name)
getattr(module, function_name)(builder)
# Add the spout and bolt Python scripts to the jar. Create a
# setup_<script>.sh for each Python script.
# Add Python scripts and any other per-script resources.
for k, v in chain(builder._spouts.iteritems(), builder._bolts.iteritems()):
add_file_to_jar(jar, topology_dir, v.script)
# Create a bootstrap script.
if venv is not None:
# Allow overriding the execution command from the "petrel"
# command line. This is handy if the server already has a
# virtualenv set up with the necessary libraries.
v.execution_command = os.path.join(venv, 'bin/python')
# If a parallelism value was specified in the configuration YAML,
# override any setting provided in the topology definition script.
if k in parallelism:
builder._commons[k].parallelism_hint = int(parallelism.pop(k))
v.execution_command, v.script = \
intercept(venv, v.execution_command, os.path.splitext(v.script)[0],
jar, pip_options)
if len(parallelism):
raise ValueError(
'Parallelism settings error: There are no components named: %s' %
','.join(parallelism.keys()))
# Build the Thrift topology object and serialize it to the .jar. Must do
# this *after* the intercept step above since that step may modify the
# topology definition.
io = StringIO()
topology = builder.write(io)
add_to_jar(jar, 'topology.ser', io.getvalue())
finally:
jar.close()
if added_path_entry:
# Undo our sys.path change.
sys.path[:] = sys.path[1:]
import data
import neural_network
import support_vector_machine
import predict
import test_accuracy
import util
if __name__ == "__main__":
config = util.read_yaml()
all_students_data_path = config["all_students_data_path"]
target_cap = config["target_cap"]
# Build Neural Network
print("---------------------------------------------------------")
print("Generating Neural Network:\n")
try:
model_NN = neural_network.create_model()
except:
model_NN = neural_network.retrieve_model()
print("\n")
# Build Support Vector Machine
print("---------------------------------------------------------")
print("Generating Support Vector Machine:\n")
try:
model_SVM = support_vector_machine.create_model()
except:
model_SVM = support_vector_machine.retrieve_model()
print("\n")
# Evaluate Neural Network
def build_contest_pdf(contest, problems, tmpdir, solutions=False, web=False):
builddir = tmpdir / contest
builddir.mkdir(parents=True, exist_ok=True)
build_type = 'solution' if solutions else 'problem'
main_file = 'solutions' if solutions else 'contest'
main_file += '-web.tex' if web else '.tex'
config_data = util.read_yaml(Path('contest.yaml'))
config_data['testsession'] = '\\testsession' if config_data.get('testsession') else ''
config_data['logofile'] = find_logo().as_posix()
util.copy_and_substitute(config.tools_root / 'latex/contest-data.tex',
builddir / 'contest_data.tex', config_data)
problems_data = ''
if solutions:
# Link the solve stats directory if it exists.
solve_stats = Path('solve_stats')
# include a header slide in the solutions PDF
headertex = Path('solution_header.tex')
if headertex.exists():
problems_data += f'\\input{{{headertex}}}\n'
per_problem_data = (config.tools_root / 'latex' / f'contest-{build_type}.tex').read_text()
# Some logic to prevent duplicate problem IDs.
for problem in problems:
if build_type == 'problem':
prepare_problem(problem)
problems_data += util.substitute(
per_problem_data, {
'problemlabel': problem.label,
'problemyamlname': problem.settings.name.replace('_', ' '),
'problemauthor': problem.settings.author,
'timelimit': problem.settings.timelimit,
'problemdir': problem.path.absolute().as_posix(),
'builddir': problem.tmpdir.as_posix(),
})
if solutions:
# include a statistics slide in the solutions PDF
footer_tex = Path('solution_footer.tex')
if footer_tex.exists():
problems_data += f'\\input{{{footer_tex}}}\n'
(builddir / f'contest-{build_type}s.tex').write_text(problems_data)
env = get_environment()
for i in range(3):
ret = util.exec_command(
PDFLATEX + ['-output-directory', builddir,
config.tools_root / 'latex' / main_file],
0,
False,
cwd=builddir,
stdout=subprocess.PIPE,
env=env,
)
if ret.ok is not True:
print(f'{Fore.RED}Failure compiling pdf:{Style.RESET_ALL}\n{ret.out}')
return False
# link the output pdf
output_pdf = Path(main_file).with_suffix('.pdf')
ensure_symlink(output_pdf, builddir / output_pdf, True)
print(f'{Fore.GREEN}Pdf written to {output_pdf}{Style.RESET_ALL}')
return True
