def start(project_folder, user_values, workers): freeze_support() input_params = update_params(_recipe_default_inputs, user_values) if 'simulation_folder' not in input_params or not input_params[ 'simulation_folder']: if 'simulation_id' not in input_params or not input_params[ 'simulation_id']: simulation_id = 'daylight_factor_%d' % int( round(time.time(), 2) * 100) else: simulation_id = input_params['simulation_id'] simulation_folder = pathlib.Path(project_folder, simulation_id).as_posix() input_params['simulation_folder'] = simulation_folder else: simulation_folder = input_params['simulation_folder'] # copy project folder content to simulation folder artifacts = ['model'] optional_artifacts = [] for artifact in artifacts: value = input_params[artifact] if value is None: if artifact in optional_artifacts: continue raise ValueError('None value for required artifact input: %s' % artifact) from_ = pathlib.Path(project_folder, input_params[artifact]).resolve().as_posix() to_ = pathlib.Path(simulation_folder, input_params[artifact]).resolve().as_posix() _copy_artifacts(from_, to_) # set up logs log_folder = pathlib.Path(simulation_folder, '__logs__') log_folder.mkdir(exist_ok=True) cfg_file = pathlib.Path(simulation_folder, '__logs__', 'logs.cfg') log_file = pathlib.Path(simulation_folder, '__logs__', 'logs.log').as_posix() with cfg_file.open('w') as lf: lf.write(LOGS_CONFIG.replace('WORKFLOW.LOG', log_file)) luigi.build([LetDaylightFactorFly(_input_params=input_params)], local_scheduler=local_scheduler(), workers=workers, logging_conf_file=cfg_file.as_posix())
def _try(self, inputs: Dict[str, Any], folder: str = None) -> str: """Try running a function locally for testing. This method does not return the output values. See the folder to validate the outputs. Args: inputs: A dictionary that maps input names to values (e.g. {'input_one': 5, ...}). folder: An optional folder to run the function. A temporary folder will be created if this folder is not provided. Returns: str -- path to run_folder. """ func = self.queenbee # check all the required inputs are provided for inp in func.inputs: name = inp.name.replace('-', '_') if inp.required: assert name in inputs, f'Required input "{name}" is missing from inputs.' continue # see if default value should be used if name not in inputs: inputs[name] = inp.default dst = folder or tempfile.TemporaryDirectory().name command = ' '.join(func.command.split()) refs = parse_double_quotes_vars(command) command = command.replace('{{', '{').replace('}}', '}') for ref in refs: assert ref.startswith('inputs.'), \ 'All referenced values must start with {{inputs followed with' \ f' variable name. Invalid referenced value: {ref}' var = ref.replace('inputs.', '').replace('-', '_') command = command.replace('{%s}' % ref, str(inputs[var])) for art in func.artifact_inputs: print(f"copying input artifact: {art.name}...") name = art.name.replace('-', '_') _copy_artifacts(inputs[name], os.path.join(dst, art.path)) cur_dir = os.getcwd() os.chdir(dst) print(f'command: {command}') p = subprocess.Popen(command, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, env=os.environ) stdout, stderr = p.communicate() if p.returncode != 0 and stderr != b'': raise RuntimeError(stderr.decode('utf-8')) if stderr.decode('utf-8'): print(stderr.decode('utf-8')) if stdout.decode('utf-8'): print(stdout.decode('utf-8')) # change back to initial directory os.chdir(cur_dir) return dst