def start(project_folder, user_values, workers):
freeze_support()
input_params = update_params(_recipe_default_inputs, user_values)
if 'simulation_folder' not in input_params or not input_params[
'simulation_folder']:
if 'simulation_id' not in input_params or not input_params[
'simulation_id']:
simulation_id = 'daylight_factor_%d' % int(
round(time.time(), 2) * 100)
else:
simulation_id = input_params['simulation_id']
simulation_folder = pathlib.Path(project_folder,
simulation_id).as_posix()
input_params['simulation_folder'] = simulation_folder
else:
simulation_folder = input_params['simulation_folder']
# copy project folder content to simulation folder
artifacts = ['model']
optional_artifacts = []
for artifact in artifacts:
value = input_params[artifact]
if value is None:
if artifact in optional_artifacts:
continue
raise ValueError('None value for required artifact input: %s' %
artifact)
from_ = pathlib.Path(project_folder,
input_params[artifact]).resolve().as_posix()
to_ = pathlib.Path(simulation_folder,
input_params[artifact]).resolve().as_posix()
_copy_artifacts(from_, to_)
# set up logs
log_folder = pathlib.Path(simulation_folder, '__logs__')
log_folder.mkdir(exist_ok=True)
cfg_file = pathlib.Path(simulation_folder, '__logs__', 'logs.cfg')
log_file = pathlib.Path(simulation_folder, '__logs__',
'logs.log').as_posix()
with cfg_file.open('w') as lf:
lf.write(LOGS_CONFIG.replace('WORKFLOW.LOG', log_file))
luigi.build([LetDaylightFactorFly(_input_params=input_params)],
local_scheduler=local_scheduler(),
workers=workers,
logging_conf_file=cfg_file.as_posix())
def _try(self, inputs: Dict[str, Any], folder: str = None) -> str:
"""Try running a function locally for testing.
This method does not return the output values. See the folder to validate the
outputs.
Args:
inputs: A dictionary that maps input names to values
(e.g. {'input_one': 5, ...}).
folder: An optional folder to run the function. A temporary folder
will be created if this folder is not provided.
Returns:
str -- path to run_folder.
"""
func = self.queenbee
# check all the required inputs are provided
for inp in func.inputs:
name = inp.name.replace('-', '_')
if inp.required:
assert name in inputs, f'Required input "{name}" is missing from inputs.'
continue
# see if default value should be used
if name not in inputs:
inputs[name] = inp.default
dst = folder or tempfile.TemporaryDirectory().name
command = ' '.join(func.command.split())
refs = parse_double_quotes_vars(command)
command = command.replace('{{', '{').replace('}}', '}')
for ref in refs:
assert ref.startswith('inputs.'), \
'All referenced values must start with {{inputs followed with' \
f' variable name. Invalid referenced value: {ref}'
var = ref.replace('inputs.', '').replace('-', '_')
command = command.replace('{%s}' % ref, str(inputs[var]))
for art in func.artifact_inputs:
print(f"copying input artifact: {art.name}...")
name = art.name.replace('-', '_')
_copy_artifacts(inputs[name], os.path.join(dst, art.path))
cur_dir = os.getcwd()
os.chdir(dst)
print(f'command: {command}')
p = subprocess.Popen(command,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
shell=True,
env=os.environ)
stdout, stderr = p.communicate()
if p.returncode != 0 and stderr != b'':
raise RuntimeError(stderr.decode('utf-8'))
if stderr.decode('utf-8'):
print(stderr.decode('utf-8'))
if stdout.decode('utf-8'):
print(stdout.decode('utf-8'))
# change back to initial directory
os.chdir(cur_dir)
return dst