def collect_releases(context): ocds_loader: OCDSLoader = context.resources.ocds_loader last_loading = ocds_loader.get_last_data_loading_records( n_records=1).iloc[0] app_dir = os.path.dirname(os.path.abspath(__file__)) script_path = os.path.join(app_dir, 'shell', 'kingfisher-collect.sh') from_date = context.solid_config.get('from_date') last_item_date = last_loading.last_item_date start_date = from_date or last_item_date.strftime('%Y-%m-%d') shell_command = 'bash {} {}'.format(script_path, start_date) output, return_code = execute(shell_command=shell_command, log=context.log, **without_keys(context.solid_config, ['from_date'])) if return_code: raise Failure( description='Shell command execution failed with output: {output}'. format(output=output)) return output
def test_execute_inline(tmp_file): with tmp_file("some file contents") as (tmp_path, tmp_file): res, retcode = execute("ls", cwd=tmp_path, output_logging="BUFFER", log=logging) assert os.path.basename(tmp_file) in res assert retcode == 0
def test_execute_inline(tmp_file): with tmp_file('some file contents') as (tmp_path, tmp_file): res, retcode = execute('ls', cwd=tmp_path, output_logging='BUFFER', log=logging) assert os.path.basename(tmp_file) in res assert retcode == 0
def test_output_logging_stream(caplog): caplog.set_level(logging.INFO) _, retcode = execute("ls", output_logging="STREAM", log=logging) log_messages = [r.message for r in caplog.records] assert log_messages[0].startswith("Using temporary directory: ") assert log_messages[1].startswith("Temporary script location: ") assert log_messages[2] == "Running command:\nls" assert log_messages[3].startswith("Command pid:") assert log_messages[4] assert retcode == 0 caplog.clear() _, retcode = execute("ls", output_logging="STREAM", log=logging) log_messages = [r.message for r in caplog.records] assert log_messages[0].startswith("Using temporary directory: ") assert log_messages[1].startswith("Temporary script location: ") assert log_messages[2] == "Running command:\nls" assert log_messages[3].startswith("Command pid:") assert log_messages[4] assert retcode == 0 caplog.clear() _, retcode = execute( 'for i in 1 2 3; do echo "iter $i"; done;', output_logging="STREAM", log=logging, ) log_messages = [r.message for r in caplog.records] assert retcode == 0 assert log_messages[4:7] == ["iter 1", "iter 2", "iter 3"] caplog.clear() _, retcode = execute( 'for i in 1 2 3; do echo "iter $i"; done;', output_logging="BUFFER", log=logging, ) log_messages = [r.message for r in caplog.records] assert retcode == 0 assert log_messages[4] == "iter 1\niter 2\niter 3\n"
def test_output_logging_stream(caplog): caplog.set_level(logging.INFO) _, retcode = execute('ls', output_logging='STREAM', log=logging) log_messages = [r.message for r in caplog.records] assert log_messages[0].startswith('Using temporary directory: ') assert log_messages[1].startswith('Temporary script location: ') assert log_messages[2] == 'Running command:\nls' assert log_messages[3] assert retcode == 0 caplog.clear() _, retcode = execute('ls', output_logging='STREAM', log=logging) log_messages = [r.message for r in caplog.records] assert log_messages[0].startswith('Using temporary directory: ') assert log_messages[1].startswith('Temporary script location: ') assert log_messages[2] == 'Running command:\nls' assert log_messages[3] assert retcode == 0 caplog.clear() _, retcode = execute( 'for i in 1 2 3; do echo "iter $i"; done;', output_logging='STREAM', log=logging, ) log_messages = [r.message for r in caplog.records] assert retcode == 0 assert log_messages[3:6] == ['iter 1', 'iter 2', 'iter 3'] caplog.clear() _, retcode = execute( 'for i in 1 2 3; do echo "iter $i"; done;', output_logging='BUFFER', log=logging, ) log_messages = [r.message for r in caplog.records] assert retcode == 0 assert log_messages[3] == 'iter 1\niter 2\niter 3\n'
def test_env(tmp_file): cmd = "echo $TEST_VAR" res, retcode = execute( cmd, output_logging="BUFFER", log=logging, env={"TEST_VAR": "some_env_value"} ) assert res.strip() == "some_env_value" assert retcode == 0 with tmp_file(cmd) as (_, tmp_file): res, retcode = execute_script_file( tmp_file, output_logging="BUFFER", log=logging, env={"TEST_VAR": "some_env_value"}, ) assert res.strip() == "some_env_value" assert retcode == 0
def test_env(tmp_file): cmd = 'echo $TEST_VAR' res, retcode = execute(cmd, output_logging='BUFFER', log=logging, env={'TEST_VAR': 'some_env_value'}) assert res.strip() == 'some_env_value' assert retcode == 0 with tmp_file(cmd) as (_, tmp_file): res, retcode = execute_script_file( tmp_file, output_logging='BUFFER', log=logging, env={'TEST_VAR': 'some_env_value'}, ) assert res.strip() == 'some_env_value' assert retcode == 0
def refresh_views(context): app_dir = os.path.dirname(os.path.abspath(__file__)) script_path = os.path.join(app_dir, 'shell', 'kingfisher-views.sh') view_name = context.solid_config.get('view_name') shell_command = 'bash {} {}'.format(script_path, view_name) output, return_code = execute(shell_command=shell_command, log=context.log, **without_keys(context.solid_config, ['view_name'])) if return_code: raise Failure( description='Shell command execution failed with output: {output}'. format(output=output)) return output
def process_releases(context): app_dir = os.path.dirname(os.path.abspath(__file__)) script_path = os.path.join(app_dir, 'shell', 'kingfisher-process.sh') collection_id = context.solid_config.get('collection_id') shell_command = 'bash {} {}'.format(script_path, collection_id) output, return_code = execute(shell_command=shell_command, log=context.log, **without_keys(context.solid_config, ['collection_id'])) if return_code: raise Failure( description='Shell command execution failed with output: {output}'. format(output=output)) return output
def test_bad_output_logging(): with pytest.raises(Exception, match="Unrecognized output_logging NOT_A_VALID_LOGGING_VALUE"): execute("ls", output_logging="NOT_A_VALID_LOGGING_VALUE", log=logging)
def sleepy_solid(context): # execute a sleep in the background execute("sleep 60", "NONE", context.log)
def test_bad_output_logging(): with pytest.raises( Exception, match='Unrecognized output_logging NOT_A_VALID_LOGGING_VALUE'): execute('ls', output_logging='NOT_A_VALID_LOGGING_VALUE', log=logging)