def collect_releases(context):
ocds_loader: OCDSLoader = context.resources.ocds_loader
last_loading = ocds_loader.get_last_data_loading_records(
n_records=1).iloc[0]
app_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.join(app_dir, 'shell', 'kingfisher-collect.sh')
from_date = context.solid_config.get('from_date')
last_item_date = last_loading.last_item_date
start_date = from_date or last_item_date.strftime('%Y-%m-%d')
shell_command = 'bash {} {}'.format(script_path, start_date)
output, return_code = execute(shell_command=shell_command,
log=context.log,
**without_keys(context.solid_config,
['from_date']))
if return_code:
raise Failure(
description='Shell command execution failed with output: {output}'.
format(output=output))
return output
def test_execute_inline(tmp_file):
with tmp_file("some file contents") as (tmp_path, tmp_file):
res, retcode = execute("ls",
cwd=tmp_path,
output_logging="BUFFER",
log=logging)
assert os.path.basename(tmp_file) in res
assert retcode == 0
def test_execute_inline(tmp_file):
with tmp_file('some file contents') as (tmp_path, tmp_file):
res, retcode = execute('ls',
cwd=tmp_path,
output_logging='BUFFER',
log=logging)
assert os.path.basename(tmp_file) in res
assert retcode == 0
def test_output_logging_stream(caplog):
caplog.set_level(logging.INFO)
_, retcode = execute("ls", output_logging="STREAM", log=logging)
log_messages = [r.message for r in caplog.records]
assert log_messages[0].startswith("Using temporary directory: ")
assert log_messages[1].startswith("Temporary script location: ")
assert log_messages[2] == "Running command:\nls"
assert log_messages[3].startswith("Command pid:")
assert log_messages[4]
assert retcode == 0
caplog.clear()
_, retcode = execute("ls", output_logging="STREAM", log=logging)
log_messages = [r.message for r in caplog.records]
assert log_messages[0].startswith("Using temporary directory: ")
assert log_messages[1].startswith("Temporary script location: ")
assert log_messages[2] == "Running command:\nls"
assert log_messages[3].startswith("Command pid:")
assert log_messages[4]
assert retcode == 0
caplog.clear()
_, retcode = execute(
'for i in 1 2 3; do echo "iter $i"; done;',
output_logging="STREAM",
log=logging,
)
log_messages = [r.message for r in caplog.records]
assert retcode == 0
assert log_messages[4:7] == ["iter 1", "iter 2", "iter 3"]
caplog.clear()
_, retcode = execute(
'for i in 1 2 3; do echo "iter $i"; done;',
output_logging="BUFFER",
log=logging,
)
log_messages = [r.message for r in caplog.records]
assert retcode == 0
assert log_messages[4] == "iter 1\niter 2\niter 3\n"
def test_output_logging_stream(caplog):
caplog.set_level(logging.INFO)
_, retcode = execute('ls', output_logging='STREAM', log=logging)
log_messages = [r.message for r in caplog.records]
assert log_messages[0].startswith('Using temporary directory: ')
assert log_messages[1].startswith('Temporary script location: ')
assert log_messages[2] == 'Running command:\nls'
assert log_messages[3]
assert retcode == 0
caplog.clear()
_, retcode = execute('ls', output_logging='STREAM', log=logging)
log_messages = [r.message for r in caplog.records]
assert log_messages[0].startswith('Using temporary directory: ')
assert log_messages[1].startswith('Temporary script location: ')
assert log_messages[2] == 'Running command:\nls'
assert log_messages[3]
assert retcode == 0
caplog.clear()
_, retcode = execute(
'for i in 1 2 3; do echo "iter $i"; done;',
output_logging='STREAM',
log=logging,
)
log_messages = [r.message for r in caplog.records]
assert retcode == 0
assert log_messages[3:6] == ['iter 1', 'iter 2', 'iter 3']
caplog.clear()
_, retcode = execute(
'for i in 1 2 3; do echo "iter $i"; done;',
output_logging='BUFFER',
log=logging,
)
log_messages = [r.message for r in caplog.records]
assert retcode == 0
assert log_messages[3] == 'iter 1\niter 2\niter 3\n'
def test_env(tmp_file):
cmd = "echo $TEST_VAR"
res, retcode = execute(
cmd, output_logging="BUFFER", log=logging, env={"TEST_VAR": "some_env_value"}
)
assert res.strip() == "some_env_value"
assert retcode == 0
with tmp_file(cmd) as (_, tmp_file):
res, retcode = execute_script_file(
tmp_file,
output_logging="BUFFER",
log=logging,
env={"TEST_VAR": "some_env_value"},
)
assert res.strip() == "some_env_value"
assert retcode == 0
def test_env(tmp_file):
cmd = 'echo $TEST_VAR'
res, retcode = execute(cmd,
output_logging='BUFFER',
log=logging,
env={'TEST_VAR': 'some_env_value'})
assert res.strip() == 'some_env_value'
assert retcode == 0
with tmp_file(cmd) as (_, tmp_file):
res, retcode = execute_script_file(
tmp_file,
output_logging='BUFFER',
log=logging,
env={'TEST_VAR': 'some_env_value'},
)
assert res.strip() == 'some_env_value'
assert retcode == 0
def refresh_views(context):
app_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.join(app_dir, 'shell', 'kingfisher-views.sh')
view_name = context.solid_config.get('view_name')
shell_command = 'bash {} {}'.format(script_path, view_name)
output, return_code = execute(shell_command=shell_command,
log=context.log,
**without_keys(context.solid_config,
['view_name']))
if return_code:
raise Failure(
description='Shell command execution failed with output: {output}'.
format(output=output))
return output
def process_releases(context):
app_dir = os.path.dirname(os.path.abspath(__file__))
script_path = os.path.join(app_dir, 'shell', 'kingfisher-process.sh')
collection_id = context.solid_config.get('collection_id')
shell_command = 'bash {} {}'.format(script_path, collection_id)
output, return_code = execute(shell_command=shell_command,
log=context.log,
**without_keys(context.solid_config,
['collection_id']))
if return_code:
raise Failure(
description='Shell command execution failed with output: {output}'.
format(output=output))
return output
def test_bad_output_logging():
with pytest.raises(Exception, match="Unrecognized output_logging NOT_A_VALID_LOGGING_VALUE"):
execute("ls", output_logging="NOT_A_VALID_LOGGING_VALUE", log=logging)
def sleepy_solid(context):
# execute a sleep in the background
execute("sleep 60", "NONE", context.log)
def test_bad_output_logging():
with pytest.raises(
Exception,
match='Unrecognized output_logging NOT_A_VALID_LOGGING_VALUE'):
execute('ls', output_logging='NOT_A_VALID_LOGGING_VALUE', log=logging)
