def _update_cmd_config_and_execute(cmd: List[str]):
env = _os.environ.copy()
if _aws_config.ENABLE_DEBUG.get():
cmd.insert(1, "--debug")
if _aws_config.S3_ENDPOINT.get() is not None:
cmd.insert(1, _aws_config.S3_ENDPOINT.get())
cmd.insert(1, _aws_config.S3_ENDPOINT_ARG_NAME)
if _aws_config.S3_ACCESS_KEY_ID.get() is not None:
env[_aws_config.
S3_ACCESS_KEY_ID_ENV_NAME] = _aws_config.S3_ACCESS_KEY_ID.get()
if _aws_config.S3_SECRET_ACCESS_KEY.get() is not None:
env[_aws_config.
S3_SECRET_ACCESS_KEY_ENV_NAME] = _aws_config.S3_SECRET_ACCESS_KEY.get(
)
retry = 0
try:
return _subprocess.check_call(cmd, env=env)
except Exception as e:
logging.error("Exception when trying to execute {}, reason: {}", cmd,
str(e))
retry += 1
if retry > _aws_config.RETRIES:
raise
secs = _aws_config.BACKOFF_SECONDS
logging.info(
"Sleeping before retrying again, after {} seconds".format(secs))
time.sleep(secs)
logging.info("Retrying again")
def _update_cmd_config_and_execute(s3_cfg: S3Config, cmd: List[str]):
env = _os.environ.copy()
if s3_cfg.enable_debug:
cmd.insert(1, "--debug")
if s3_cfg.endpoint is not None:
cmd.insert(1, s3_cfg.endpoint)
cmd.insert(1, "--endpoint-url")
if S3_ACCESS_KEY_ID_ENV_NAME not in os.environ:
if s3_cfg.access_key_id:
env[S3_ACCESS_KEY_ID_ENV_NAME] = s3_cfg.access_key_id
if S3_SECRET_ACCESS_KEY_ENV_NAME not in os.environ:
if s3_cfg.secret_access_key:
env[S3_SECRET_ACCESS_KEY_ENV_NAME] = s3_cfg.secret_access_key
retry = 0
while True:
try:
try:
return subprocess.check_call(cmd, env=env)
except Exception as e:
if retry > 0:
logger.info(
f"AWS command failed with error {e}, command: {cmd}, retry {retry}"
)
logger.debug(
f"Appending anonymous flag and retrying command {cmd}")
anonymous_cmd = cmd[:] # strings only, so this is deep enough
anonymous_cmd.insert(1, S3_ANONYMOUS_FLAG)
return subprocess.check_call(anonymous_cmd, env=env)
except Exception as e:
logger.error(
f"Exception when trying to execute {cmd}, reason: {str(e)}")
retry += 1
if retry > s3_cfg.retries:
raise
secs = s3_cfg.backoff
logger.info(
f"Sleeping before retrying again, after {secs.total_seconds()} seconds"
)
time.sleep(secs.total_seconds())
logger.info("Retrying again")
def test_check_call(mock_call):
mock_call.return_value = _MockProcess()
op = subprocess.check_call(["ls", "-l"], shell=True, env={"a": "b"}, cwd="/tmp")
assert op == 0
mock_call.assert_called()
assert mock_call.call_args[0][0] == ["ls", "-l"]
assert mock_call.call_args[1]["shell"] is True
assert mock_call.call_args[1]["env"] == {"a": "b"}
assert mock_call.call_args[1]["cwd"] == "/tmp"
def test_check_call_shellex(mock_call):
mock_call.return_value = _MockProcess()
op = subprocess.check_call("ls -l", shell=True, env={'a': 'b'}, cwd="/tmp")
assert op == 0
assert op == 0
mock_call.assert_called()
assert mock_call.call_args[0][0] == ["ls", "-l"]
assert mock_call.call_args[1]['shell'] is True
assert mock_call.call_args[1]['env'] == {'a': 'b'}
assert mock_call.call_args[1]['cwd'] == "/tmp"
def _update_cmd_config_and_execute(cmd):
env = _os.environ.copy()
if _aws_config.S3_ENDPOINT.get() is not None:
cmd.insert(1, _aws_config.S3_ENDPOINT.get())
cmd.insert(1, _aws_config.S3_ENDPOINT_ARG_NAME)
if _aws_config.S3_ACCESS_KEY_ID.get() is not None:
env[_aws_config.
S3_ACCESS_KEY_ID_ENV_NAME] = _aws_config.S3_ACCESS_KEY_ID.get()
if _aws_config.S3_SECRET_ACCESS_KEY.get() is not None:
env[_aws_config.
S3_SECRET_ACCESS_KEY_ENV_NAME] = _aws_config.S3_SECRET_ACCESS_KEY.get(
)
return _subprocess.check_call(cmd, env=env)
def _update_cmd_config_and_execute(cmd):
env = _os.environ.copy()
return _subprocess.check_call(cmd, env=env)
