def upsert(self, image_request, temp_fp, image_info):
target_fp = self.create_dir_and_return_file_path(
image_request=image_request,
image_info=image_info
)
safe_rename(temp_fp, target_fp)
return target_fp
def test_renames_file_correctly(self, src, dst):
assert os.path.exists(src)
assert not os.path.exists(dst)
utils.safe_rename(src, dst)
assert not os.path.exists(src)
assert os.path.exists(dst)
assert open(dst, 'rb').read() == b'hello world'
def test_if_error_is_unexpected_then_is_raised(self, src, dst):
"""
If the error from ``os.rename()`` isn't because we're trying to copy
across a filesystem boundary, we get an error
"""
message = "Exception thrown in utils_t.py for TestRename"
m = mock.Mock(side_effect=OSError(-1, message))
with mock.patch('loris.utils.os.rename', m):
with pytest.raises(OSError):
utils.safe_rename(src, dst)
def copy_to_cache(self, ident):
ident = unquote(ident)
#get source image and write to temporary file
(source_url, options) = self._web_request_url(ident)
assert source_url is not None
cache_dir = self.cache_dir_path(ident)
mkdir_p(cache_dir)
with closing(requests.get(source_url, stream=True,
**options)) as response:
if not response.ok:
logger.warn(
"Source image not found at %s for identifier: %s. "
"Status code returned: %s.", source_url, ident,
response.status_code)
raise ResolverException(
"Source image not found for identifier: %s. "
"Status code returned: %s." %
(ident, response.status_code))
extension = self.cache_file_extension(ident, response)
local_fp = join(cache_dir, "loris_cache." + extension)
with tempfile.NamedTemporaryFile(dir=cache_dir,
delete=False) as tmp_file:
for chunk in response.iter_content(2048):
tmp_file.write(chunk)
# Now rename the temp file to the desired file name if it still
# doesn't exist (another process could have created it).
#
# Note: This is purely an optimisation; if the file springs into
# existence between the existence check and the copy, it will be
# overridden.
if exists(local_fp):
logger.info('Another process downloaded src image %s', local_fp)
remove(tmp_file.name)
else:
safe_rename(tmp_file.name, local_fp)
logger.info("Copied %s to %s", source_url, local_fp)
# Check for rules file associated with image file
# These files are < 2k in size, so fetch in one go.
# Assumes that the rules will be next to the image
# cache_dir is image specific, so this is easy
bits = split(source_url)
fn = bits[1].rsplit('.')[0] + "." + self.auth_rules_ext
rules_url = bits[0] + '/' + fn
try:
resp = requests.get(rules_url)
if resp.status_code == 200:
local_rules_fp = join(cache_dir,
"loris_cache." + self.auth_rules_ext)
if not exists(local_rules_fp):
with open(local_rules_fp, 'w') as fh:
fh.write(resp.text)
except:
# No connection available
pass
return local_fp
def copy_to_cache(self, ident):
""" downloads image source file from s3, if not in cache already."""
ident = unquote(ident)
# get source image and write to temporary file
(bucketname, keyname) = self.s3bucket_from_ident(ident)
try:
s3obj = self.s3.Object(bucketname, keyname)
content_type = s3obj.content_type
except Exception as e:
msg = "no content_type for s3 object ({}:{}): {}".format(
bucketname, keyname, e)
logger.error(msg)
raise ResolverException(msg)
extension = self.cache_file_extension(ident, content_type)
cache_dir = self.cache_dir_path(ident)
os.makedirs(cache_dir, exist_ok=True)
local_fp = os.path.join(cache_dir, "loris_cache." + extension)
with tempfile.NamedTemporaryFile(dir=cache_dir,
delete=False) as tmp_file:
try:
self.s3.Bucket(bucketname).download_fileobj(keyname, tmp_file)
except Exception as e:
msg = "unable to access or save s3 object ({}:{}): {}".format(
bucketname, keyname, e)
logger.error(msg)
raise ResolverException(msg)
# Now rename the temp file to the desired file name if it still
# doesn't exist (another process could have created it).
#
# Note: This is purely an optimisation; if the file springs into
# existence between the existence check and the copy, it will be
# overridden.
if os.path.exists(local_fp):
logger.info(
"Another process downloaded src image {}".format(local_fp))
os.remove(tmp_file.name)
else:
safe_rename(tmp_file.name, local_fp)
logger.info("Copied {}:{} to {}".format(bucketname, keyname,
local_fp))
# Check for rules file associated with image file
# These files are < 2k in size, so fetch in one go.
# Assumes that the rules will be next to the image
# cache_dir is image specific, so this is easy
bits = os.path.split(keyname) # === bash basename
fn = bits[1].rsplit(".")[0] + "." + self.auth_rules_ext
rules_keyname = bits[0] + "/" + fn
local_rules_fp = os.path.join(cache_dir,
"loris_cache." + self.auth_rules_ext)
try:
self.s3.Object(bucketname,
rules_keyname).download_file(local_rules_fp)
except Exception as e:
# no connection available?
msg = "ignoring rules file({}/{}) for ident({}): {}".format(
bucketname, rules_keyname, ident, e)
logger.warn(msg)
return local_fp