def touch_dst(src, dst, *args, **kwargs):
import os
from baiji.util.shutillib import mkdir_p
_ = src, args, kwargs
mkdir_p(os.path.dirname(dst))
with open(dst, 'w'):
pass
def touch_dst(src, dst, *args, **kwargs):
import os
from baiji.util.shutillib import mkdir_p
_ = src, args, kwargs
mkdir_p(os.path.dirname(dst))
with open(dst, 'w'):
pass
def __init__(self,
key,
mode='r',
connection=None,
encrypt=True,
version_id=None):
from baiji.connection import S3Connection
self.encrypt = encrypt
self.key = key
if path.islocal(key):
self.should_upload_on_close = False
self.mode = FileMode(mode, allowed_modes='arwxb+t')
from six.moves import builtins
local_path = path.parse(key).path
if self.mode.is_output and not os.path.exists(
os.path.dirname(local_path)):
from baiji.util.shutillib import mkdir_p
mkdir_p(os.path.dirname(local_path))
try:
# Use os.open to catch exclusive access to the file, but use open to get a nice, useful file object
self.fd = os.open(local_path, self.mode.flags)
self.f = builtins.open(local_path,
self.mode.mode.replace('x', 'w'))
os.close(self.fd)
except OSError as e:
import errno
if e.errno is errno.EEXIST:
raise KeyExists("Local file exists: %s" % local_path)
elif e.errno is errno.ENOENT:
raise KeyNotFound("Local file does not exist: %s" %
local_path)
else:
raise IOError(e.errno, "%s: %s" % (e.strerror, e.filename))
else:
if connection is None:
connection = S3Connection()
self.connection = connection
self.mode = FileMode(mode, allowed_modes='rwxbt')
self.should_upload_on_close = self.mode.is_output
if self.mode.creating_exclusively:
if self.connection.exists(self.key):
raise KeyExists("Key exists in bucket: %s" % self.key)
else:
self.connection.touch(self.key, encrypt=self.encrypt)
# Use w+ so we can read back the contents in upload()
new_mode = ('w+' + (self.mode.binary and 'b' or '') +
(self.mode.text and 't' or ''))
from baiji.util import tempfile
self.f = tempfile.NamedTemporaryFile(
mode=new_mode,
suffix=os.path.splitext(path.parse(self.key).path)[1])
self.name = self.f.name
self.remotename = key # Used by some serialization code to find files which sit along side the file in question, like textures which sit next to a mesh file
if self.mode.reading:
self.connection.cp(self.key,
self.name,
force=True,
version_id=version_id)
def test_s3_cp_errors_without_permissions(self):
from baiji.util.shutillib import mkdir_p
locked_dir = os.path.join(self.tmp_dir, 'locked')
mkdir_p(locked_dir)
os.chmod(locked_dir, 666)
with self.assertRaises(s3.S3Exception):
s3.cp(self.local_file, os.path.join(locked_dir, 'nope.txt'))
self.assertFalse(os.path.exists(os.path.join(locked_dir, 'nope.txt')))
def test_s3_cp_errors_without_permissions(self):
from baiji.util.shutillib import mkdir_p
locked_dir = os.path.join(self.tmp_dir, 'locked')
mkdir_p(locked_dir)
os.chmod(locked_dir, 666)
with self.assertRaises(s3.S3Exception):
s3.cp(self.local_file, os.path.join(locked_dir, 'nope.txt'))
self.assertFalse(os.path.exists(os.path.join(locked_dir, 'nope.txt')))
def sync_file(self, src, dst, update=True, delete=False, progress=False, policy=None, encoding=None, encrypt=True, guess_content_type=False):
'''
Sync a file from src to dst.
update: When True, update dst if it exists but contents do not match.
delete: When True, remove dst if src does not exist. When False, raise
an error if src does not exist.
As this function is a file by file sync, not applicable to directories
nor recursive, src being a directory is best treated as mkdir_p(dst).
'''
from baiji.util.console import create_conditional_print
print_verbose = create_conditional_print(progress)
if path.isdirlike(src):
print_verbose('{} is a directory'.format(src))
if path.islocal(dst): # for remote paths, don't bother creating dirs; they don't really exist.
from baiji.util.shutillib import mkdir_p
mkdir_p(dst)
return
src_exists = self.exists(src)
if not delete and not src_exists:
raise KeyNotFound(
"Error syncing {} to {}: Source doesn't exist".format(src, dst))
dst_exists = self.exists(dst)
needs_delete = dst_exists and not src_exists
needs_fresh_copy = src_exists and not dst_exists
needs_update = dst_exists and src_exists and self.etag(src) != self.etag(dst)
if not needs_delete and not needs_fresh_copy and not needs_update:
print_verbose('{} is up to date'.format(dst))
return
# At this point, exactly one of these should be true.
assert needs_delete ^ needs_fresh_copy ^ needs_update
if needs_fresh_copy:
print_verbose('copying {} to {}'.format(src, dst))
self.cp(src, dst, force=False, progress=progress, policy=policy, encoding=encoding, encrypt=encrypt, guess_content_type=guess_content_type)
elif needs_update:
print_verbose('file is out of date: {}'.format(dst))
if update:
print_verbose('copying {} to {}'.format(src, dst))
self.cp(src, dst, force=True, progress=progress, policy=policy, encoding=encoding, encrypt=encrypt, guess_content_type=guess_content_type)
elif needs_delete:
print_verbose('source file does not exist: {}'.format(src))
if delete:
print_verbose('removing {}'.format(dst))
self.rm(dst)
def sync_file(self, src, dst, update=True, delete=False, progress=False, policy=None, encoding=None, encrypt=True, guess_content_type=False):
'''
Sync a file from src to dst.
update: When True, update dst if it exists but contents do not match.
delete: When True, remove dst if src does not exist. When False, raise
an error if src does not exist.
As this function is a file by file sync, not applicable to directories
nor recursive, src being a directory is best treated as mkdir_p(dst).
'''
from baiji.util.console import create_conditional_print
print_verbose = create_conditional_print(progress)
if path.isdirlike(src):
print_verbose('{} is a directory'.format(src))
if path.islocal(dst): # for remote paths, don't bother creating dirs; they don't really exist.
from baiji.util.shutillib import mkdir_p
mkdir_p(dst)
return
src_exists = self.exists(src)
if not delete and not src_exists:
raise KeyNotFound(
"Error syncing {} to {}: Source doesn't exist".format(src, dst))
dst_exists = self.exists(dst)
needs_delete = dst_exists and not src_exists
needs_fresh_copy = src_exists and not dst_exists
needs_update = dst_exists and src_exists and self.etag(src) != self.etag(dst)
if not needs_delete and not needs_fresh_copy and not needs_update:
print_verbose('{} is up to date'.format(dst))
return
# At this point, exactly one of these should be true.
assert needs_delete ^ needs_fresh_copy ^ needs_update
if needs_fresh_copy:
print_verbose('copying {} to {}'.format(src, dst))
self.cp(src, dst, force=False, progress=progress, policy=policy, encoding=encoding, encrypt=encrypt, guess_content_type=guess_content_type)
elif needs_update:
print_verbose('file is out of date: {}'.format(dst))
if update:
print_verbose('copying {} to {}'.format(src, dst))
self.cp(src, dst, force=True, progress=progress, policy=policy, encoding=encoding, encrypt=encrypt, guess_content_type=guess_content_type)
elif needs_delete:
print_verbose('source file does not exist: {}'.format(src))
if delete:
print_verbose('removing {}'.format(dst))
self.rm(dst)
def prep_local_destination(self):
from baiji.util.shutillib import mkdir_p
mkdir_p(os.path.dirname(self.dst.path))
def test_uploading_a_directory_without_slash(self):
from baiji.util.shutillib import mkdir_p
mkdir_p(os.path.join(self.tmp_dir, "foo"))
with self.assertRaises(ValueError):
s3.cp(os.path.join(self.tmp_dir, "foo"), self.remote_file("foo"))
self.assertFalse(s3.exists(self.remote_file("")))
def prep_local_destination(self):
from baiji.util.shutillib import mkdir_p
mkdir_p(os.path.dirname(self.dst.path))
def test_uploading_a_directory_without_slash(self):
from baiji.util.shutillib import mkdir_p
mkdir_p(os.path.join(self.tmp_dir, "foo"))
with self.assertRaises(ValueError):
s3.cp(os.path.join(self.tmp_dir, "foo"), self.remote_file("foo"))
self.assertFalse(s3.exists(self.remote_file("")))
def _override_tmp(fn, args, kwargs):
if 'dir' not in kwargs:
kwargs['dir'] = os.environ.get('BAIJI_TMP', None)
if kwargs['dir'] is not None:
mkdir_p(kwargs['dir'])
return fn(*args, **kwargs)
def update_timestamp(self):
from baiji.util.shutillib import mkdir_p
if not os.path.exists(self.timestamp_file):
mkdir_p(os.path.dirname(self.timestamp_file))
open(self.timestamp_file, 'w').close()
def update_timestamp(self):
from baiji.util.shutillib import mkdir_p
if not os.path.exists(self.timestamp_file):
mkdir_p(os.path.dirname(self.timestamp_file))
open(self.timestamp_file, 'w').close()
