def check_file(self, local_name, url):
if not self._file_exists(local_name):
raise DistillPublishError(
'File does not exist: {}'.format(local_name))
local_hash = self._get_local_file_hash(local_name)
remote_hash = self._get_url_hash(url)
return local_hash == remote_hash
def compare_file(self, local_name, remote_name):
if not self._file_exists(local_name):
raise DistillPublishError('File does not exist: {}'.format(
local_name))
local_hash = self._get_local_file_hash(local_name)
o = self.d['container'].get_object(remote_name)
return o.etag == local_hash
def upload_file(self, local_name, remote_name):
if not self._file_exists(local_name):
raise DistillPublishError('File does not exist: {}'.format(
local_name))
local_hash = self._get_local_file_hash(local_name)
remote_obj = self.d['container'].upload_file(local_name, remote_name,
etag=local_hash)
return local_hash == remote_obj.etag
def remote_url(self, local_name):
if local_name[:len(self.source_dir)] != self.source_dir:
raise DistillPublishError('File {} is not in source dir {}'.format(
local_name, self.source_dir))
truncated = local_name[len(self.source_dir):]
remote_file = '/'.join(truncated.split(os.sep))
remote_uri = self.remote_url_parts.path + remote_file
return urlunsplit((self.remote_url_parts.scheme,
self.remote_url_parts.netloc, remote_uri, '', ''))
def authenticate(self):
credentials_file = self.options.get('JSON_CREDENTIALS', '')
if not os.path.exists(credentials_file):
err = 'Credentials file does not exist: {}'
raise DistillPublishError(err.format(credentials_file))
os.environ['GOOGLE_APPLICATION_CREDENTIALS'] = credentials_file
bucket = self.account_container()
self.d['connection'] = storage.Client()
self.d['bucket'] = self.d['connection'].get_bucket(bucket)
def authenticate(self):
username = self.account_username()
api_key = self.options.get('API_KEY', '')
region = self.options.get('REGION', '')
container = self.account_container()
pyrax.set_setting('identity_type', 'rackspace')
with warnings.catch_warnings():
warnings.simplefilter('ignore')
pyrax.set_credentials(username, api_key, region=region)
self.d['connection'] = pyrax.cloudfiles
if not self.d['connection']:
e = 'Failed to connect to Rackspace Cloud Files, check credentials'
raise DistillPublishError(e)
self.d['container'] = self.d['connection'].get_container(container)
def publish_dir(local_dir, backend, stdout):
stdout('Authenticating')
backend.authenticate()
stdout('Getting file indexes')
remote_files = backend.list_remote_files()
local_files = backend.list_local_files()
local_dirs = backend.list_local_dirs()
to_upload = set()
to_delete = set()
local_files_r = set()
# check local files to upload
for f in local_files:
remote_f = backend.remote_path(f)
local_files_r.add(remote_f)
if remote_f not in remote_files:
# local file not present remotely, upload it
to_upload.add(f)
else:
# file is present remotely, check its hash
if not backend.compare_file(f, remote_f):
stdout('File stale (hash different): {}'.format(remote_f))
to_upload.add(f)
else:
stdout('File fresh: {}'.format(remote_f))
# check for remote files to delete
for f in remote_files:
if f not in local_files_r:
to_delete.add(f)
# upload any new or changed files
for f in to_upload:
remote_f = backend.remote_path(f)
stdout('Publishing: {} -> {}'.format(f, remote_f))
backend.upload_file(f, backend.remote_path(f))
url = backend.remote_url(f)
stdout('Verifying: {}'.format(url))
if not backend.check_file(f, url):
err = 'Remote file {} failed hash check'
raise DistillPublishError(err.format(url))
# delete any orphan files
for f in to_delete:
stdout('Deleting remote: {}'.format(f))
backend.delete_remote_file(f)
def _validate_options(self):
for o in self.REQUIRED_OPTIONS:
if o not in self.options:
e = 'Missing required settings value for this backend: {}'
raise DistillPublishError(e.format(o))