def download_file(self, filename, destination=None):
if not destination:
destination = filename
key = self.bucket.get_key(filename)
handler = ResumableDownloadHandler(tempfile.mktemp())
handler._save_tracker_info(key) # Ugly but necessary
self.rate_limiter.reset( self.rate_limit, -1 )
with file(destination, "a") as f:
handler.get_file(key, f, {}, cb=self.callbacks, num_cb=-1)
#validate
md5 = utils.md5_for_large_file(destination)
s3_md5 = key.metadata.get('nimbus-md5', None)
if s3_md5:
if s3_md5 != md5:
raise utils.Md5CheckError("md5 mismatch")
else:
if not '-' in key.etag:
s3_md5 = key.etag.strip('"\'')
if md5 != s3_md5:
raise utils.Md5CheckError("md5 mismatch")
else:
size = os.path.getsize(destination)
if size != key.size:
raise FileSizeError("error")
def check_integrity():
s3 = Offsite.get_s3_interface()
offsite_keys = s3.list_files()
for key in offsite_keys:
filename = key.name
filename_on_disk = os.path.join(settings.NIMBUS_DEFAULT_ARCHIVE,
filename)
if os.path.exists(filename_on_disk):
print "Checking",filename,"...",
s3_md5 = key.metadata.get('nimbus-md5', None)
local_md5 = outils.md5_for_large_file(filename_on_disk)
is_ok = True
if s3_md5:
if s3_md5 != local_md5:
is_ok = False
else:
if not '-' in key.etag:
s3_md5 = key.etag.strip('"\'')
if local_md5 != s3_md5:
is_ok = False
else:
size = os.path.getsize(filename_on_disk)
if size != key.size:
is_ok = False
if is_ok:
print "Ok"
else:
print "Error"
f = utils.filesizeformat
size = os.path.getsize(filename_on_disk)
print "Local file size: {0}. Remote file size: {1}".format(f(size), f(f.size))
print "Local file md5: {0}. Remote file md5: {1}".format(local_md5, s3_md5)
def _upload(self, filename, keyname):
from nimbus.offsite import queue_service
self.logger.info('simple upload')
key = self.bucket.new_key(keyname)
self.rate_limiter.reset( queue_service.get_worker_ratelimit() )
md5 = utils.md5_for_large_file(filename)
headers={'x-amz-meta-nimbus-md5' : md5}
with file(filename) as f_obj:
key.set_contents_from_file(f_obj,
cb=self.callbacks,
headers=headers,
num_cb=-1)
def _upload_multipart(self, filename, keyname, part):
from nimbus.offsite import queue_service
self.logger.info('initiate multipart upload ')
md5 = utils.md5_for_large_file(filename)
headers={'x-amz-meta-nimbus-md5' : md5}
multipart = self.bucket.initiate_multipart_upload(keyname, headers=headers)
self.logger.info('initiate multipart with part = %d' % (part) )
with MultipartFileManager(filename, part) as manager:
for (part_number, part_content) in enumerate(manager, part):
self.rate_limiter.reset( queue_service.get_worker_ratelimit() )
# FIX THIS: quantidade transferida e total sempre
#eh reportada como o da parte
part_file = StringIO(part_content)
multipart.upload_part_from_file(part_file,
part_number + 1,
cb=self.callbacks,
num_cb=-1)
self.multipart_status_callbacks(filename, part_number)
size = os.path.getsize(filename)
sent_size = (part_number * MIN_MULTIPART_SIZE) + len(part_content)
if size == sent_size:
multipart.complete_upload()
else:
raise IncompleteUpload("Not sent full file size")
while True:
data = ori.read(self.SIZE_512KB)
if not data:
break
dest.write(data)
ori.close()
dest.close()
try:
os.chmod(destination, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP)
os.chown(destination, getpwnam("nimbus").pw_uid, getpwnam("bacula").pw_gid)
except (OSError, IOError), error:
pass #FIX
if outils.md5_for_large_file(destination) != outils.md5_for_large_file(origin):
raise outils.Md5CheckError("md5 mismatch")
def finish(self):
self.device_manager.umount()
def check_integrity():
s3 = Offsite.get_s3_interface()
offsite_keys = s3.list_files()
for key in offsite_keys:
filename = key.name
filename_on_disk = os.path.join(settings.NIMBUS_DEFAULT_ARCHIVE,
filename)
