def download_file(self, filename, destination=None): if not destination: destination = filename key = self.bucket.get_key(filename) handler = ResumableDownloadHandler(tempfile.mktemp()) handler._save_tracker_info(key) # Ugly but necessary self.rate_limiter.reset( self.rate_limit, -1 ) with file(destination, "a") as f: handler.get_file(key, f, {}, cb=self.callbacks, num_cb=-1) #validate md5 = utils.md5_for_large_file(destination) s3_md5 = key.metadata.get('nimbus-md5', None) if s3_md5: if s3_md5 != md5: raise utils.Md5CheckError("md5 mismatch") else: if not '-' in key.etag: s3_md5 = key.etag.strip('"\'') if md5 != s3_md5: raise utils.Md5CheckError("md5 mismatch") else: size = os.path.getsize(destination) if size != key.size: raise FileSizeError("error")
def check_integrity(): s3 = Offsite.get_s3_interface() offsite_keys = s3.list_files() for key in offsite_keys: filename = key.name filename_on_disk = os.path.join(settings.NIMBUS_DEFAULT_ARCHIVE, filename) if os.path.exists(filename_on_disk): print "Checking",filename,"...", s3_md5 = key.metadata.get('nimbus-md5', None) local_md5 = outils.md5_for_large_file(filename_on_disk) is_ok = True if s3_md5: if s3_md5 != local_md5: is_ok = False else: if not '-' in key.etag: s3_md5 = key.etag.strip('"\'') if local_md5 != s3_md5: is_ok = False else: size = os.path.getsize(filename_on_disk) if size != key.size: is_ok = False if is_ok: print "Ok" else: print "Error" f = utils.filesizeformat size = os.path.getsize(filename_on_disk) print "Local file size: {0}. Remote file size: {1}".format(f(size), f(f.size)) print "Local file md5: {0}. Remote file md5: {1}".format(local_md5, s3_md5)
def _upload(self, filename, keyname): from nimbus.offsite import queue_service self.logger.info('simple upload') key = self.bucket.new_key(keyname) self.rate_limiter.reset( queue_service.get_worker_ratelimit() ) md5 = utils.md5_for_large_file(filename) headers={'x-amz-meta-nimbus-md5' : md5} with file(filename) as f_obj: key.set_contents_from_file(f_obj, cb=self.callbacks, headers=headers, num_cb=-1)
def _upload_multipart(self, filename, keyname, part): from nimbus.offsite import queue_service self.logger.info('initiate multipart upload ') md5 = utils.md5_for_large_file(filename) headers={'x-amz-meta-nimbus-md5' : md5} multipart = self.bucket.initiate_multipart_upload(keyname, headers=headers) self.logger.info('initiate multipart with part = %d' % (part) ) with MultipartFileManager(filename, part) as manager: for (part_number, part_content) in enumerate(manager, part): self.rate_limiter.reset( queue_service.get_worker_ratelimit() ) # FIX THIS: quantidade transferida e total sempre #eh reportada como o da parte part_file = StringIO(part_content) multipart.upload_part_from_file(part_file, part_number + 1, cb=self.callbacks, num_cb=-1) self.multipart_status_callbacks(filename, part_number) size = os.path.getsize(filename) sent_size = (part_number * MIN_MULTIPART_SIZE) + len(part_content) if size == sent_size: multipart.complete_upload() else: raise IncompleteUpload("Not sent full file size")
while True: data = ori.read(self.SIZE_512KB) if not data: break dest.write(data) ori.close() dest.close() try: os.chmod(destination, stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IWGRP) os.chown(destination, getpwnam("nimbus").pw_uid, getpwnam("bacula").pw_gid) except (OSError, IOError), error: pass #FIX if outils.md5_for_large_file(destination) != outils.md5_for_large_file(origin): raise outils.Md5CheckError("md5 mismatch") def finish(self): self.device_manager.umount() def check_integrity(): s3 = Offsite.get_s3_interface() offsite_keys = s3.list_files() for key in offsite_keys: filename = key.name filename_on_disk = os.path.join(settings.NIMBUS_DEFAULT_ARCHIVE, filename)