def main(self): pbar = self.get_progressbar(maxval=self.args.get('image_size')) unbundle_out_r, unbundle_out_w = open_pipe_fileobjs() unbundle_sha1_r = create_unbundle_pipeline( self.args['source'], unbundle_out_w, self.args['enc_key'], self.args['enc_iv'], debug=self.debug) unbundle_out_w.close() actual_size = copy_with_progressbar(unbundle_out_r, self.args['dest'], progressbar=pbar) actual_sha1 = unbundle_sha1_r.recv() unbundle_sha1_r.close() expected_sha1 = self.args.get('sha1_digest') or '' expected_sha1 = expected_sha1.lower().strip('0x') expected_size = self.args.get('image_size') if expected_sha1 and expected_sha1 != actual_sha1: self.log.error('rejecting unbundle due to SHA1 mismatch ' '(expected SHA1: %s, actual: %s)', expected_sha1, actual_sha1) raise RuntimeError('bundle appears to be corrupt (expected SHA1: ' '{0}, actual: {1})' .format(expected_sha1, actual_sha1)) expected_size = self.args.get('image_size') if expected_size and expected_size != actual_size: self.log.error('rejecting unbundle due to size mismatch ' '(expected: %i, actual: %i)', expected_size, actual_size) raise RuntimeError('bundle appears to be corrupt (expected size: ' '{0}, actual: {1})' .format(expected_size, actual_size)) return actual_sha1, actual_size
def __init__(self, tarball): """ This method takes a tarfile.TarFile object and spawns *two* new processes: an xz process for decompression and an additional python process that simply feeds data from the TarFile to it. The latter is necessary because the file-like object we get from TarFile.extractfile cannot be passed to a subprocess directly. For that reason, one is also free to close the tarball after this object is created. """ self.__subp_pid = None self.__read_fh = None member = tarball.getmember(IMAGE_ARCNAME) compressed_image = tarball.extractfile(member) pipe_r, pipe_w = open_pipe_fileobjs() self.__subp_pid = os.fork() if self.__subp_pid == 0: os.setpgrp() pipe_r.close() self.__xz_proc = subprocess.Popen(('xz', '-d'), stdin=subprocess.PIPE, stdout=pipe_w, close_fds=True) pipe_w.close() shutil.copyfileobj(compressed_image, self.__xz_proc.stdin) self.__xz_proc.stdin.close() self.__xz_proc.wait() os._exit(os.EX_OK) else: self.__read_fh = pipe_r
def main(self): pbar = self.get_progressbar(maxval=self.args.get('image_size')) unbundle_out_r, unbundle_out_w = open_pipe_fileobjs() unbundle_sha1_r = create_unbundle_pipeline(self.args['source'], unbundle_out_w, self.args['enc_key'], self.args['enc_iv'], debug=self.debug) unbundle_out_w.close() actual_size = copy_with_progressbar(unbundle_out_r, self.args['dest'], progressbar=pbar) actual_sha1 = int(unbundle_sha1_r.recv(), 16) unbundle_sha1_r.close() expected_sha1 = int(self.args.get('sha1_digest') or '0', 16) expected_size = self.args.get('image_size') if expected_sha1 and expected_sha1 != actual_sha1: self.log.error( 'rejecting unbundle due to SHA1 mismatch ' '(expected SHA1: %x, actual: %x)', expected_sha1, actual_sha1) raise RuntimeError('bundle appears to be corrupt (expected SHA1: ' '{0:x}, actual: {1:x})'.format( expected_sha1, actual_sha1)) expected_size = self.args.get('image_size') if expected_size and expected_size != actual_size: self.log.error( 'rejecting unbundle due to size mismatch ' '(expected: %i, actual: %i)', expected_size, actual_size) raise RuntimeError('bundle appears to be corrupt (expected size: ' '{0}, actual: {1})'.format( expected_size, actual_size)) return actual_sha1, actual_size
def main(self): manifest = BundleManifest.read_from_fileobj( self.args['manifest'], privkey_filename=self.args['privatekey']) for part in manifest.image_parts: part_path = os.path.join(self.args['source'], part.filename) while part_path.startswith('./'): part_path = part_path[2:] if os.path.exists(part_path): part.filename = part_path else: raise RuntimeError( "bundle part '{0}' does not exist; you may need to use " "-s to specify where to find the bundle's parts" .format(part_path)) part_reader_out_r, part_reader_out_w = open_pipe_fileobjs() part_reader = multiprocessing.Process( target=self.__read_bundle_parts, args=(manifest, part_reader_out_w)) part_reader.start() part_reader_out_w.close() waitpid_in_thread(part_reader.pid) image_filename = os.path.join(self.args['destination'], manifest.image_name) with open(image_filename, 'w') as image: unbundlestream = UnbundleStream.from_other( self, source=part_reader_out_r, dest=image, enc_key=manifest.enc_key, enc_iv=manifest.enc_iv, image_size=manifest.image_size, sha1_digest=manifest.image_digest, show_progress=self.args.get('show_progress', False)) unbundlestream.main() return image_filename
def __init__(self, tarball): """ This method takes a tarfile.TarFile object and spawns *two* new processes: an xz process for decompression and an additional python process that simply feeds data from the TarFile to it. The latter is necessary because the file-like object we get from TarFile.extractfile cannot be passed to a subprocess directly. For that reason, one is also free to close the tarball after this object is created. """ self.__subp_pid = None self.__read_fh = None member = tarball.getmember(IMAGE_ARCNAME) compressed_image = tarball.extractfile(member) pipe_r, pipe_w = open_pipe_fileobjs() self.__subp_pid = os.fork() if self.__subp_pid == 0: os.setpgrp() pipe_r.close() self.__xz_proc = subprocess.Popen( ('xz', '-d'), stdin=subprocess.PIPE, stdout=pipe_w, close_fds=True) pipe_w.close() shutil.copyfileobj(compressed_image, self.__xz_proc.stdin) self.__xz_proc.stdin.close() self.__xz_proc.wait() os._exit(os.EX_OK) else: self.__read_fh = pipe_r
def main(self): manifest = self.fetch_manifest( self.service, privkey_filename=self.args['privatekey']) download_out_r, download_out_w = open_pipe_fileobjs() try: self.__create_download_pipeline(download_out_w) finally: download_out_w.close() image_filename = self.__open_dest(manifest) unbundlestream = UnbundleStream.from_other( self, source=download_out_r, dest=self.args['dest'], enc_key=manifest.enc_key, enc_iv=manifest.enc_iv, image_size=manifest.image_size, sha1_digest=manifest.image_digest, show_progress=self.args.get('show_progress', False)) unbundlestream.main() return image_filename
def main(self): pbar = self.args.get('progressbar', None) manifest = self.args.get('manifest') enc_key = self.args.get('enc_key') enc_iv = self.args.get('enc_iv') debug = self.args.get('debug') maxbytes = self.args.get('maxbytes') #Setup the destination fileobj... if isinstance(self.args.get('destination'), file): #Use provided file obj... self.log.debug('Writing image to provided fileobj') dest_file = self.args.get('destination') dest_file_name = str(dest_file.name) self.log.debug('Writing image to provided fileobj:' + dest_file_name) elif self.args.get('destination') == '-': #Use stdout... self.log.debug('Writing image to stdout') dest_file_name = '<stdout>' dest_file = os.fdopen(os.dup(os.sys.stdout.fileno()), 'w') else: #Open file at path provided... self.log.debug('Writing image to ') dest_file_name = self.args.get('destination') dest_file = open(self.args.get('destination'), 'w') #Start the unbundle... try: if self.args.get('source') and not self.args.get('source') == "-": if isinstance(self.args.get('source'), file): self.log.debug('Reading from provided fileobj') infile = self.args.get('source') else: self.log.debug('Reading from file at path %s', str(self.args.get('source'))) infile = open(self.args.get('source')) else: #Unbundle from stdin stream... self.log.debug('Reading from stdin') infile = os.fdopen(os.dup(os.sys.stdin.fileno())) try: progress_r, progress_w = open_pipe_fileobjs() sha1pipe = create_unbundle_pipeline( infile=infile, outfile=progress_w, enc_key=enc_key, enc_iv=enc_iv, debug=debug) progress_w.close() copy_with_progressbar(infile=progress_r, outfile=dest_file, progressbar=pbar, maxbytes=maxbytes) progress_r.close() finally: if infile: infile.close() if progress_r: progress_r.close() if progress_w: progress_w.close() digest = sha1pipe.recv() if manifest: #Verify the resulting unbundled Checksum matches the manifest if digest != manifest.image_digest: raise ValueError('Digest mismatch. ' 'Extracted image appears to be corrupt ' '(expected digest: {0}, actual: {1})' .format(manifest.image_digest, digest)) self.log.debug("\nExpected digest:{0}\n Actual digest:{1}" .format(manifest.image_digest, digest)) self.log.debug('Wrote stream to destination:' + dest_file_name) self.log.debug('Digest for unbundled image:' + str(digest)) except KeyboardInterrupt: print 'Caught keyboard interrupt' return return digest
def main(self): dest_dir = self.args.get('destination') manifest = self.args.get('manifest') #Setup the destination fileobj... if dest_dir == "-": #Write to stdout dest_file = os.fdopen(os.dup(os.sys.stdout.fileno()), 'w') dest_file_name = None else: #write to local file path... dest_file = open(dest_dir + "/" + manifest.image_name, 'w') dest_file_name = dest_file.name #check for avail space if the resulting image size is known if manifest: d_stat = os.statvfs(dest_file_name) avail_space = d_stat.f_frsize * d_stat.f_favail if manifest.image_size > avail_space: raise ValueError('Image size:{0} exceeds destination free ' 'space:{1}' .format(manifest.image_size, avail_space)) #setup progress bar... try: label = self.args.get('progressbar_label', 'UnBundling image') pbar = self.get_progressbar(label=label, maxval=manifest.image_size) except NameError: pbar = None try: #Start the unbundle... unbundle_r, unbundle_w = open_pipe_fileobjs() writer = spawn_process(self._concatenate_parts_to_file_for_pipe, outfile=unbundle_w, image_parts=manifest.image_parts, source_dir=self.args.get('source'), debug=self.args.get('debug')) unbundle_w.close() waitpid_in_thread(writer.pid) self.log.debug('Using enc key:' + str(manifest.enc_key)) self.log.debug('using enc iv:' + str(manifest.enc_iv)) digest = UnbundleStream(source=unbundle_r, destination=dest_file, enc_key=manifest.enc_key, enc_iv=manifest.enc_iv, progressbar=pbar, maxbytes=self.args.get('maxbytes'), config=self.config).main() digest = digest.strip() if dest_file: dest_file.close() #Verify the Checksum matches the manifest if digest != manifest.image_digest: raise ValueError('Digest mismatch. Extracted image appears ' 'to be corrupt (expected digest: {0}, ' 'actual: {1})' .format(manifest.image_digest, digest)) self.log.debug("\nExpected digest:{0}\n Actual digest:{1}" .format(str(manifest.image_digest), str(digest))) except KeyboardInterrupt: print 'Caught keyboard interrupt' if dest_file: os.remove(dest_file.name) return except Exception: traceback.print_exc() if dest_file: try: os.remove(dest_file.name) except OSError: print "Could not remove failed destination file." raise finally: dest_file.close() return dest_file_name