def main(self): pbar = self.get_progressbar(maxval=self.args.get('image_size')) unbundle_out_r, unbundle_out_w = open_pipe_fileobjs() unbundle_sha1_r = create_unbundle_pipeline( self.args['source'], unbundle_out_w, self.args['enc_key'], self.args['enc_iv'], debug=self.debug) unbundle_out_w.close() actual_size = copy_with_progressbar(unbundle_out_r, self.args['dest'], progressbar=pbar) actual_sha1 = unbundle_sha1_r.recv() unbundle_sha1_r.close() expected_sha1 = self.args.get('sha1_digest') or '' expected_sha1 = expected_sha1.lower().strip('0x') expected_size = self.args.get('image_size') if expected_sha1 and expected_sha1 != actual_sha1: self.log.error('rejecting unbundle due to SHA1 mismatch ' '(expected SHA1: %s, actual: %s)', expected_sha1, actual_sha1) raise RuntimeError('bundle appears to be corrupt (expected SHA1: ' '{0}, actual: {1})' .format(expected_sha1, actual_sha1)) expected_size = self.args.get('image_size') if expected_size and expected_size != actual_size: self.log.error('rejecting unbundle due to size mismatch ' '(expected: %i, actual: %i)', expected_size, actual_size) raise RuntimeError('bundle appears to be corrupt (expected size: ' '{0}, actual: {1})' .format(expected_size, actual_size)) return actual_sha1, actual_size
def main(self): pbar = self.get_progressbar(maxval=self.args.get('image_size')) unbundle_out_r, unbundle_out_w = open_pipe_fileobjs() unbundle_sha1_r = create_unbundle_pipeline(self.args['source'], unbundle_out_w, self.args['enc_key'], self.args['enc_iv'], debug=self.debug) unbundle_out_w.close() actual_size = copy_with_progressbar(unbundle_out_r, self.args['dest'], progressbar=pbar) actual_sha1 = int(unbundle_sha1_r.recv(), 16) unbundle_sha1_r.close() expected_sha1 = int(self.args.get('sha1_digest') or '0', 16) expected_size = self.args.get('image_size') if expected_sha1 and expected_sha1 != actual_sha1: self.log.error( 'rejecting unbundle due to SHA1 mismatch ' '(expected SHA1: %x, actual: %x)', expected_sha1, actual_sha1) raise RuntimeError('bundle appears to be corrupt (expected SHA1: ' '{0:x}, actual: {1:x})'.format( expected_sha1, actual_sha1)) expected_size = self.args.get('image_size') if expected_size and expected_size != actual_size: self.log.error( 'rejecting unbundle due to size mismatch ' '(expected: %i, actual: %i)', expected_size, actual_size) raise RuntimeError('bundle appears to be corrupt (expected size: ' '{0}, actual: {1})'.format( expected_size, actual_size)) return actual_sha1, actual_size
def create_bundle(self, path_prefix): # Fill out all the relevant info needed for a tarball tarinfo = tarfile.TarInfo(self.args['prefix']) tarinfo.size = self.args['image_size'] # The pipeline begins with self.args['image'] feeding a bundling pipe # segment through a progress meter, which has to happen on the main # thread, so we add that to the pipeline last. # meter --(bytes)--> bundler bundle_in_r, bundle_in_w = euca2ools.bundle.util.open_pipe_fileobjs() partwriter_in_r, partwriter_in_w = \ euca2ools.bundle.util.open_pipe_fileobjs() digest_result_mpconn = create_bundle_pipeline(bundle_in_r, partwriter_in_w, self.args['enc_key'], self.args['enc_iv'], tarinfo, debug=self.debug) bundle_in_r.close() partwriter_in_w.close() # bundler --(bytes)-> part writer bundle_partinfo_mpconn = create_bundle_part_writer( partwriter_in_r, path_prefix, self.args['part_size'], debug=self.debug) partwriter_in_r.close() # part writer --(part info)-> part info aggregator # (needed for building the manifest) bundle_partinfo_aggr_mpconn = create_mpconn_aggregator( bundle_partinfo_mpconn, debug=self.debug) bundle_partinfo_mpconn.close() # disk --(bytes)-> bundler # (synchronous) label = self.args.get('progressbar_label') or 'Bundling image' pbar = self.get_progressbar(label=label, maxval=self.args['image_size']) with self.args['image'] as image: try: read_size = copy_with_progressbar(image, bundle_in_w, progressbar=pbar) except ValueError: self.log.debug('error from copy_with_progressbar', exc_info=True) raise RuntimeError('corrupt bundle: input size was larger ' 'than expected image size of {0}'.format( self.args['image_size'])) bundle_in_w.close() if read_size != self.args['image_size']: raise RuntimeError('corrupt bundle: input size did not match ' 'expected image size (expected size: {0}, ' 'read: {1})'.format(self.args['image_size'], read_size)) # All done; now grab info about the bundle we just created try: digest = digest_result_mpconn.recv() partinfo = bundle_partinfo_aggr_mpconn.recv() except EOFError: self.log.debug('EOFError from reading bundle info', exc_info=True) raise RuntimeError( 'corrupt bundle: bundle process was interrupted') finally: digest_result_mpconn.close() bundle_partinfo_aggr_mpconn.close() self.log.info('%i bundle parts written to %s', len(partinfo), os.path.dirname(path_prefix)) self.log.debug('bundle digest: %s', digest) return digest, partinfo
def create_bundle(self, path_prefix): # Fill out all the relevant info needed for a tarball tarinfo = tarfile.TarInfo(self.args['prefix']) tarinfo.size = self.args['image_size'] # The pipeline begins with self.args['image'] feeding a bundling pipe # segment through a progress meter, which has to happen on the main # thread, so we add that to the pipeline last. # meter --(bytes)--> bundler bundle_in_r, bundle_in_w = euca2ools.bundle.util.open_pipe_fileobjs() partwriter_in_r, partwriter_in_w = \ euca2ools.bundle.util.open_pipe_fileobjs() digest_result_mpconn = create_bundle_pipeline( bundle_in_r, partwriter_in_w, self.args['enc_key'], self.args['enc_iv'], tarinfo, debug=self.debug) bundle_in_r.close() partwriter_in_w.close() # bundler --(bytes)-> part writer bundle_partinfo_mpconn = create_bundle_part_writer( partwriter_in_r, path_prefix, self.args['part_size'], debug=self.debug) partwriter_in_r.close() # part writer --(part info)-> part info aggregator # (needed for building the manifest) bundle_partinfo_aggr_mpconn = create_mpconn_aggregator( bundle_partinfo_mpconn, debug=self.debug) bundle_partinfo_mpconn.close() # disk --(bytes)-> bundler # (synchronous) label = self.args.get('progressbar_label') or 'Bundling image' pbar = self.get_progressbar(label=label, maxval=self.args['image_size']) with self.args['image'] as image: try: read_size = copy_with_progressbar(image, bundle_in_w, progressbar=pbar) except ValueError: self.log.debug('error from copy_with_progressbar', exc_info=True) raise RuntimeError('corrupt bundle: input size was larger ' 'than expected image size of {0}' .format(self.args['image_size'])) bundle_in_w.close() if read_size != self.args['image_size']: raise RuntimeError('corrupt bundle: input size did not match ' 'expected image size (expected size: {0}, ' 'read: {1})' .format(self.args['image_size'], read_size)) # All done; now grab info about the bundle we just created try: digest = digest_result_mpconn.recv() partinfo = bundle_partinfo_aggr_mpconn.recv() except EOFError: self.log.debug('EOFError from reading bundle info', exc_info=True) raise RuntimeError( 'corrupt bundle: bundle process was interrupted') finally: digest_result_mpconn.close() bundle_partinfo_aggr_mpconn.close() self.log.info('%i bundle parts written to %s', len(partinfo), os.path.dirname(path_prefix)) self.log.debug('bundle digest: %s', digest) return digest, partinfo
def main(self): pbar = self.args.get('progressbar', None) manifest = self.args.get('manifest') enc_key = self.args.get('enc_key') enc_iv = self.args.get('enc_iv') debug = self.args.get('debug') maxbytes = self.args.get('maxbytes') #Setup the destination fileobj... if isinstance(self.args.get('destination'), file): #Use provided file obj... self.log.debug('Writing image to provided fileobj') dest_file = self.args.get('destination') dest_file_name = str(dest_file.name) self.log.debug('Writing image to provided fileobj:' + dest_file_name) elif self.args.get('destination') == '-': #Use stdout... self.log.debug('Writing image to stdout') dest_file_name = '<stdout>' dest_file = os.fdopen(os.dup(os.sys.stdout.fileno()), 'w') else: #Open file at path provided... self.log.debug('Writing image to ') dest_file_name = self.args.get('destination') dest_file = open(self.args.get('destination'), 'w') #Start the unbundle... try: if self.args.get('source') and not self.args.get('source') == "-": if isinstance(self.args.get('source'), file): self.log.debug('Reading from provided fileobj') infile = self.args.get('source') else: self.log.debug('Reading from file at path %s', str(self.args.get('source'))) infile = open(self.args.get('source')) else: #Unbundle from stdin stream... self.log.debug('Reading from stdin') infile = os.fdopen(os.dup(os.sys.stdin.fileno())) try: progress_r, progress_w = open_pipe_fileobjs() sha1pipe = create_unbundle_pipeline( infile=infile, outfile=progress_w, enc_key=enc_key, enc_iv=enc_iv, debug=debug) progress_w.close() copy_with_progressbar(infile=progress_r, outfile=dest_file, progressbar=pbar, maxbytes=maxbytes) progress_r.close() finally: if infile: infile.close() if progress_r: progress_r.close() if progress_w: progress_w.close() digest = sha1pipe.recv() if manifest: #Verify the resulting unbundled Checksum matches the manifest if digest != manifest.image_digest: raise ValueError('Digest mismatch. ' 'Extracted image appears to be corrupt ' '(expected digest: {0}, actual: {1})' .format(manifest.image_digest, digest)) self.log.debug("\nExpected digest:{0}\n Actual digest:{1}" .format(manifest.image_digest, digest)) self.log.debug('Wrote stream to destination:' + dest_file_name) self.log.debug('Digest for unbundled image:' + str(digest)) except KeyboardInterrupt: print 'Caught keyboard interrupt' return return digest