def create_bundle(self, path_prefix):
# Fill out all the relevant info needed for a tarball
tarinfo = tarfile.TarInfo(self.args['prefix'])
tarinfo.size = self.args['image_size']
# The pipeline begins with self.args['image'] feeding a bundling pipe
# segment through a progress meter, which has to happen on the main
# thread, so we add that to the pipeline last.
# meter --(bytes)--> bundler
bundle_in_r, bundle_in_w = euca2ools.bundle.util.open_pipe_fileobjs()
partwriter_in_r, partwriter_in_w = \
euca2ools.bundle.util.open_pipe_fileobjs()
digest_result_mpconn = create_bundle_pipeline(bundle_in_r,
partwriter_in_w,
self.args['enc_key'],
self.args['enc_iv'],
tarinfo,
debug=self.debug)
bundle_in_r.close()
partwriter_in_w.close()
# bundler --(bytes)-> part writer
bundle_partinfo_mpconn = create_bundle_part_writer(
partwriter_in_r,
path_prefix,
self.args['part_size'],
debug=self.debug)
partwriter_in_r.close()
# part writer --(part info)-> part info aggregator
# (needed for building the manifest)
bundle_partinfo_aggr_mpconn = create_mpconn_aggregator(
bundle_partinfo_mpconn, debug=self.debug)
bundle_partinfo_mpconn.close()
# disk --(bytes)-> bundler
# (synchronous)
label = self.args.get('progressbar_label') or 'Bundling image'
pbar = self.get_progressbar(label=label,
maxval=self.args['image_size'])
with self.args['image'] as image:
try:
read_size = copy_with_progressbar(image,
bundle_in_w,
progressbar=pbar)
except ValueError:
self.log.debug('error from copy_with_progressbar',
exc_info=True)
raise RuntimeError('corrupt bundle: input size was larger '
'than expected image size of {0}'.format(
self.args['image_size']))
bundle_in_w.close()
if read_size != self.args['image_size']:
raise RuntimeError('corrupt bundle: input size did not match '
'expected image size (expected size: {0}, '
'read: {1})'.format(self.args['image_size'],
read_size))
# All done; now grab info about the bundle we just created
try:
digest = digest_result_mpconn.recv()
partinfo = bundle_partinfo_aggr_mpconn.recv()
except EOFError:
self.log.debug('EOFError from reading bundle info', exc_info=True)
raise RuntimeError(
'corrupt bundle: bundle process was interrupted')
finally:
digest_result_mpconn.close()
bundle_partinfo_aggr_mpconn.close()
self.log.info('%i bundle parts written to %s', len(partinfo),
os.path.dirname(path_prefix))
self.log.debug('bundle digest: %s', digest)
return digest, partinfo
def create_and_upload_bundle(self, path_prefix, key_prefix):
part_write_sem = multiprocessing.Semaphore(
max(1, self.args['max_pending_parts']))
# Fill out all the relevant info needed for a tarball
tarinfo = tarfile.TarInfo(self.args['prefix'])
tarinfo.size = self.args['image_size']
# disk --(bytes)-> bundler
partwriter_in_r, partwriter_in_w = \
euca2ools.bundle.util.open_pipe_fileobjs()
digest_result_mpconn = create_bundle_pipeline(self.args['image'],
partwriter_in_w,
self.args['enc_key'],
self.args['enc_iv'],
tarinfo,
debug=self.debug)
partwriter_in_w.close()
# bundler --(bytes)-> part writer
bundle_partinfo_mpconn = create_bundle_part_writer(
partwriter_in_r,
path_prefix,
self.args['part_size'],
part_write_sem=part_write_sem,
debug=self.debug)
partwriter_in_r.close()
# part writer --(part info)-> part uploader
# This must be driven on the main thread since it has a progress bar,
# so for now we'll just set up its output pipe so we can attach it to
# the remainder of the pipeline.
uploaded_partinfo_mpconn_r, uploaded_partinfo_mpconn_w = \
multiprocessing.Pipe(duplex=False)
# part uploader --(part info)-> part deleter
if not self.args.get('preserve_bundle', False):
deleted_partinfo_mpconn_r, deleted_partinfo_mpconn_w = \
multiprocessing.Pipe(duplex=False)
create_bundle_part_deleter(uploaded_partinfo_mpconn_r,
out_mpconn=deleted_partinfo_mpconn_w)
uploaded_partinfo_mpconn_r.close()
deleted_partinfo_mpconn_w.close()
else:
# Bypass this stage
deleted_partinfo_mpconn_r = uploaded_partinfo_mpconn_r
# part deleter --(part info)-> part info aggregator
# (needed for building the manifest)
bundle_partinfo_aggregate_mpconn = create_mpconn_aggregator(
deleted_partinfo_mpconn_r, debug=self.debug)
deleted_partinfo_mpconn_r.close()
# Now drive the pipeline by uploading parts.
try:
self.upload_bundle_parts(
bundle_partinfo_mpconn,
key_prefix,
partinfo_out_mpconn=uploaded_partinfo_mpconn_w,
part_write_sem=part_write_sem,
show_progress=self.args.get('show_progress'))
finally:
# Make sure the writer gets a chance to exit
part_write_sem.release()
# All done; now grab info about the bundle we just created
try:
digest = digest_result_mpconn.recv()
partinfo = bundle_partinfo_aggregate_mpconn.recv()
except EOFError:
self.log.debug('EOFError from reading bundle info', exc_info=True)
raise RuntimeError(
'corrupt bundle: bundle process was interrupted')
finally:
digest_result_mpconn.close()
bundle_partinfo_aggregate_mpconn.close()
self.log.info('%i bundle parts uploaded to %s', len(partinfo),
self.args['bucket'])
self.log.debug('bundle digest: %s', digest)
return digest, partinfo
def create_bundle(self, path_prefix):
# Fill out all the relevant info needed for a tarball
tarinfo = tarfile.TarInfo(self.args['prefix'])
tarinfo.size = self.args['image_size']
# The pipeline begins with self.args['image'] feeding a bundling pipe
# segment through a progress meter, which has to happen on the main
# thread, so we add that to the pipeline last.
# meter --(bytes)--> bundler
bundle_in_r, bundle_in_w = euca2ools.bundle.util.open_pipe_fileobjs()
partwriter_in_r, partwriter_in_w = \
euca2ools.bundle.util.open_pipe_fileobjs()
digest_result_mpconn = create_bundle_pipeline(
bundle_in_r, partwriter_in_w, self.args['enc_key'],
self.args['enc_iv'], tarinfo, debug=self.debug)
bundle_in_r.close()
partwriter_in_w.close()
# bundler --(bytes)-> part writer
bundle_partinfo_mpconn = create_bundle_part_writer(
partwriter_in_r, path_prefix, self.args['part_size'],
debug=self.debug)
partwriter_in_r.close()
# part writer --(part info)-> part info aggregator
# (needed for building the manifest)
bundle_partinfo_aggr_mpconn = create_mpconn_aggregator(
bundle_partinfo_mpconn, debug=self.debug)
bundle_partinfo_mpconn.close()
# disk --(bytes)-> bundler
# (synchronous)
label = self.args.get('progressbar_label') or 'Bundling image'
pbar = self.get_progressbar(label=label,
maxval=self.args['image_size'])
with self.args['image'] as image:
try:
read_size = copy_with_progressbar(image, bundle_in_w,
progressbar=pbar)
except ValueError:
self.log.debug('error from copy_with_progressbar',
exc_info=True)
raise RuntimeError('corrupt bundle: input size was larger '
'than expected image size of {0}'
.format(self.args['image_size']))
bundle_in_w.close()
if read_size != self.args['image_size']:
raise RuntimeError('corrupt bundle: input size did not match '
'expected image size (expected size: {0}, '
'read: {1})'
.format(self.args['image_size'], read_size))
# All done; now grab info about the bundle we just created
try:
digest = digest_result_mpconn.recv()
partinfo = bundle_partinfo_aggr_mpconn.recv()
except EOFError:
self.log.debug('EOFError from reading bundle info', exc_info=True)
raise RuntimeError(
'corrupt bundle: bundle process was interrupted')
finally:
digest_result_mpconn.close()
bundle_partinfo_aggr_mpconn.close()
self.log.info('%i bundle parts written to %s', len(partinfo),
os.path.dirname(path_prefix))
self.log.debug('bundle digest: %s', digest)
return digest, partinfo
def create_and_upload_bundle(self, path_prefix, key_prefix):
part_write_sem = multiprocessing.Semaphore(
max(1, self.args['max_pending_parts']))
# Fill out all the relevant info needed for a tarball
tarinfo = tarfile.TarInfo(self.args['prefix'])
tarinfo.size = self.args['image_size']
# disk --(bytes)-> bundler
partwriter_in_r, partwriter_in_w = \
euca2ools.bundle.util.open_pipe_fileobjs()
digest_result_mpconn = create_bundle_pipeline(
self.args['image'], partwriter_in_w, self.args['enc_key'],
self.args['enc_iv'], tarinfo, debug=self.debug)
partwriter_in_w.close()
# bundler --(bytes)-> part writer
bundle_partinfo_mpconn = create_bundle_part_writer(
partwriter_in_r, path_prefix, self.args['part_size'],
part_write_sem=part_write_sem, debug=self.debug)
partwriter_in_r.close()
# part writer --(part info)-> part uploader
# This must be driven on the main thread since it has a progress bar,
# so for now we'll just set up its output pipe so we can attach it to
# the remainder of the pipeline.
uploaded_partinfo_mpconn_r, uploaded_partinfo_mpconn_w = \
multiprocessing.Pipe(duplex=False)
# part uploader --(part info)-> part deleter
if not self.args.get('preserve_bundle', False):
deleted_partinfo_mpconn_r, deleted_partinfo_mpconn_w = \
multiprocessing.Pipe(duplex=False)
create_bundle_part_deleter(uploaded_partinfo_mpconn_r,
out_mpconn=deleted_partinfo_mpconn_w)
uploaded_partinfo_mpconn_r.close()
deleted_partinfo_mpconn_w.close()
else:
# Bypass this stage
deleted_partinfo_mpconn_r = uploaded_partinfo_mpconn_r
# part deleter --(part info)-> part info aggregator
# (needed for building the manifest)
bundle_partinfo_aggregate_mpconn = create_mpconn_aggregator(
deleted_partinfo_mpconn_r, debug=self.debug)
deleted_partinfo_mpconn_r.close()
# Now drive the pipeline by uploading parts.
try:
self.upload_bundle_parts(
bundle_partinfo_mpconn, key_prefix,
partinfo_out_mpconn=uploaded_partinfo_mpconn_w,
part_write_sem=part_write_sem,
show_progress=self.args.get('show_progress'))
finally:
# Make sure the writer gets a chance to exit
part_write_sem.release()
# All done; now grab info about the bundle we just created
try:
digest = digest_result_mpconn.recv()
partinfo = bundle_partinfo_aggregate_mpconn.recv()
except EOFError:
self.log.debug('EOFError from reading bundle info', exc_info=True)
raise RuntimeError(
'corrupt bundle: bundle process was interrupted')
finally:
digest_result_mpconn.close()
bundle_partinfo_aggregate_mpconn.close()
self.log.info('%i bundle parts uploaded to %s', len(partinfo),
self.args['bucket'])
self.log.debug('bundle digest: %s', digest)
return digest, partinfo