def _download_parts_to_fileobj(self, manifest, dest_fileobj):
'''
Attempts to iterate through all parts contained in 'manifest' and
download and concatenate each part to 'dest_fileobj'. If the
manifest contains the intended image size, and the resulting bytes
downloaded does not match this size, ValueError is raised.
:param manifest: downloadmanifest obj
:param dest_fileobj: file like object to write downloaded parts to
:returns bytes downloaded
'''
bytes = 0
for part_index in xrange(0, manifest.part_count):
part = manifest.get_part_by_index(part_index)
self.log.debug('Downloading part#:' + str(part.part_index))
bytes += part.download(dest_fileobj=dest_fileobj) or 0
self.log.debug('Wrote bytes:' + str(bytes) + "/"
+ str(manifest.download_image_size) + ", digest:"
+ str(part.written_digest))
if self.args.reportprogress:
stages.report_status('"bytes_downloaded":%d' % bytes)
if manifest.download_image_size is not None and not self.args.skipsizevalidation:
if bytes != manifest.download_image_size:
raise ValueError('Bytes Downloaded:"{0}" does not equal '
'manifest image size:"{1}"'
.format(bytes, manifest.download_image_size))
return bytes
def _download_to_unbundlestream(self,
dest_fileobj,
manifest=None,
tools_path=None,
inactivity_timeout=120):
'''
Attempts to iterate through all parts contained in 'manifest' and
download and concatenate each part to euca2ools unbundle stream.
:params manifest: downloadmanifest obj
:tools_path: optional path to euca2ools euca-bundle-stream cmd
'''
download_r = None
download_w = None
monitor_w = None
monitor_r = None
unbundle_ps = None
download_ps = None
wait_threads = []
if not dest_fileobj or isinstance(dest_fileobj, basestring):
raise AttributeError('Dest fileobj must be file like obj, value:'
'"{0}"'.format(str(dest_fileobj)))
manifest = manifest or self.args.manifest
if tools_path is None:
tools_path = self.args.toolspath or ""
unbundle_tool_path = tools_path+'euca-unbundle-stream'
unbundle_ps_args = [unbundle_tool_path,
'--enc-key', str(manifest.enc_key),
'--enc-iv', str(manifest.enc_iv)]
#Enable debug on this subprocess if local arg is set
if self.args.debug:
unbundle_ps_args.append('--debug')
self.log.debug('Running "' + str(unbundle_tool_path) + '" with '
'args:' + ",".join(str(x) for x in unbundle_ps_args))
try:
download_r, download_w = open_pipe_fileobjs()
monitor_r, monitor_w = open_pipe_fileobjs()
#Begin the unbundle portion of this pipeline...
unbundle_ps = subprocess.Popen(unbundle_ps_args,
stdin=download_r,
stdout=monitor_w,
stderr=subprocess.PIPE,
close_fds=True,
bufsize=-1)
download_r.close()
monitor_w.close()
self.log.debug('Starting download parts process to feed unbundle')
#Iterate through all parts in manifest and download to unbundle
download_ps = spawn_process(self._download_parts_pipe_wrapper,
manifest=manifest,
dest_fileobj=download_w)
download_w.close()
self.log.debug('Starting process monitor')
# Process io monitor sits on top/end of unbundle pipe
# It attempts to gather information on the progress of the
# unbundle pipeline and provide information as to the bytes
# written to the destination file obj.
bytes = monitor_subprocess_io(infile=monitor_r,
outfile=dest_fileobj,
sub_stderr=unbundle_ps.stderr,
log_method=self.log.debug,
inactivity_timeout=inactivity_timeout)
self.log.debug('Done with unbundle pipeline...')
if self.args.reportprogress:
stages.report_status('"bytes_unbundled":%d' % bytes)
#Do some final wait/cleanup...
for ps in [unbundle_ps, download_ps]:
if ps:
wait_thread = wait_process_in_thread(ps.pid)
if wait_thread:
wait_threads.append(wait_thread)
# Monitor the subprocess pids in a separate threads, use join()
# timeout to kill processes if needed
for wait_thread in wait_threads:
if wait_thread:
wait_thread.join(timeout=inactivity_timeout)
except Exception, UBE:
if not self.args.reportprogress:
traceback.print_exc()
for ps in [unbundle_ps, download_ps]:
if ps:
try:
ps.terminate()
except:
pass
raise UBE
