def _download_parts_to_fileobj(self, manifest, dest_fileobj): ''' Attempts to iterate through all parts contained in 'manifest' and download and concatenate each part to 'dest_fileobj'. If the manifest contains the intended image size, and the resulting bytes downloaded does not match this size, ValueError is raised. :param manifest: downloadmanifest obj :param dest_fileobj: file like object to write downloaded parts to :returns bytes downloaded ''' bytes = 0 for part_index in xrange(0, manifest.part_count): part = manifest.get_part_by_index(part_index) self.log.debug('Downloading part#:' + str(part.part_index)) bytes += part.download(dest_fileobj=dest_fileobj) or 0 self.log.debug('Wrote bytes:' + str(bytes) + "/" + str(manifest.download_image_size) + ", digest:" + str(part.written_digest)) if self.args.reportprogress: stages.report_status('"bytes_downloaded":%d' % bytes) if manifest.download_image_size is not None and not self.args.skipsizevalidation: if bytes != manifest.download_image_size: raise ValueError('Bytes Downloaded:"{0}" does not equal ' 'manifest image size:"{1}"' .format(bytes, manifest.download_image_size)) return bytes
def _download_to_unbundlestream(self, dest_fileobj, manifest=None, tools_path=None, inactivity_timeout=120): ''' Attempts to iterate through all parts contained in 'manifest' and download and concatenate each part to euca2ools unbundle stream. :params manifest: downloadmanifest obj :tools_path: optional path to euca2ools euca-bundle-stream cmd ''' download_r = None download_w = None monitor_w = None monitor_r = None unbundle_ps = None download_ps = None wait_threads = [] if not dest_fileobj or isinstance(dest_fileobj, basestring): raise AttributeError('Dest fileobj must be file like obj, value:' '"{0}"'.format(str(dest_fileobj))) manifest = manifest or self.args.manifest if tools_path is None: tools_path = self.args.toolspath or "" unbundle_tool_path = tools_path+'euca-unbundle-stream' unbundle_ps_args = [unbundle_tool_path, '--enc-key', str(manifest.enc_key), '--enc-iv', str(manifest.enc_iv)] #Enable debug on this subprocess if local arg is set if self.args.debug: unbundle_ps_args.append('--debug') self.log.debug('Running "' + str(unbundle_tool_path) + '" with ' 'args:' + ",".join(str(x) for x in unbundle_ps_args)) try: download_r, download_w = open_pipe_fileobjs() monitor_r, monitor_w = open_pipe_fileobjs() #Begin the unbundle portion of this pipeline... unbundle_ps = subprocess.Popen(unbundle_ps_args, stdin=download_r, stdout=monitor_w, stderr=subprocess.PIPE, close_fds=True, bufsize=-1) download_r.close() monitor_w.close() self.log.debug('Starting download parts process to feed unbundle') #Iterate through all parts in manifest and download to unbundle download_ps = spawn_process(self._download_parts_pipe_wrapper, manifest=manifest, dest_fileobj=download_w) download_w.close() self.log.debug('Starting process monitor') # Process io monitor sits on top/end of unbundle pipe # It attempts to gather information on the progress of the # unbundle pipeline and provide information as to the bytes # written to the destination file obj. bytes = monitor_subprocess_io(infile=monitor_r, outfile=dest_fileobj, sub_stderr=unbundle_ps.stderr, log_method=self.log.debug, inactivity_timeout=inactivity_timeout) self.log.debug('Done with unbundle pipeline...') if self.args.reportprogress: stages.report_status('"bytes_unbundled":%d' % bytes) #Do some final wait/cleanup... for ps in [unbundle_ps, download_ps]: if ps: wait_thread = wait_process_in_thread(ps.pid) if wait_thread: wait_threads.append(wait_thread) # Monitor the subprocess pids in a separate threads, use join() # timeout to kill processes if needed for wait_thread in wait_threads: if wait_thread: wait_thread.join(timeout=inactivity_timeout) except Exception, UBE: if not self.args.reportprogress: traceback.print_exc() for ps in [unbundle_ps, download_ps]: if ps: try: ps.terminate() except: pass raise UBE