def main(self): sources = list(self.args['sources']) label_template = build_progressbar_label_template(sources) for index, source_filename in enumerate(sources, 1): if self.args.get('literal_dest', False): (bucket, __, keyname) = self.args['dest'].partition('/') if not keyname: raise ArgumentError('destination must contain a key name') else: (bucket, __, prefix) = self.args['dest'].partition('/') keyname = prefix + os.path.basename(source_filename) self.path = bucket + '/' + keyname self.headers['Content-Length'] = os.path.getsize(source_filename) self.headers.pop('Content-Type', None) if self.args.get('acl'): self.headers['x-amz-acl'] = self.args['acl'] if self.args.get('guess_mime_type', False): mtype = mimetypes.guess_type(source_filename) if mtype: self.headers['Content-Type'] = mtype self.log.info('uploading %s to %s', source_filename, self.path) with self._lock: self.last_upload_error = None with open(source_filename) as source: upload_thread = threading.Thread( target=self.try_send, args=(source, ), kwargs={'retries_left': self.args['retries']}) # The upload thread is daemonic so ^C will kill the program # more cleanly. upload_thread.daemon = True upload_thread.start() label = label_template.format( index=index, fname=os.path.basename(source_filename)) pbar = self.get_progressbar( label=label, maxval=os.path.getsize(source_filename)) pbar.start() while upload_thread.is_alive(): pbar.update(source.tell()) time.sleep(0.01) pbar.finish() upload_thread.join() with self._lock: if self.last_upload_error is not None: # pylint: disable=E0702 raise self.last_upload_error
def main(self): sources = list(self.args['sources']) label_template = build_progressbar_label_template(sources) for index, source_filename in enumerate(sources, 1): if self.args.get('literal_dest', False): (bucket, __, keyname) = self.args['dest'].partition('/') if not keyname: raise ArgumentError('destination must contain a key name') else: (bucket, __, prefix) = self.args['dest'].partition('/') keyname = prefix + os.path.basename(source_filename) self.path = bucket + '/' + keyname self.headers['Content-Length'] = os.path.getsize(source_filename) self.headers.pop('Content-Type', None) if self.args.get('acl'): self.headers['x-amz-acl'] = self.args['acl'] if self.args.get('guess_mime_type', False): mtype = mimetypes.guess_type(source_filename) if mtype: self.headers['Content-Type'] = mtype self.log.info('uploading %s to %s', source_filename, self.path) with self._lock: self.last_upload_error = None with open(source_filename) as source: upload_thread = threading.Thread(target=self.try_send, args=(source,), kwargs={'retries_left': self.args['retries']}) # The upload thread is daemonic so ^C will kill the program # more cleanly. upload_thread.daemon = True upload_thread.start() label = label_template.format(index=index, fname=os.path.basename(source_filename)) pbar = self.get_progressbar(label=label, maxval=os.path.getsize(source_filename)) pbar.start() while upload_thread.is_alive(): pbar.update(source.tell()) time.sleep(0.01) pbar.finish() upload_thread.join() with self._lock: if self.last_upload_error is not None: # pylint: disable=E0702 raise self.last_upload_error
def main(self): opath = self.args['opath'] label_template = build_progressbar_label_template(self.args['paths']) if opath.endswith('/') and not os.path.isdir(opath): # Ends with '/' and does not exist -> create it os.mkdir(opath) if os.path.isdir(opath): # Download one per directory for index, path in enumerate(self.args['paths'], 1): ofile_name = os.path.join(opath, path.rsplit('/', 1)[-1]) self.path = path response = self.send() if 'Content-Length' in response.headers: maxval = int(response.headers['Content-Length']) else: maxval = None label = label_template.format(index=index, fname=path) pbar = self.get_progressbar(label=label, maxval=maxval) pbar.start() with open(ofile_name, 'w') as ofile: for chunk in response.iter_content(chunk_size=16384): ofile.write(chunk) pbar.update(ofile.tell()) ofile.flush() pbar.finish() else: # Download everything to one file with open(opath, 'w') as ofile: for index, path in enumerate(self.args['paths'], 1): self.path = path response = self.send() bytes_written = 0 if 'Content-Length' in response.headers: maxval = int(response.headers['Content-Length']) else: maxval = None label = label_template.format(index=index, fname=path) pbar = self.get_progressbar(label=label, maxval=maxval) pbar.start() for chunk in response.iter_content(chunk_size=16384): ofile.write(chunk) bytes_written += len(chunk) pbar.update(bytes_written) pbar.finish() ofile.flush()
def main(self): opath = self.args['opath'] label_template = build_progressbar_label_template(self.args['paths']) if opath.endswith('/') and not os.path.isdir(opath): # Ends with '/' and does not exist -> create it os.mkdir(opath) if os.path.isdir(opath): # Download one per directory for index, path in enumerate(self.args['paths'], 1): ofile_name = os.path.join(opath, path.rsplit('/', 1)[-1]) self.path = path response = self.send() if 'Content-Length' in response.headers: maxval = int(response.headers['Content-Length']) else: maxval = None label = label_template.format(index=index, fname=path) pbar = self.get_progressbar(label=label, maxval=maxval) pbar.start() with open(ofile_name, 'w') as ofile: for chunk in response.iter_content(chunk_size=16384): ofile.write(chunk) pbar.update(ofile.tell()) ofile.flush() pbar.finish() else: # Download everything to one file with open(opath, 'w') as ofile: for index, path in enumerate(self.args['paths'], 1): self.path = path response = self.send() bytes_written = 0 if 'Content-Length' in response.headers: maxval = int(response.headers['Content-Length']) else: maxval = None label = label_template.format(index=index, fname=path) pbar = self.get_progressbar(label=label, maxval=maxval) pbar.start() for chunk in response.iter_content(chunk_size=16384): ofile.write(chunk) bytes_written += len(chunk) pbar.update(bytes_written) pbar.finish() ofile.flush()
def main(self): sha1_dict = {} opath = self.args['opath'] paths = self.args['paths'] show_progress = self.args.get('show_progress') self.log.debug('GOT SHOW PROGRESS: ' + str(show_progress)) label = None if show_progress: label_template = build_progressbar_label_template(paths) if (opath) and (os.path.isdir(opath) or opath.endswith('/')): #Download paths to individual files under provided directory... if not os.path.isdir(opath): # Ends with '/' and does not exist -> create it os.mkdir(opath) # Download one per directory for index, path in enumerate(paths, 1): ofile_name = os.path.join(opath, path.rsplit('/', 1)[-1]) if show_progress: label = label_template.format(index=index, fname=path) with open(ofile_name, 'w') as ofile: sha1sum = (self._download_to_fileobj( path=path, outfile=ofile, show_progress=show_progress, pbar_label=label)) sha1_dict[path] = sha1sum else: # Download everything to one file ofile = self.args.get('fileobj') or open(opath, 'w') try: for index, path in enumerate(paths, 1): if show_progress: label = label_template.format(index=index, fname=path) sha1sum = (self._download_to_fileobj( path=path, outfile=ofile, show_progress=show_progress, pbar_label=label)) sha1_dict[path] = sha1sum finally: #only close the file if it was opened within this method... if opath and ofile: ofile.close() return sha1_dict