def push(self, path, name, tag=None): '''push an image to your Storage. If the collection doesn't exist, it is created. Parameters ========== path: should correspond to an absolute image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.error('%s does not exist.' % path) sys.exit(1) # Parse image names names = parse_image_name(remove_uri(name), tag=tag) # Get the size of the file file_size = os.path.getsize(path) chunk_size = 4 * 1024 * 1024 storage_path = "/%s" % names['storage'] # Create / get the collection collection = self._get_or_create_collection(names['collection']) # The image name is the name followed by tag image_name = os.path.basename(names['storage']) # prepare the progress bar progress = 0 bot.show_progress(progress, file_size, length=35) # Put the (actual) container into the collection with open(path, 'rb') as F: self.conn.put_object(names['collection'], image_name, contents=F.read(), content_type='application/octet-stream') # Finish up bot.show_progress(iteration=file_size, total=file_size, length=35, carriage_return=True) # Newline to finish download sys.stdout.write('\n')
def stream(url, headers, stream_to=None, retry=True): """stream is a get that will stream to file_name. Since this is a worker task, it differs from the client provided version in that it requires headers. """ bot.debug("GET %s" % url) if DISABLE_SSL_CHECK is True: bot.warning("Verify of certificates disabled! ::TESTING USE ONLY::") # Ensure headers are present, update if not response = requests.get(url, headers=headers, verify=not DISABLE_SSL_CHECK, stream=True) # Deal with token if necessary if response.status_code == 401 and retry is True: headers = update_token(response, headers) return stream(url, headers, stream_to, retry=False) if response.status_code == 200: # Keep user updated with Progress Bar content_size = None if "Content-Length" in response.headers: progress = 0 content_size = int(response.headers["Content-Length"]) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, "wb") as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress( iteration=progress, total=content_size, length=35, carriage_return=False, ) # Newline to finish download sys.stdout.write("\n") return stream_to bot.exit("Problem with stream, response %s" % response.status_code)
def stream(url, headers, stream_to=None, retry=True): '''stream is a get that will stream to file_name. Since this is a worker task, it differs from the client provided version in that it requires headers. ''' bot.debug("GET %s" % url) if DISABLE_SSL_CHECK is True: bot.warning('Verify of certificates disabled! ::TESTING USE ONLY::') # Ensure headers are present, update if not response = requests.get(url, headers=headers, verify=not DISABLE_SSL_CHECK, stream=True) # If we get permissions error, one more try with updated token if response.status_code in [401, 403]: headers = update_token(headers) return stream(url, headers, stream_to, retry=False) # Successful Response elif response.status_code == 200: # Keep user updated with Progress Bar content_size = None if 'Content-Length' in response.headers: progress = 0 content_size = int(response.headers['Content-Length']) bot.show_progress(progress,content_size,length=35) chunk_size = 1 << 20 with open(stream_to,'wb') as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress+=chunk_size bot.show_progress(iteration=progress, total=content_size, length=35, carriage_return=False) # Newline to finish download sys.stdout.write('\n') return stream_to bot.error("Problem with stream, response %s" %(response.status_code)) sys.exit(1)
def stream_response(self, response, stream_to=None, show_progress=True): """ stream response is one level higher up than stream, starting with a response object and then performing the stream without making the requests.get. The expectation is that the request was successful (status code 20*). show_progress: boolean to show progress bar Parameters ========== response: a response that is ready to be iterated over to download in streamed chunks stream_to: the file to stream to """ if response.status_code == 200: if show_progress is False: bot.quiet = True # Keep user updated with Progress Bar, if not quiet content_size = None if "Content-Length" in response.headers: progress = 0 content_size = int(response.headers["Content-Length"]) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, "wb") as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress( iteration=progress, total=content_size, length=35, carriage_return=False, ) # Newline to finish download sys.stdout.write("\n") return stream_to bot.exit("Problem with stream, response %s" % (response.status_code))
def stream_response(self, response, stream_to=None): ''' stream response is one level higher up than stream, starting with a response object and then performing the stream without making the requests.get. The expectation is that the request was successful (status code 20*). Parameters ========== response: a response that is ready to be iterated over to download in streamed chunks stream_to: the file to stream to ''' if response.status_code == 200: # Keep user updated with Progress Bar content_size = None if 'Content-Length' in response.headers: progress = 0 content_size = int(response.headers['Content-Length']) bot.show_progress(progress, content_size, length=35) chunk_size = 1 << 20 with open(stream_to, 'wb') as filey: for chunk in response.iter_content(chunk_size=chunk_size): filey.write(chunk) if content_size is not None: progress += chunk_size bot.show_progress(iteration=progress, total=content_size, length=35, carriage_return=False) # Newline to finish download sys.stdout.write('\n') return stream_to bot.error("Problem with stream, response %s" % (response.status_code)) sys.exit(1)
def run(self, func, tasks, func2=None): '''run will send a list of tasks, a tuple with arguments, through a function. the arguments should be ordered correctly. :param func: the function to run with multiprocessing.pool :param tasks: a list of tasks, each a tuple of arguments to process :param func2: filter function to run result from func through (optional) ''' # Keep track of some progress for the user progress = 1 total = len(tasks) # if we don't have tasks, don't run if len(tasks) == 0: return # If two functions are run per task, double total jobs if func2 is not None: total = total * 2 finished = [] level1 = [] results = [] try: prefix = "[%s/%s]" % (progress, total) bot.show_progress(0, total, length=35, prefix=prefix) pool = multiprocessing.Pool(self.workers, init_worker) self.start() for task in tasks: result = pool.apply_async(multi_wrapper, multi_package(func, [task])) results.append(result) level1.append(result._job) while len(results) > 0: result = results.pop() result.wait() bot.show_progress(progress, total, length=35, prefix=prefix) progress += 1 prefix = "[%s/%s]" % (progress, total) # Pass the result through a second function? if func2 is not None and result._job in level1: result = pool.apply_async( multi_wrapper, multi_package(func2, [(result.get(), )])) results.append(result) else: finished.append(result.get()) self.end() pool.close() pool.join() except (KeyboardInterrupt, SystemExit): bot.error("Keyboard interrupt detected, terminating workers!") pool.terminate() sys.exit(1) except Exception as e: bot.error(e) return finished
def push(self, path, name, tag=None): '''push an image to your Dropbox Parameters ========== path: should correspond to an absolute image path (or derive it) name: should be the complete uri that the user has requested to push. tag: should correspond with an image tag. This is provided to mirror Docker if the image is less than 150MB, the standard file_upload is used. If larger, the image is uploaded in chunks with a progress bar. ''' path = os.path.abspath(path) bot.debug("PUSH %s" % path) if not os.path.exists(path): bot.exit('%s does not exist.' % path) # here is an exampole of getting metadata for a container names = parse_image_name(remove_uri(name), tag=tag) # Get the size of the file file_size = os.path.getsize(path) chunk_size = 4 * 1024 * 1024 storage_path = "/%s" % names['storage'] # This is MB # image_size = os.path.getsize(path) >> 20 # prepare the progress bar progress = 0 bot.show_progress(progress, file_size, length=35) # If image is smaller than 150MB, use standard upload with open(path, 'rb') as F: if file_size <= chunk_size: self.dbx.files_upload(F.read(), storage_path) # otherwise upload in chunks else: start = self.dbx.files_upload_session_start(F.read(chunk_size)) cursor = dropbox.files.UploadSessionCursor( session_id=start.session_id, offset=F.tell()) commit = dropbox.files.CommitInfo(path=storage_path) while F.tell() < file_size: progress += chunk_size # Finishing up the file, less than chunk_size to go if file_size - F.tell() <= chunk_size: self.dbx.files_upload_session_finish( F.read(chunk_size), cursor, commit) # Finishing up the file, less than chunk_size to go else: self.dbx.files_upload_session_append( F.read(chunk_size), cursor.session_id, cursor.offset) cursor.offset = F.tell() # Update the progress bar bot.show_progress(iteration=progress, total=file_size, length=35, carriage_return=False) # Finish up bot.show_progress(iteration=file_size, total=file_size, length=35, carriage_return=True) # Newline to finish download sys.stdout.write('\n')