class TestStreamingIteratorWithLargeChunks(unittest.TestCase): def setUp(self): self.letters = ['a', 'b', 'c', 'd', 'e'] self.chunks = (letter * 2000 for letter in self.letters) self.size = 5 * 2000 self.uploader = StreamingIterator(self.size, self.chunks) def test_returns_the_amount_requested(self): chunk_size = 1000 bytes_read = 0 while True: b = self.uploader.read(chunk_size) if not b: break assert len(b) == chunk_size bytes_read += len(b) assert bytes_read == self.size def test_returns_all_of_the_bytes(self): chunk_size = 8192 bytes_read = 0 while True: b = self.uploader.read(chunk_size) if not b: break bytes_read += len(b) assert bytes_read == self.size
class TestStreamingIteratorWithLargeChunks(object): @pytest.fixture(autouse=True) def setup(self, get_iterable): self.letters = [b'a', b'b', b'c', b'd', b'e'] self.chunks = (letter * 2000 for letter in self.letters) self.size = 5 * 2000 self.uploader = StreamingIterator(self.size, get_iterable(self.chunks)) def test_returns_the_amount_requested(self): chunk_size = 1000 bytes_read = 0 while True: b = self.uploader.read(chunk_size) if not b: break assert len(b) == chunk_size bytes_read += len(b) assert bytes_read == self.size def test_returns_all_of_the_bytes(self): chunk_size = 8192 bytes_read = 0 while True: b = self.uploader.read(chunk_size) if not b: break bytes_read += len(b) assert bytes_read == self.size
def handle_uploaded_file(file, callback): """ :param file: Of type UploadedFile """ generator = file.chunks() # Create generator size = file.size # Get size # content_type = file.content_type # Get the content-type of the data # def my_callback(monitor): # bar.show(monitor.bytes_read) streamer = StreamingIterator(size, generator, callback=callback) # encoder_len = streamer.size # bar = ProgressBar(expected_size=encoder_len, filled_char='=') # r = requests.put('http://localhost:4444/', data=streamer, # headers={'Content-Type': content_type}) http_url = 'http://localhost:4444/' params = {'dsName': 'BigDisk', 'dcPath': 'Datacenter'} headers = {'Content-Type': 'application/octet-stream'} cookie = { 'chocolate_chip': ' "cd9c342ce3df6d2944d45132c6feca1e161f39bc"; $Path=/' } request = requests.put(http_url, params=params, data=streamer, headers=headers, cookies=cookie, verify=False)
class TestStreamingIterator(object): @pytest.fixture(autouse=True) def setup(self, get_iterable): self.chunks = [b'here', b'are', b'some', b'chunks'] self.size = 17 self.uploader = StreamingIterator(self.size, get_iterable(self.chunks)) def test_read_returns_all_chunks_in_one(self): assert self.uploader.read() == b''.join(self.chunks) def test_read_returns_empty_string_after_exhausting_the_iterator(self): for i in range(0, 4): self.uploader.read(8192) assert self.uploader.read() == b'' assert self.uploader.read(8192) == b''
class TestStreamingIterator(unittest.TestCase): def setUp(self): self.chunks = [b'here', b'are', b'some', b'chunks'] self.iterator = iter(self.chunks) self.size = 17 self.uploader = StreamingIterator(self.size, self.iterator) def test_read_returns_all_chunks_in_one(self): assert self.uploader.read() == b''.join(self.chunks) def test_read_returns_empty_string_after_exhausting_the_iterator(self): for i in range(0, 4): self.uploader.read(8192) assert self.uploader.read() == b'' assert self.uploader.read(8192) == b''
def upload(self, local_file_path, rest_resource_path): """ Add log message @type local_file_path: string @param local_file_path: Local file path @type rest_resource_path: string @param rest_resource_path: Resource file path (without leading slash!) @rtype: string @return: Relative HDFS path """ file_resource_uri = self.rest_endpoint.get_resource_uri( rest_resource_path) if local_file_path is not None: # with open(local_file_path, 'r') as f: # strip path and extension from absolute file path to get filename filename = local_file_path.rpartition('/')[2] chunks = FileBinaryDataChunks(local_file_path, 65536, self.progress_reporter).chunks() file_size = os.path.getsize(local_file_path) streamer = StreamingIterator(file_size, chunks) content_type = 'multipart/form-data' s = requests.Session() s.auth = ('admin', 'admin') r = s.put(file_resource_uri.format(filename), data=streamer, headers={'Content-Type': content_type}) return ResponseWrapper(success=True, response=r) else: return ResponseWrapper(success=False)
def stream_serializer(self, data, request_schema=None): """ Stream Serializer will compress and stream :param data: :return: """ zipped_payload = self._compress(data) logging.info('compressed payload size is {} bytes'.format( self.get_size(zipped_payload))) return rest_config.MIME_JSON,\ StreamingIterator(size=len(zipped_payload), iterator=self._gen(zipped_payload))
def post_to_bbg_transport(self, req): """ Call BT api [POST] and keep polling until response has complete status or times out :param req: :return: """ logging.info('Submitting to BT...') json_payload = self.get_request_item(req).to_json() logging.info('payload size is {} bytes for {} requests'.format( self.get_size(json_payload), len(req))) buf = io.BytesIO() with gzip.GzipFile(fileobj=buf, mode='wb') as f: # todo: fix this to use pypimco serialize.py json.dump(json_payload, f) zipped_payload = buf.getvalue() logging.info( 'compressed payload size is {} bytes for {} requests'.format( self.get_size(zipped_payload), len(req))) payload = zipped_payload def gen(data): for i in range(0, len(data), 1000): yield data[i:i + 1000] streamer = StreamingIterator(size=len(payload), iterator=gen(payload)) try: response = self._post(data=streamer, headers={ 'Content-Type': 'application/json', 'Content-Encoding': 'gzip' }, is_stream=True) except ClientException as ex: status = ex.detail_json.get('status') logging.exception('BBG Transport API call failed: ' + ex.message) if status == 403: raise BbgTransportForbidden('Forbidden: ' + ex.detail_json.get('msg')) except Exception as ex: logging.exception('BBG Transport Server Error: ' + ex.message) raise else: return self.handle_response(response)
def setUp(self): self.chunks = [b'here', b'are', b'some', b'chunks'] self.iterator = iter(self.chunks) self.size = 17 self.uploader = StreamingIterator(self.size, self.iterator)
def main(): # parse arguments args = parse_args(sys.argv[1:]) # parse config file gns3_proxy_config.ini config = configparser.ConfigParser() config.read_file(open(args.config_file)) config.read(args.config_file) logging.basicConfig( level=getattr(logging, args.log_level), format= '%(asctime)s - %(levelname)s - %(funcName)s:%(lineno)d - %(message)s') # get backend_user # # description: Username to use to access backend GNS3 server # default: admin if config.get('proxy', 'backend_user'): backend_user = config.get('proxy', 'backend_user') else: backend_user = "******" # get backend_password # # description: Password to use to access backend GNS3 server # default: password if config.get('proxy', 'backend_password'): backend_password = config.get('proxy', 'backend_password') else: backend_password = "******" # get backend_port # # description: TCP port to use to access backend GNS3 server # default: 3080 if config.get('proxy', 'backend_port'): backend_port = config.getint('proxy', 'backend_port') else: backend_port = 3080 # Compute Image Backend image_backend_url = '/compute/' + args.image_type + '/images' # Alternate location for image access, used for upload by the GNS3 client, but download (GET) throws an error alt_image_backend_url = '/computes/local/' + args.image_type + '/images' # read servers from config if config.items('servers'): config_servers = dict() server_items = config.items('servers') for server, value in server_items: try: ip_address(value) except ValueError: logger.fatal( "server config %s is not a valid IP address (e.g., 1.2.3.4)" % value) raise ProxyError() config_servers[server] = value else: config_servers = None logger.debug("Config backend_user: %s" % backend_user) logger.debug("Config backend_password: %s" % backend_password) logger.debug("Config backend_port: %s" % backend_port) logger.debug("Config servers: %s" % config_servers) try: username = backend_user password = backend_password # get source server IP if args.source_server in config_servers: src_server = config_servers[args.source_server] logger.debug("Source server will be %s:%s" % (src_server, backend_port)) else: logger.fatal("Source server not found in config.") raise ProxyError() base_src_api_url = "http://" + src_server + ":" + str( backend_port) + "/v2" logger.debug("Searching source images") images = list() url = base_src_api_url + image_backend_url r = requests.get(url, auth=(username, password)) if not r.status_code == 200: logger.fatal("Could not list images.") logger.debug("Status code: " + str(r.status_code) + " Text:" + r.text) raise ProxyError() else: image_results = json.loads(r.text) for image in image_results: if re.fullmatch(args.image_filename, image['filename']): logger.debug('matched image: %s' % image['filename']) images.append(image) if len(images) == 0: logger.fatal("Specified image not found.") raise ProxyError() for image in images: image_filename = image['filename'] print("#### Replicating image: %s from server: %s (%s)" % (image_filename, args.source_server, src_server)) # target handling # Try to find match for target server in config target_servers = list() if len(config_servers) > 0: for key in config_servers: if re.fullmatch(args.target_server, key): logger.debug( "Target server found: %s (%s) using provided match: %s" % (key, config_servers[key], args.target_server)) if key == args.source_server: logger.debug( "Target server %s is the same as the source server %s . Filtered out." % (key, args.source_server)) else: target_servers.append({ 'name': key, 'address': config_servers[key] }) else: logger.fatal( "No servers defined in config. Could not select target server." ) raise ProxyError() if len(target_servers) == 0: logger.fatal( "No target servers found using match: %s. Could not select target server." % args.target_server) raise ProxyError() for target_server in target_servers: target_server_name = target_server['name'] target_server_address = target_server['address'] logger.debug(" Replicating image: %s to server: %s" % (image_filename, target_server_name)) base_dst_api_url = "http://" + target_server_address + ":" + str( backend_port) + "/v2" logger.debug("Checking if target image exists...") url = base_dst_api_url + image_backend_url r = requests.get(url, auth=(username, password)) if r.status_code == 200: target_image_exists = False target_image_md5sum = '' target_image_to_delete = '' target_image_results = json.loads(r.text) for target_image in target_image_results: if re.fullmatch(image_filename, target_image['filename']): logger.debug( "image: %s already exists on server %s" % (target_image['filename'], target_server_name)) if target_image_exists: logger.fatal( "Multiple images matched %s on server %s. " "Import can only be used for single image." % (image_filename, target_server_name)) raise ProxyError() else: target_image_exists = True target_image_md5sum = target_image['md5sum'] target_image_to_delete = image['filename'] if target_image_exists: if args.force: # deleting image # print("Deleting existing image %s on server: %s" # % (image_to_delete, config_servers[server])) # url = base_dst_api_url + image_backend_url + '/' + image_to_delete # r = requests.delete(url, auth=(username, password)) # if not r.status_code == 204: # if r.status_code == 404: # logger.debug("Image did not exist before, not deleted") # else: # logger.fatal("unable to delete image") # raise ProxyError() logger.debug( "image: %s (%s) already exists on server %s. Overwriting it." % (image_filename, target_image_to_delete, target_server_name)) elif image['md5sum'] == target_image_md5sum: logger.debug( "image: %s (%s) already exists on server %s, skipping transfer. " "Use --force to overwrite it during import." % (image_filename, target_image_to_delete, target_server_name)) continue else: logger.fatal( "image: %s (%s) already exists on server, but the md5sum does not match." "on target %s. Use --force to overwrite it during import." % (image_filename, target_image_to_delete, target_server_name)) raise ProxyError() # export source image logger.debug("Opening source image") url = base_src_api_url + image_backend_url + '/' + image_filename r_export = requests.get(url, stream=True, auth=(username, password)) if not r_export.status_code == 200: logger.fatal( "Unable to export image from source server.") logger.debug("Status code: " + str(r.status_code) + " Text:" + r.text) raise ProxyError() start_timestamp = int(round(time.time())) def generate_chunk(): transferred_length_upload = 0 prev_transferred_length_upload = 0 next_percentage_to_print_upload = 0 prev_timestamp_upload = int(round(time.time() * 1000)) for in_chunk in r_export.iter_content( chunk_size=args.buffer): if in_chunk: yield in_chunk transferred_length_upload += len(in_chunk) if total_length > 0: transferred_percentage_upload = int( (transferred_length_upload / total_length) * 100) else: transferred_percentage_upload = 0 if transferred_percentage_upload >= next_percentage_to_print_upload: curr_timestamp_upload = int( round(time.time() * 1000)) duration_upload = curr_timestamp_upload - prev_timestamp_upload delta_length_upload = \ transferred_length_upload - prev_transferred_length_upload if duration_upload > 0: rate_upload = delta_length_upload / ( duration_upload / 1000) else: rate_upload = 0 prev_timestamp_upload = curr_timestamp_upload prev_transferred_length_upload = transferred_length_upload print( "Replicating to %s (%s) ... %d%% (%.3f MB/s)" % (target_server_name, target_server_address, transferred_percentage_upload, (rate_upload / 1000000))) next_percentage_to_print_upload = next_percentage_to_print_upload + 5 # import target image logger.debug("Opening target image") url = base_dst_api_url + alt_image_backend_url + '/' + image_filename total_length = int(r_export.headers.get('content-length')) # r_import = requests.post(url, auth=(username, password), data=generate_chunk()) streamer = StreamingIterator(total_length, generate_chunk()) r_import = requests.post(url, auth=(username, password), data=streamer) if not r_import.status_code == 200: if r_import.status_code == 403: logger.fatal( "Forbidden to import image on target server.") logger.debug("Status code: " + str(r.status_code) + " Text:" + r.text) raise ProxyError() else: logger.fatal( "Unable to import image on target server.") logger.debug("Status code: " + str(r.status_code) + " Text:" + r.text) raise ProxyError() else: end_timestamp = int(round(time.time())) print( "#### image %s (%s bytes) replicated from server: %s to server: %s (in %i secs)" % (image_filename, total_length, args.source_server, target_server_name, (end_timestamp - start_timestamp))) else: logger.fatal( "Could not get status of images from server %s." % target_server_name) logger.debug("Status code: " + str(r.status_code) + " Text:" + r.text) raise ProxyError() print("Done.") except KeyboardInterrupt: pass
def setup(self, get_iterable): self.chunks = [b'here', b'are', b'some', b'chunks'] self.size = 17 self.uploader = StreamingIterator(self.size, get_iterable(self.chunks))
def setUp(self): self.letters = ['a', 'b', 'c', 'd', 'e'] self.chunks = (letter * 2000 for letter in self.letters) self.size = 5 * 2000 self.uploader = StreamingIterator(self.size, self.chunks)
def setup(self, get_iterable): self.letters = [b'a', b'b', b'c', b'd', b'e'] self.chunks = (letter * 2000 for letter in self.letters) self.size = 5 * 2000 self.uploader = StreamingIterator(self.size, get_iterable(self.chunks))
import requests import os from os import path #pip install requests-toolbelt from requests_toolbelt.streaming_iterator import StreamingIterator import os # To Upload a file Just Change This Line: filePath = "C:\\Uploads\\Source\\4M.dat" # This is URL of file-upload-webservice serverURL="http://localhost:8080/upload-webservice/bigfileupload" #retrieve fileName from path, this will be sent as a header fileName = path.split(filePath)[1] size =os.path.getsize(filePath); headers = {"File-Name": fileName, "Content-Type": "application/octet-stream"} resp = requests.post(serverURL, headers=headers, data=StreamingIterator(size, open(filePath, 'r+b',1024))) #print result print resp.status_code print resp.text
auth = HTTPBasicAuth(config['Server']['user'], config['Server']['password']) url = "http://{}/v2/compute/qemu/images".format(gns3_server) result = requests.get(url, auth=auth) result.raise_for_status() images = result.json() image_filenames = [j['filename'] for j in images] if args.ls: for image in images: print(image['filename']) exit(0) if args.filename in image_filenames and not args.overwrite: print("Won't overwrite existing image") exit(1) if args.filename: with open(args.filename, 'rb') as f: url = "http://{}/v2/compute/qemu/images/{}".format( gns3_server, args.filename) size = os.stat(args.filename).st_size streamer = StreamingIterator(size, f) result = requests.post( url, auth=auth, data=streamer, headers={'Content-Type': 'application/octet-stream'}) result.raise_for_status()