def setUp(self): if "agent_remote_stress_test" not in config: raise SkipTest() virtualenv_home = sysconfig.get_config_vars("exec_prefix")[0] log_path = os.path.join(virtualenv_home, "logs") mkdir_p(log_path) log_file = os.path.join(log_path, self.id() + ".log") logger = logging.getLogger() handler = logging.FileHandler(log_file, mode='w', encoding=None, delay=False) logger.addHandler(handler) logging.info("PY_STRESS_START:%d" % int(time.time() * 1000)) if "host" in config["agent_remote_stress_test"]: self.hosts = [config["agent_remote_stress_test"]["host"]] elif "hostfile" in config["agent_remote_stress_test"]: with open(config["agent_remote_stress_test"]["hostfile"]) as f: self.hosts = f.read().splitlines() self.vms_per_thread = int(config["agent_remote_stress_test"].get( "vms_per_thread", self.DEFAULT_VMS_PER_THREAD)) self.threads_per_host = int(config["agent_remote_stress_test"].get( "threads_per_host", self.DEFAULT_THREADS_PER_HOST)) self.place_to_create_ratio = int( config["agent_remote_stress_test"].get( "place_to_create_ratio", self.DEFAULT_PLACE_TO_CREATE_RATIO)) self.clear()
def create_vm(self, vm_id, create_spec): """Create a new Virtual Maching given a VM create spec. :param vm_id: The Vm id :type vm_id: string :param create_spec: The VM spec builder :type ConfigSpec :raise: VmAlreadyExistException """ folder = self.vim_client.vm_folder resource_pool = self.vim_client.root_resource_pool # sanity check since VIM does not prevent this try: if self.vim_client.get_vm_in_cache(vm_id): raise VmAlreadyExistException("VM already exists") except VmNotFoundException: pass # The scenario of the vm creation at ESX where intermediate directory # has to be created has not been well exercised and is known to be # racy and not informative on failures. So be defensive and proactively # create the intermediate directory ("/vmfs/volumes/<dsid>/vms/xy"). vm_parent_dir = datastore_to_os_path(create_spec.files.vmPathName) if os.path.exists(vm_parent_dir): self._logger.debug("Parent directory %s exists" % vm_parent_dir) else: mkdir_p(vm_parent_dir) task = folder.CreateVm(create_spec, resource_pool, None) self.vim_client.wait_for_task(task) self.vim_client.wait_for_vm_create(vm_id)
def _make_image_dir(self, datastore, image_id, parent_folder_name=IMAGE_FOLDER_NAME): path = os.path.dirname( os_vmdk_path( datastore, image_id, parent_folder_name)) if os.path.isdir(path): return # On shared volumes makedirs can fail with not found in rare corner # cases if two directory creates collide. Just retry in that case for attempt in range(1, self.NUM_MAKEDIRS_ATTEMPTS+1): try: mkdir_p(path) except OSError: self._logger.debug("Retrying (%u) while creating %s" % (attempt, path)) if attempt == self.NUM_MAKEDIRS_ATTEMPTS: raise else: continue # Directory got created, stop the for loop break
def _make_image_dir(self, datastore, image_id, parent_folder_name=IMAGE_FOLDER_NAME_PREFIX): path = os.path.dirname( os_vmdk_path( datastore, image_id, parent_folder_name)) if os.path.isdir(path): return # On shared volumes makedirs can fail with not found in rare corner # cases if two directory creates collide. Just retry in that case for attempt in range(1, self.NUM_MAKEDIRS_ATTEMPTS+1): try: mkdir_p(path) except OSError: self._logger.debug("Retrying (%u) while creating %s" % (attempt, path)) if attempt == self.NUM_MAKEDIRS_ATTEMPTS: raise else: continue # Directory got created, stop the for loop break
def spawn_agents(self): os.setpgrp() if os.path.exists(MultiAgent.IMAGE_PATH): rm_rf(MultiAgent.IMAGE_PATH) mkdir_p(MultiAgent.IMAGE_PATH) for index in xrange(self.agent_count): # Create config sub-dirs config_path = self.create_config(index) log_file = self.get_log_file(index) # Set up argument list args = self.argv[:] args.append("--multi-agent-id") args.append(str(index)) args.append("--config-path") args.append(config_path) args.append("--logging-file") args.append(log_file) args.append("--port") args.append(str(self.agent_port + index)) args.append("--datastores") args.append("DataStore-" + str(index).zfill(4)) args.append("--vm-network") args.append("Network-" + str(index).zfill(4)) command = '' for arg in args: command += ' ' command += arg proc = subprocess.Popen(args) self.procs.append(proc) signal.signal(signal.SIGTERM, self._signal_handler) signal.pause() self.cleanup() sys.exit(0)
def __init__(self, hypervisor, disk_path=None, capacity_map=None): self.hypervisor = hypervisor self.capacity_map = capacity_map self.disk_path = disk_path or mkdtemp(delete=True) self._logger = logging.getLogger(__name__) self._logger.debug("Fake disk manager uses tempdir %s" % self.disk_path) mkdir_p(self.disk_path)
def move_disk(self, source_datastore, source, dest_datastore, dest): self._assert_exists(source, source_datastore) self._assert_not_exists(dest, dest_datastore) dest_file = self._disk_path(dest_datastore, dest) source_file = self._disk_path(source_datastore, source) self._check_free_space(dest_datastore, os.path.getsize(source_file)) mkdir_p(os.path.dirname(dest_file)) os.rename(self._disk_path(source_datastore, source), dest_file)
def create_disk(self, datastore, name, size, force=False): if not force: self._assert_not_exists(name, datastore) self._check_free_space(datastore, size) disk_file = self._disk_path(datastore, name) mkdir_p(os.path.dirname(disk_file)) with open(disk_file, "a") as f: f.truncate(size)
def _create_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id): """ Copy an image into a temp location. 1. Lock a tmp image destination file with an exclusive lock. This is to prevent the GC thread from garbage collecting directories that are actively being used. The temp directory name contains a random UUID to prevent collisions with concurrent copies 2. Create the temp directory. 3. Copy the metadata file over. 4. Copy the vmdk over. @return the tmp image directory on success. """ source = vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME) temp_dest = tmp_image_path(dest_datastore, dest_id) ds_type = self._get_datastore_type(dest_datastore) tmp_image_dir_path = os.path.dirname(datastore_to_os_path(temp_dest)) # Try grabbing the lock on the temp directory if it fails # (very unlikely) someone else is copying an image just retry # later. with FileBackedLock(tmp_image_dir_path, ds_type): source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME) # Create the temp directory mkdir_p(tmp_image_dir_path) # Copy the metadata file if it exists. if os.path.exists(source_meta): try: dest_meta = os.path.join(tmp_image_dir_path, metadata_filename(dest_id)) shutil.copy(source_meta, dest_meta) except: self._logger.exception("Failed to copy metadata file %s", source_meta) raise # Copy the manifest file if it exists source_manifest = os_image_manifest_path(source_datastore, source_id) if os.path.exists(source_manifest): try: dest_manifest = os.path.join(tmp_image_dir_path, manifest_filename(dest_id)) shutil.copy(source_manifest, dest_manifest) except: # Swallow it. Not critical. pass # Create the timestamp file self._create_image_timestamp_file(tmp_image_dir_path) _vd_spec = self._prepare_virtual_disk_spec( vim.VirtualDiskManager.VirtualDiskType.thin, vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic ) self._manage_disk( vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName=source, destName=temp_dest, destSpec=_vd_spec ) return tmp_image_dir_path
def __init__(self, test_id): """ test_id - Used to create a directory to put all the files generated by this instance of runtime utils. The directory path is /tmp/photon-controller-python/$RUNTIME_ID/$test_id """ self.test_id = test_id self.test_dir = os.path.join(self.BASE_DIR, self.RUNTIME_ID, test_id) mkdir_p(self.test_dir) self.agent_procs = [] self.chairman_procs = [] self.thrift_procs = []
def __init__(self, test_id): """ test_id - Used to create a directory to put all the files generated by this instance of runtime utils. The directory path is /tmp/photon-controller-python/$RUNTIME_ID/$test_id """ self.test_id = test_id self.test_dir = os.path.join(self.BASE_DIR, self.RUNTIME_ID, test_id) mkdir_p(self.test_dir) self.agent_procs = [] self.root_procs = [] self.chairman_procs = [] self.thrift_procs = []
def _setup_datastores(self): datastores = self._hypervisor.datastore_manager.get_datastore_ids() self._logger.info("setup datastore (%s)" % ",".join(datastores)) for i, datastore_id in enumerate(datastores): self._logger.info("create dir for %s" % datastore_id) datastore_id_path = os.path.join(self.image_path, datastore_id) mkdir_p(datastore_id_path) datastore_name = \ self._hypervisor.datastore_manager.datastore_name(datastore_id) self._logger.info("create symlink for %s" % datastore_name) datastore_name_path = os.path.join(self.image_path, datastore_name) if not os.path.exists(datastore_name_path): os.symlink(datastore_id_path, datastore_name_path)
def start_agent(self, config): """ config - Use get_default_agent_config() to get the default config, and modify the dict as needed. """ address = config["--hostname"] port = int(config["--port"]) mkdir_p(config["--config-path"]) arg_list = ["photon-controller-agent"] for (key, val) in config.items(): arg_list.append(key) if val: arg_list.append(val) # Keeping track of what is created for clean up purposes agent_client = DirectClient("Host", Host.Client, address, port) control_client = DirectClient("AgentControl", AgentControl.Client, address, port) try: agent_client.connect() agent_client.close() raise Exception("Agent already running on port %s" % port) except TTransport.TTransportException: pass proc = subprocess.Popen(arg_list) self.agent_procs.append(proc) def wait(process): if process: try: os.waitpid(process.pid, os.WUNTRACED) except OSError: # Process might already exit pass threading.Thread(target=wait, args=(proc, )).start() # Back off on failure to connect to agent max_sleep_time = 5 sleep_time = 0.1 while sleep_time < max_sleep_time: try: agent_client.connect() control_client.connect() return (proc, agent_client, control_client) except TTransport.TTransportException: time.sleep(sleep_time) sleep_time *= 2 return (None, None, None)
def start_agent(self, config): """ config - Use get_default_agent_config() to get the default config, and modify the dict as needed. """ address = config["--hostname"] port = int(config["--port"]) mkdir_p(config["--config-path"]) arg_list = ["photon-controller-agent"] for (key, val) in config.items(): arg_list.append(key) if val: arg_list.append(val) # Keeping track of what is created for clean up purposes agent_client = DirectClient("Host", Host.Client, address, port) control_client = DirectClient("AgentControl", AgentControl.Client, address, port) try: agent_client.connect() agent_client.close() raise Exception("Agent already running on port %s" % port) except TTransport.TTransportException: pass proc = subprocess.Popen(arg_list) self.agent_procs.append(proc) def wait(process): if process: try: os.waitpid(process.pid, os.WUNTRACED) except OSError: # Process might already exit pass threading.Thread(target=wait, args=(proc,)).start() # Back off on failure to connect to agent max_sleep_time = 5 sleep_time = 0.1 while sleep_time < max_sleep_time: try: agent_client.connect() control_client.connect() return (proc, agent_client, control_client) except TTransport.TTransportException: time.sleep(sleep_time) sleep_time *= 2 return (None, None, None)
def _move_image(self, image_id, datastore, tmp_dir): """ Atomic move of a tmp folder into the image datastore. Handles concurrent moves by locking a well know derivative of the image_id while doing the atomic move. The exclusive file lock ensures that only one move is successful. Has the following side effects: a - If the destination image already exists, it is assumed that someone else successfully copied the image over and the temp directory is deleted. b - If we fail to acquire the file lock after retrying 3 times, or the atomic move fails, the tmp image directory will be left behind and needs to be garbage collected later. image_id: String.The image id of the image being moved. datastore: String. The datastore id of the datastore. tmp_dir: String. The absolute path of the temp image directory. raises: OsError if the move fails AcquireLockFailure, InvalidFile if we fail to lock the destination image. """ ds_type = self._get_datastore_type(datastore) image_path = os.path.dirname( os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)) parent_path = os.path.dirname(image_path) # Create the parent image directory if it doesn't exist. try: mkdir_p(parent_path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(parent_path): # Parent directory exists nothing to do. pass else: raise try: with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds if self._check_image_repair(image_id, datastore): raise DiskAlreadyExistException("Image already exists") shutil.move(tmp_dir, image_path) except (AcquireLockFailure, InvalidFile): self._logger.info("Unable to lock %s for atomic move" % image_id) raise except DiskAlreadyExistException: self._logger.info("Image %s already copied" % image_id) rm_rf(tmp_dir) raise
def _move_image(self, image_id, datastore, tmp_dir): """ Atomic move of a tmp folder into the image datastore. Handles concurrent moves by locking a well know derivative of the image_id while doing the atomic move. The exclusive file lock ensures that only one move is successful. Has the following side effects: a - If the destination image already exists, it is assumed that someone else successfully copied the image over and the temp directory is deleted. b - If we fail to acquire the file lock after retrying 3 times, or the atomic move fails, the tmp image directory will be left behind and needs to be garbage collected later. image_id: String.The image id of the image being moved. datastore: String. The datastore id of the datastore. tmp_dir: String. The absolute path of the temp image directory. raises: OsError if the move fails AcquireLockFailure, InvalidFile if we fail to lock the destination image. """ ds_type = self._get_datastore_type(datastore) image_path = os.path.dirname(os_vmdk_path(datastore, image_id, IMAGE_FOLDER_NAME)) parent_path = os.path.dirname(image_path) # Create the parent image directory if it doesn't exist. try: mkdir_p(parent_path) except OSError as e: if e.errno == errno.EEXIST and os.path.isdir(parent_path): # Parent directory exists nothing to do. pass else: raise try: with FileBackedLock(image_path, ds_type, retry=300, wait_secs=0.01): # wait lock for 3 seconds if self._check_image_repair(image_id, datastore): raise DiskAlreadyExistException("Image already exists") shutil.move(tmp_dir, image_path) except (AcquireLockFailure, InvalidFile): self._logger.info("Unable to lock %s for atomic move" % image_id) raise except DiskAlreadyExistException: self._logger.info("Image %s already copied" % image_id) rm_rf(tmp_dir) raise
def _create_tmp_image(self, source_datastore, source_id, dest_datastore, dest_id): """ Copy an image into a temp location. 1. Lock a tmp image destination file with an exclusive lock. This is to prevent the GC thread from garbage collecting directories that are actively being used. The temp directory name contains a random UUID to prevent collisions with concurrent copies 2. Create the temp directory. 3. Copy the metadata file over. 4. Copy the vmdk over. @return the tmp image directory on success. """ source = vmdk_path(source_datastore, source_id, IMAGE_FOLDER_NAME) temp_dest = tmp_image_path(dest_datastore, dest_id) ds_type = self._get_datastore_type(dest_datastore) tmp_image_dir_path = os.path.dirname(datastore_to_os_path(temp_dest)) # Try grabbing the lock on the temp directory if it fails # (very unlikely) someone else is copying an image just retry # later. with FileBackedLock(tmp_image_dir_path, ds_type): source_meta = os_metadata_path(source_datastore, source_id, IMAGE_FOLDER_NAME) # Create the temp directory mkdir_p(tmp_image_dir_path) # Copy the metadata file if it exists. if os.path.exists(source_meta): try: shutil.copy(source_meta, tmp_image_dir_path) except: self._logger.exception("Failed to copy metadata file %s", source_meta) raise # Create the timestamp file self._create_image_timestamp_file(tmp_image_dir_path) _vd_spec = self._prepare_virtual_disk_spec( vim.VirtualDiskManager.VirtualDiskType.thin, vim.VirtualDiskManager.VirtualDiskAdapterType.lsiLogic) self._manage_disk(vim.VirtualDiskManager.CopyVirtualDisk_Task, sourceName=source, destName=temp_dest, destSpec=_vd_spec) return tmp_image_dir_path
def test_create_vm_with_ephemeral_disks(self): image_dir = os.path.join( "/tmp/images", FakeHypervisor.datastore_id(self.get_image_datastore())) try: mkdir_p(image_dir) with tempfile.NamedTemporaryFile(dir=image_dir, suffix=".vmdk") as f: # The temp file name created is # "/tmp/image/<ds>/<uniquepart>.vmdk". # This simulates an image being present on the agent, # The file is deleted on leaving the context. image_id = f.name[f.name.rfind("/") + 1:-5] self._test_create_vm_with_ephemeral_disks(image_id) finally: rm_rf(image_dir)
def copy_disk(self, source_datastore, source, dest_datastore, dest, force=False): if not self.hypervisor.image_manager.check_image(source, source_datastore): raise DiskNotFoundException("ENOENT") dest_file = self._disk_path(dest_datastore, dest) image_file = self.hypervisor.image_manager.image_file(source_datastore, source) self._check_free_space(dest_datastore, os.path.getsize(image_file)) if not force: # force allows overwriting existing destination self._assert_not_exists(dest, dest_datastore) mkdir_p(os.path.dirname(dest_file)) with open(dest_file, "a") as f: image_file = self.hypervisor.image_manager.image_file( source_datastore, source) f.truncate(os.path.getsize(image_file))
def copy_disk(self, source_datastore, source, dest_datastore, dest, force=False): if not self.hypervisor.image_manager.check_image( source, source_datastore): raise DiskNotFoundException("ENOENT") dest_file = self._disk_path(dest_datastore, dest) image_file = self.hypervisor.image_manager.image_file( source_datastore, source) self._check_free_space(dest_datastore, os.path.getsize(image_file)) if not force: # force allows overwriting existing destination self._assert_not_exists(dest, dest_datastore) mkdir_p(os.path.dirname(dest_file)) with open(dest_file, "a") as f: image_file = self.hypervisor.image_manager.image_file( source_datastore, source) f.truncate(os.path.getsize(image_file))
def __init__(self): self.test_dir = os.path.join(self.BASE_DIR, self.RUNTIME_ID) mkdir_p(self.test_dir) self.agent_procs = [] self.root_procs = [] self.thrift_procs = []