def wait_for_request(cells, partial_check=False): """helper function checks if a request is complete After finding the request's row using the ``cells`` argument, this will wait for a request to reach the 'Finished' state and return it. In the event of an 'Error' state, it will raise an AssertionError, for use with ``pytest.raises``, if desired. Args: cells: A dict of cells use to identify the request row to inspect in the :py:attr:`request_list` Table. See :py:meth:`cfme.web_ui.Table.find_rows_by_cells` for more. Usage: # Filter on the "Description" column description = 'Provision from [%s] to [%s]' % (template_name, vm_name) cells = {'Description': description} # Filter on the "Request ID" column # Text must match exactly, you can use "{:,}".format(request_id) to add commas if needed. request_id = '{:,}'.format(1000000000001) # Becomes '1,000,000,000,001', as in the table cells = {'Request ID': request_id} # However you construct the cells dict, pass it to wait_for_request # Provisioning requests often take more than 5 minutes but less than 10. wait_for(wait_for_request, [cells], num_sec=600) Raises: AssertionError: if the matched request has status 'Error' RequestException: if multiple matching requests were found Returns: The matching :py:class:`cfme.web_ui.Table.Row` if found, ``False`` otherwise. """ for page in paginator.pages(): if sel.elements(request_list._header_loc) and not sel.is_displayed(request_list): # The table exists but it is hidden - no cells return False results = request_list.find_rows_by_cells(cells, partial_check) if len(results) == 0: # row not on this page, assume it has yet to appear continue elif len(results) > 1: raise RequestException( 'Multiple requests with matching content found - be more specific!' ) else: # found the row! row = results[0] logger.debug(' Request Message: %s' % row.last_message.text) break else: # Request not found at all, can't continue return False assert row.status.text != 'Error' if row.request_state.text == 'Finished': return row else: return False
def delete_vm(self, vm_name): logger.debug(' Deleting VM....({})'.format(vm_name)) vm = self._get_vm(vm_name) wait_for(self.stop_vm, [vm_name], fail_condition='False', num_sec=300, delay=10) vm.delete() wait_for(self.does_vm_exist, [vm_name], fail_condition=True) return True
def login(username, password, submit_method=_click_on_login): """ Login to CFME with the given username and password. Optionally, submit_method can be press_enter_after_password to use the enter key to login, rather than clicking the button. Args: user: The username to fill in the username field. password: The password to fill in the password field. submit_method: A function to call after the username and password have been input. Raises: RuntimeError: If the login fails, ie. if a flash message appears """ if not logged_in() or username is not current_username(): if logged_in(): logout() # workaround for strange bug where we are logged out # as soon as we click something on the dashboard sel.sleep(1.0) logger.debug('Logging in as user %s' % username) fill(form, {'username': username, 'password': password}) submit_method() flash.assert_no_errors() thread_locals.current_user = User(username, password, _full_name())
def wait_for_miq_server_workers_started(evm_tail=None, poll_interval=5): """Waits for the CFME's workers to be started by tailing evm.log for: 'INFO -- : MIQ(MiqServer#wait_for_started_workers) All workers have been started' Verified works with 5.5 and 5.6 appliances. """ if evm_tail is None: logger.info('Opening /var/www/miq/vmdb/log/evm.log for tail') evm_tail = SSHTail('/var/www/miq/vmdb/log/evm.log') evm_tail.set_initial_file_end() attempts = 0 detected = False max_attempts = 60 while (not detected and attempts < max_attempts): logger.debug('Attempting to detect MIQ Server workers started: {}'.format(attempts)) for line in evm_tail: if 'MiqServer#wait_for_started_workers' in line: if ('All workers have been started' in line): logger.info('Detected MIQ Server is ready.') detected = True break time.sleep(poll_interval) # Allow more log lines to accumulate attempts += 1 if not (attempts < max_attempts): logger.error('Could not detect MIQ Server workers started in {}s.'.format( poll_interval * max_attempts)) evm_tail.close()
def ensure_advanced_search_closed(): """Checks if the advanced search box is open and if it does, closes it.""" if is_advanced_search_opened(): logger.debug('search.ensure_advanced_search_closed: search was open, closing') sel.click(search_box.close_button) wait_for(is_advanced_search_opened, fail_condition=True, num_sec=10, delay=2, fail_func=check_and_click_close, message='Waiting for advanced search to close')
def refresh_provider_vms_bulk(vm_ids): """Refresh the specified VMs with one request via the REST API""" starttime = time.time() appliance = cfme_performance['appliance']['ip_address'] resources = [] for vm_id in vm_ids: resources.append({ "href": "https://" + appliance + "/api/vms/" + str(vm_id) }) data_dict = { "action": "refresh", "resources": resources } data_json = json.dumps(data_dict) response = requests.post("https://" + appliance + "/api/vms/", data=data_json, auth=(cfme_performance['appliance']['rest_api']['username'], cfme_performance['appliance']['rest_api']['password']), verify=False, headers={"content-type": "application/json"}, allow_redirects=False) if response.status_code != 200: logger.debug(response.text) logger.debug('Queued Refresh {} VMs in: {}s'.format(len(vm_ids), round(time.time() - starttime, 2)))
def login(username, password, submit_method=_click_on_login): """ Login to CFME with the given username and password. Optionally, submit_method can be press_enter_after_password to use the enter key to login, rather than clicking the button. Args: user: The username to fill in the username field. password: The password to fill in the password field. submit_method: A function to call after the username and password have been input. Raises: RuntimeError: If the login fails, ie. if a flash message appears """ # TODO: Should probably do the username check here, but there are pretty usernames to deal with # e.g. 'admin' shows up in the UI as 'Administrator' if not logged_in(): # workaround for strange bug where we are logged out # as soon as we click something on the dashboard sel.sleep(1.0) logger.debug('Logging in as user %s' % username) fill(form, {'username': username, 'password': password}) submit_method() flash.assert_no_errors()
def refresh_provider_vms(vm_ids): """Refresh the specified VMs with one request per VM via the REST API""" starttime = time.time() for vm in vm_ids: refresh_provider_vm(vm) logger.debug('Queued Refresh {} VMs in: {}s'.format(len(vm_ids), round(time.time() - starttime, 2)))
def beans_admin(cur_user,user,form,issue): global G_applicationmgr if issue == 'applymsgs': result = G_applicationmgr.queryUnRead(cur_user = cur_user) logger.debug("applymsg success") return json.dumps(result) elif issue == 'agree': msgid = form.get("msgid",None) username = form.get("username",None) if msgid is None or username is None: return json.dumps({'success':'false', 'message':'msgid and username can\'t be null.'}) G_lockmgr.acquire("__beans_"+str(username)) G_lockmgr.acquire("__applymsg_"+str(msgid)) result = G_applicationmgr.agree(msgid, cur_user = cur_user) G_lockmgr.release("__applymsg_"+str(msgid)) G_lockmgr.release("__beans_"+str(username)) return json.dumps(result) elif issue == 'reject': msgid = form.get("msgid",None) if msgid is None: return json.dumps({'success':'false', 'message':'msgid can\'t be null.'}) G_lockmgr.acquire("__applymsg_"+str(msgid)) result = G_applicationmgr.reject(msgid, cur_user = cur_user) G_lockmgr.release("__applymsg_"+str(msgid)) return json.dumps(result) else: return json.dumps({'success':'false', 'message':'Unsupported URL!'})
def fill_snmp_trap_field_tuple(field, val): assert 2 <= len(val) <= 3, "The tuple must be at least 2 items and max 3 items!" if len(val) == 2: val = val + (None,) field.oid, field.type, field.value = val logger.debug(' Filling in SNMPTrapField #%d with values %s, %s, %s', field.seq_id, field.oid, field.type, field.value)
def _fill_multibox_dict(multi, d): """ Filler function for MultiBoxSelect Designed for `dict` styled items. It expects a dictionary in format: >>> {"Some item": True, "Another item": False} Where key stands for the item name and value its selection status. Any items that have to be unselected will be unselected before selecting the unselected happens. Args: multi: :py:class:`MultiBoxSelect` to fill d: :py:class:`dict` with values. Returns: :py:class:`bool` with success. """ enable_list, disable_list = [], [] for key, value in d.iteritems(): if value: enable_list.append(key) else: disable_list.append(key) logger.debug(' Disabling values %s in %s', str(disable_list), str(multi)) logger.debug(' Enabling values %s in %s', str(enable_list), str(multi)) multi.remove(*disable_list) if multi._async: sync, async, dont_care = [], [], [] MultiBoxSelect.categorize(enable_list, sync, async, dont_care) multi.add(*dont_care) multi.set_async(*async) multi.set_sync(*sync) else: multi.add(*map(str, enable_list))
def checkbox(loc, set_to=False): """ Checks or unchecks a given checkbox Finds an element given by loc and checks it Args: loc: The locator of the element value: The value the checkbox should represent as a bool (or None to do nothing) Returns: Previous state of the checkbox """ if set_to is not None: el = move_to_element(loc) if el.tag_name == 'img': # Yeah, CFME sometimes uses images for check boxen. *sigh* # item_chk0 = unchecked, item_chk1 = checked selected = 'item_chk1' in el.get_attribute('src') else: selected = el.is_selected() if selected is not set_to: logger.debug("Setting checkbox %s to %s" % (str(loc), str(set_to))) click(el) return selected
def map_vms_to_ids(provider_names_to_vm_names): """Takes a dictionary of providers with a list of vms and generates a list of vm_ids for each vm in the data structure. We need this because more than one provider can lead to a """ starttime = time.time() expected_num_ids = sum(len(x) for x in provider_names_to_vm_names.itervalues()) expected_num_providers = len(provider_names_to_vm_names.keys()) # Intended ouput here (List of vm ids): vm_ids = [] # Intermediate data structure holding provider_id to list of vm names provider_ids_to_vm_names = {} # First get all providers details all_providers_details = [] for pro_id in get_all_provider_ids(): details = get_provider_details(pro_id) all_providers_details.append(details) providers_to_vms_copy = dict(provider_names_to_vm_names) # Next map provider_name to the provider_id for provider_name in provider_names_to_vm_names: for provider_detail in all_providers_details: if provider_name == provider_detail['name']: # Copy VMs from that provider to the Intermediate data structure provider_ids_to_vm_names[provider_detail['id']] = list( provider_names_to_vm_names[provider_name]) del providers_to_vms_copy[provider_name] break if len(providers_to_vms_copy) > 0: # Error, we did not find all providers, likely there is an issue with the scenario data # inside of cfme_performance.yml or cfme_performance.local.yml logger.error('Provider(s) + vm(s) not found in CFME Inventory: {}'.format( providers_to_vms_copy)) provider_ids_to_vm_names_copy = copy.deepcopy(provider_ids_to_vm_names) # Now map each vm_name+ems_id to the actual vm_id and append to our list for vm_id in get_all_vm_ids(): vm_details = get_vm_details(vm_id) for provider_id in provider_ids_to_vm_names: if ('ems_id' in vm_details and provider_id == vm_details['ems_id']): # Match provider_id, now check vm_name for vm_name in provider_ids_to_vm_names[provider_id]: if vm_name == vm_details['name']: logger.debug('Matching {} to vm id: {}'.format(vm_name, vm_id)) vm_ids.append(vm_id) del (provider_ids_to_vm_names_copy[provider_id] [provider_ids_to_vm_names_copy[provider_id].index(vm_name)]) break if (sum(len(x) for x in provider_ids_to_vm_names_copy.itervalues()) == 0): break # Now check for left over vms that we did not match: leftover_num_ids = sum(len(x) for x in provider_ids_to_vm_names_copy.itervalues()) if leftover_num_ids > 0: logger.error('(Provider_id(s)) + VM(s) not found in CFME inventory: {}'.format( provider_ids_to_vm_names_copy)) logger.debug('Mapped {}/{} vm ids/names over {}/{} provider ids/names in {}s'.format( len(vm_ids), expected_num_ids, len(provider_ids_to_vm_names.keys()), expected_num_providers, round(time.time() - starttime, 2))) return vm_ids
def _wait_for_vm_running(): state = provider_mgmt.vm_status(vm_name) if ON_REGEX.match(state): return True elif DOWN_REGEX.match(state) or SUSPEND_REGEX.match(state): provider_mgmt.start_vm(vm_name) logger.debug("Sleeping 15secs...(current state: " + state + ", needed state: running)")
def refresh_providers(provider_ids): """Refresh the specified providers with one request per provider via the REST API""" starttime = time.time() for provider in provider_ids: refresh_provider(provider) logger.debug('Queued Refresh {} Provider(s) in: {}s'.format(len(provider_ids), round(time.time() - starttime, 2)))
def __init__(self, stream_output=False, keystate=_ssh_keystate.not_installed, **connect_kwargs): super(SSHClient, self).__init__() self._streaming = stream_output self._keystate = keystate logger.debug('client initialized with keystate {}'.format(_ssh_keystate[keystate])) # Load credentials and destination from confs, set up sane defaults parsed_url = urlparse(store.base_url) default_connect_kwargs = { 'username': conf.credentials['ssh']['username'], 'password': conf.credentials['ssh']['password'], 'hostname': parsed_url.hostname, 'timeout': 10, 'allow_agent': False, 'look_for_keys': False, 'gss_auth': False } default_connect_kwargs["port"] = ports.SSH # Overlay defaults with any passed-in kwargs and store default_connect_kwargs.update(connect_kwargs) self._connect_kwargs = default_connect_kwargs self.set_missing_host_key_policy(paramiko.AutoAddPolicy()) _client_session.append(self)
def _setup_providers(cloud_or_infra, validate, check_existing): """Helper to set up all cloud or infra providers, and then validate them Args: cloud_or_infra: Like the name says: 'cloud' or 'infra' (a string) validate: see description in :py:func:`setup_provider` check_existing: see description in :py:func:`setup_provider` Returns: A list of provider objects that have been created. """ # Pivot behavior on cloud_or_infra options_map = { 'cloud': { 'navigate': 'clouds_providers', 'quad': 'cloud_prov', 'list': list_cloud_providers }, 'infra': { 'navigate': 'infrastructure_providers', 'quad': 'infra_prov', 'list': list_infra_providers } } # Check for existing providers all at once, to prevent reloading # the providers page for every provider in cfme_data if not options_map[cloud_or_infra]['list'](): return [] if check_existing: sel.force_navigate(options_map[cloud_or_infra]['navigate']) add_providers = [] for provider_key in options_map[cloud_or_infra]['list'](): provider_name = conf.cfme_data.get('management_systems', {})[provider_key]['name'] quad = Quadicon(provider_name, options_map[cloud_or_infra]['quad']) for page in paginator.pages(): if sel.is_displayed(quad): logger.debug('Provider "%s" exists, skipping' % provider_key) break else: add_providers.append(provider_key) else: # Add all cloud or infra providers unconditionally add_providers = options_map[cloud_or_infra]['list']() if add_providers: logger.info('Providers to be added: %s' % ', '.join(add_providers)) # Save the provider objects for validation and return added_providers = [] for provider_name in add_providers: # Don't validate in this step; add all providers, then go back and validate in order provider = setup_provider(provider_name, validate=False, check_existing=False) added_providers.append(provider) if validate: map(methodcaller('validate'), added_providers) return added_providers
def delete_vm(self, vm_name): self.wait_vm_steady(vm_name) if not self.is_vm_stopped(vm_name): self.stop_vm(vm_name) logger.debug(' Deleting RHEV VM %s' % vm_name) def _do_delete(): """Returns True if you have to retry""" if not self.does_vm_exist(vm_name): return False try: vm = self._get_vm(vm_name) vm.delete() except RequestError as e: # Handle some states that can occur and can be circumvented if e.status == 409 and "Related operation" in e.detail: logger.info("Waiting for RHEV: {}:{} ({})".format(e.status, e.reason, e.detail)) return True else: raise # Raise other so we can see them and eventually add them into handling # TODO: handle 400 - but I haven't seen the error message, it was empty. else: return False wait_for(_do_delete, fail_condition=True, num_sec=600, delay=15, message="execute delete") wait_for( lambda: self.does_vm_exist(vm_name), fail_condition=True, message="wait for RHEV VM %s deleted" % vm_name, num_sec=300 ) return True
def set_rails_loglevel(level, validate_against_worker='MiqUiWorker'): """Sets the logging level for level_rails and detects when change occured.""" ui_worker_pid = '#{}'.format(get_worker_pid(validate_against_worker)) logger.info('Setting log level_rails on appliance to {}'.format(level)) yaml = store.current_appliance.get_yaml_config() if not str(yaml['log']['level_rails']).lower() == level.lower(): logger.info('Opening /var/www/miq/vmdb/log/evm.log for tail') evm_tail = SSHTail('/var/www/miq/vmdb/log/evm.log') evm_tail.set_initial_file_end() yaml['log']['level_rails'] = level store.current_appliance.set_yaml_config(yaml) attempts = 0 detected = False while (not detected and attempts < 60): logger.debug('Attempting to detect log level_rails change: {}'.format(attempts)) for line in evm_tail: if ui_worker_pid in line: if 'Log level for production.log has been changed to' in line: # Detects a log level change but does not validate the log level logger.info('Detected change to log level for production.log') detected = True break time.sleep(1) # Allow more log lines to accumulate attempts += 1 if not (attempts < 60): # Note the error in the logger but continue as the appliance could be slow at logging # that the log level changed logger.error('Could not detect log level_rails change.') evm_tail.close() else: logger.info('Log level_rails already set to {}'.format(level))
def messages_to_statistics_csv(messages, statistics_file_name): all_statistics = [] for msg_id in messages: msg = messages[msg_id] added = False if len(all_statistics) > 0: for msg_statistics in all_statistics: if msg_statistics.cmd == msg.msg_cmd: if msg.del_time > 0: msg_statistics.delivertimes.append(float(msg.del_time)) msg_statistics.gets += 1 msg_statistics.dequeuetimes.append(float(msg.deq_time)) msg_statistics.totaltimes.append(float(msg.total_time)) msg_statistics.puts += 1 added = True break if not added: msg_statistics = MiqMsgLists() msg_statistics.cmd = msg.msg_cmd if msg.del_time > 0: msg_statistics.delivertimes.append(float(msg.del_time)) msg_statistics.gets = 1 msg_statistics.dequeuetimes.append(float(msg.deq_time)) msg_statistics.totaltimes.append(float(msg.total_time)) msg_statistics.puts = 1 all_statistics.append(msg_statistics) csvdata_path = log_path.join('csv_output', statistics_file_name) outputfile = csvdata_path.open('w', ensure=True) try: csvfile = csv.writer(outputfile) metrics = ['samples', 'min', 'avg', 'median', 'max', 'std', '90', '99'] measurements = ['deq_time', 'del_time', 'total_time'] headers = ['cmd', 'puts', 'gets'] for measurement in measurements: for metric in metrics: headers.append('{}_{}'.format(measurement, metric)) csvfile.writerow(headers) # Contents of CSV for msg_statistics in sorted(all_statistics, key=lambda x: x.cmd): if msg_statistics.gets > 1: logger.debug('Samples/Avg/90th/Std: {} : {} : {} : {},Cmd: {}'.format( str(len(msg_statistics.totaltimes)).rjust(7), str(round(numpy.average(msg_statistics.totaltimes), 3)).rjust(7), str(round(numpy.percentile(msg_statistics.totaltimes, 90), 3)).rjust(7), str(round(numpy.std(msg_statistics.totaltimes), 3)).rjust(7), msg_statistics.cmd)) stats = [msg_statistics.cmd, msg_statistics.puts, msg_statistics.gets] stats.extend(generate_statistics(msg_statistics.dequeuetimes, 3)) stats.extend(generate_statistics(msg_statistics.delivertimes, 3)) stats.extend(generate_statistics(msg_statistics.totaltimes, 3)) csvfile.writerow(stats) finally: outputfile.close()
def find_request(cells, partial_check=False): """Finds the request and returns the row element Args: cells: Search data for the requests table. partial_check: If to use the ``in`` operator rather than ``==`` in find_rows_by_cells(). Returns: row """ navigate_to(Request, 'All') from cfme.web_ui import paginator for page in paginator.pages(): results = fields.request_list.find_rows_by_cells(cells, partial_check) if len(results) == 0: # row not on this page, assume it has yet to appear # it might be nice to add an option to fail at this point continue elif len(results) > 1: raise RequestException( 'Multiple requests with matching content found - be more specific!' ) else: # found the row! row = results[0] logger.debug(' Request Message: %s', row.last_message.text) return row else: # Request not found at all, can't continue return False
def add_host_credentials(provider): """"Adds host credentials to a provider via the REST API""" data_dict = { "action": "edit", "resource": { "credentials": { "userid": provider['host_credentials']['username'], "password": provider['host_credentials']['password'] } } } json_data = json.dumps(data_dict) appliance = cfme_performance['appliance']['ip_address'] for host in get_all_host_ids(): response = requests.post("https://" + appliance + "/api/hosts/" + str(host), data=json_data, auth=(cfme_performance['appliance']['rest_api']['username'], cfme_performance['appliance']['rest_api']['password']), verify=False, headers={"content-type": "application/json"}, allow_redirects=False) if response.status_code != 200: logger.debug(response.text) print response # TODO: REMOVE logger.debug('Added host credentials, Response: {}'.format(response))
def _test_vm_power_on(): """Ensures power button is shown for a VM""" logger.info("Checking for power button") vm_name = virtual_machines.get_first_vm_title() logger.debug("VM " + vm_name + " selected") if not virtual_machines.is_pwr_option_visible(vm_name, option=virtual_machines.Vm.POWER_ON): raise OptionNotAvailable("Power button does not exist")
def uncollectif(item): """ Evaluates if an item should be uncollected Tests markers against a supplied lambda from the marker object to determine if the item should be uncollected or not. """ marker = item.get_marker('uncollectif') if marker: log_msg = 'Uncollecting {}: {}'.format(item.name, marker.kwargs.get('reason', 'No reason given')) try: arg_names = inspect.getargspec(marker._arglist[0][0][0]).args except TypeError: logger.debug(log_msg) return not bool(marker.args[0]) try: args = [item.callspec.params[arg] for arg in arg_names] except KeyError: missing_argnames = list(set(arg_names) - set(item._request.funcargnames)) func_name = item.name if missing_argnames: raise Exception("You asked for a fixture which wasn't in the function {} " "prototype {}".format(func_name, missing_argnames)) else: raise Exception("Failed to uncollect {}, best guess a fixture wasn't " "ready".format(func_name)) logger.debug(log_msg) return not marker.args[0](*args) else: return True
def save_cluster(user, beans, form): global G_vclustermgr clustername = form.get('clustername', None) if (clustername == None): return json.dumps({'success':'false', 'message':'clustername is null'}) imagename = form.get("image", None) description = form.get("description", None) containername = form.get("containername", None) isforce = form.get("isforce", None) G_ulockmgr.acquire(user) try: if not isforce == "true": [status,message] = G_vclustermgr.image_check(user,imagename) if not status: return json.dumps({'success':'false','reason':'exists', 'message':message}) user_info = post_to_user("/user/selfQuery/", {'token':form.get("token")}) [status,message] = G_vclustermgr.create_image(user,clustername,containername,imagename,description,user_info["data"]["groupinfo"]["image"]) if status: logger.info("image has been saved") return json.dumps({'success':'true', 'action':'save'}) else: logger.debug(message) return json.dumps({'success':'false', 'reason':'exceed', 'message':message}) except Exception as ex: logger.error(str(ex)) return json.dumps({'success':'false', 'message': str(ex)}) finally: G_ulockmgr.release(user)
def browser_setup(get_appliance, provider, vm_to_analyze, fs_type, mgmt_sys_api_clients): '''Overrides env.conf and points a browser to the appliance IP passed to it. Once finished with the test, it checks if any tests need the appliance and delete it if not the appliance specified in conf/env.yaml. ''' global appliance_vm_name global test_list test_list.remove(['', provider, vm_to_analyze, fs_type]) with browser_session(base_url='https://' + get_appliance): yield nav.home_page_logged_in(testsetup) # cleanup provisioned appliance if not more tests for it if ('appliances_provider' not in cfme_data['basic_info'].keys() or provider != cfme_data['basic_info']['appliances_provider']): more_same_provider_tests = False for outstanding_test in test_list: if outstanding_test[1] == provider: logger.debug("More provider tests found") more_same_provider_tests = True break if not more_same_provider_tests: # if rhev, remove direct_lun disk before delete if cfme_data['management_systems'][provider]['type'] == 'rhevm': logger.info('Removing RHEV direct_lun hook...') run_command("./scripts/connect_directlun.py --remove --provider " + provider + " --vm_name " + appliance_vm_name) # delete appliance logger.info("Delete provisioned appliance: " + appliance_list[provider]) destroy_cmd = ('./scripts/clone_template.py --provider ' + provider + ' ' '--destroy --vm_name ' + appliance_vm_name + ' ') run_command(destroy_cmd)
def provision_appliances( self, count=1, preconfigured=False, version=None, stream=None, provider=None, lease_time=120, ram=None, cpu=None): # If we specify version, stream is ignored because we will get that specific version if version: stream = get_stream(version) # If we specify stream but not version, sprout will give us latest version of that stream elif stream: pass # If we dont specify either, we will get the same version as current appliance else: stream = get_stream(current_appliance.version) version = current_appliance.version.vstring request_id = self.call_method( 'request_appliances', preconfigured=preconfigured, version=version, group=stream, provider=provider, lease_time=lease_time, ram=ram, cpu=cpu, count=count ) wait_for( lambda: self.call_method('request_check', str(request_id))['finished'], num_sec=300, message='provision {} appliance(s) from sprout'.format(count)) data = self.call_method('request_check', str(request_id)) logger.debug(data) appliances = [] for appliance in data['appliances']: appliances.append(IPAppliance(appliance['ip_address'])) return appliances, request_id
def test_refresh_providers(request, scenario): """Refreshes providers then waits for a specific amount of time. Memory Monitor creates graphs and summary at the end of the scenario.""" from_ts = int(time.time() * 1000) ssh_client = SSHClient() logger.debug('Scenario: {}'.format(scenario['name'])) clean_appliance(ssh_client) monitor_thread = SmemMemoryMonitor(SSHClient(), 'workload-refresh-providers', scenario['name'], 'refresh-providers', get_server_roles_workload_refresh_providers(separator=','), ', '.join(scenario['providers'])) def cleanup_workload(scenario, from_ts): starttime = time.time() to_ts = int(starttime * 1000) g_url = get_scenario_dashboard_url(scenario, from_ts, to_ts) logger.debug('Started cleaning up monitoring thread.') monitor_thread.grafana_url = g_url monitor_thread.signal = False monitor_thread.join() timediff = time.time() - starttime logger.info('Finished cleaning up monitoring thread in {}'.format(timediff)) request.addfinalizer(lambda: cleanup_workload(scenario, from_ts)) monitor_thread.start() wait_for_miq_server_ready(poll_interval=2) set_server_roles_workload_refresh_providers(ssh_client) add_providers(scenario['providers']) id_list = get_all_provider_ids() # Variable amount of time for refresh workload total_time = scenario['total_time'] starttime = time.time() time_between_refresh = scenario['time_between_refresh'] while ((time.time() - starttime) < total_time): start_refresh_time = time.time() refresh_providers(id_list) iteration_time = time.time() refresh_time = round(iteration_time - start_refresh_time, 2) elapsed_time = iteration_time - starttime logger.debug('Time to Queue Refreshes: {}'.format(refresh_time)) logger.info('Time elapsed: {}/{}'.format(round(elapsed_time, 2), total_time)) if refresh_time < time_between_refresh: wait_diff = time_between_refresh - refresh_time time_remaining = total_time - elapsed_time if (time_remaining > 0 and time_remaining < time_between_refresh): time.sleep(time_remaining) elif time_remaining > 0: time.sleep(wait_diff) else: logger.warn('Time to Queue Refreshes ({}) exceeded time between Refreshes({})'.format( refresh_time, time_between_refresh)) logger.info('Test Ending...')
def perf_click(uiworker_pid, tailer, measure_sel_time, clickable, *args): # Regular Expressions to find the ruby production completed time and select query time status_re = re.compile(r'Completed\s([0-9]*\s[a-zA-Z]*)\sin\s([0-9\.]*)ms') views_re = re.compile(r'Views:\s([0-9\.]*)ms') activerecord_re = re.compile(r'ActiveRecord:\s([0-9\.]*)ms') select_query_time_re = re.compile(r'\s\(([0-9\.]*)ms\)') worker_pid = '#' + uiworker_pid # Time the selenium transaction from "click" seleniumtime = 0 if clickable: starttime = time() clickable(*args) seleniumtime = int((time() - starttime) * 1000) pgstats = [] pgstat = PageStat() line_count = 0 starttime = time() for line in tailer: line_count += 1 if worker_pid in line: if 'SELECT' in line: pgstat.selectcount += 1 selecttime = select_query_time_re.search(line) if selecttime: if float(selecttime.group(1)) > perf_tests['ui']['threshold']['query_time']: pgstat.slowselects.append(line) if 'CACHE' in line: pgstat.cachedcount += 1 if 'INFO -- : Started' in line: # Obtain method and requested page started_idx = line.index('Started') + 8 pgstat.request = line[started_idx:line.index('for', 72)] if 'INFO -- : Completed' in line: # Obtain status code and total render time status_result = status_re.search(line) if status_result: pgstat.status = status_result.group(1) pgstat.completedintime = float(status_result.group(2)) pgstat.uncachedcount = pgstat.selectcount - pgstat.cachedcount # Redirects don't always have a view timing views_result = views_re.search(line) if views_result: pgstat.viewstime = float(views_result.group(1)) activerecord_result = activerecord_re.search(line) if activerecord_result: pgstat.activerecordtime = float(activerecord_result.group(1)) pgstats.append(pgstat) pgstat = PageStat() if pgstats: if measure_sel_time: pgstats[-1].seleniumtime = seleniumtime timediff = time() - starttime logger.debug('Parsed ({}) lines in {}'.format(line_count, timediff)) return pgstats
def refresh_provider_relationships(self, from_list_view=False): # from_list_view is ignored as it is included here for sake of compatibility with UI call. logger.debug('Refreshing provider relationships') col = self.appliance.rest_api.collections.providers.find_by(name=self.name) try: col[0].action.refresh() except IndexError: raise Exception("Provider collection empty")
def query_video_id(video_conf): mysql = Sql() if video_conf['version'] == 'rtsp': sql_video_id = "SELECT F_ID FROM t_video_channel WHERE F_Name = '{}' AND " \ "F_Video_Type = 'rtsp' AND F_Enabled = 1;".format(video_conf['name']) else: sql_video_id = "SELECT F_ID FROM t_video_server WHERE F_NAME = '{}' AND " \ "F_Video_Type = 'pvg' AND F_Enabled = 1;".format(video_conf['name']) req = mysql.query(sql_video_id) mysql.close() server_id = req[0][0] if req else None logger.debug('获取 {} ID: {}'.format(video_conf['name'], server_id)) return server_id
def delete_provisioned_vms(provision_order): """Attempts to Deletes all VMs in provision_order. Expects provision order to be a 2D list where the inner list contains the VM name in index 0, and its provider's name in index 1 Expects cleanup_size to be an integer""" starttime = time.time() startsize = len(provision_order) for vm_tuple in provision_order[:]: if delete_provisioned_vm(vm_tuple): provision_order.remove(vm_tuple) logger.debug('Deleted {} VMs in: {}s'.format(startsize - len(provision_order), round(time.time() - starttime, 2)))
def clear_infra_providers(validate=True): sel.force_navigate('infrastructure_providers') logger.debug('Checking for existing infrastructure providers...') if paginator.rec_total(): logger.info(' Providers exist, so removing all infra providers') paginator.results_per_page('100') sel.click(paginator.check_all()) toolbar.select('Configuration', 'Remove Infrastructure Providers from the VMDB', invokes_alert=True) sel.handle_alert() if validate: wait_for_no_infra_providers()
def on_request(self, command: str, message: Message) -> None: """ Message handler for all request commands except those that have personal handler. Personal request message handler name format: Capabilities-Exchange -> on_capabilities_exchange Registration-Termination -> on_registration_termination ... """ logger.debug("Command={command}, Message={message}", command=command, message=message)
def _wait_for_vm_paused(): if provider.mgmt.is_vm_paused(vm_name): return True elif provider.mgmt.is_vm_running(vm_name): provider.mgmt.pause_vm(vm_name) elif provider.mgmt.is_vm_stopped(vm_name) or \ provider.mgmt.can_suspend and provider.mgmt.is_vm_suspended(vm_name): provider.mgmt.start_vm(vm_name) logger.debug( "Sleeping 15secs...(current state: {}, needed state: paused)". format(provider.mgmt.vm_status(vm_name))) return False
def validate(data): validation_schema = ResourceSchema() try: validation_schema.load(data) except ValidationError as e: logger.debug('Error messages are: %s' %str(e)) try: error_message = e.messages.values()[0][0] except Exception as e: logger.error(str(e)) error_message = 'unknown failure' raise ValueError(error_message)
def work(self): n = -10 while 1: try: n += 10 for res in self.parse_old_link(n): yield res[0], res[1] except: logger.debug( '%s%s%s' % ('have no pageing!!!', '\t', traceback.format_exc())) yield None break
def un_zip(target_path): """ 解压缩目标压缩包 实现新需求,解压缩后相应的js文件做代码格式化 :return: """ logger.info("[Pre][Unzip] Upzip file {}...".format(target_path)) if not os.path.isfile(target_path): logger.warn("[Pre][Unzip] Target file {} is't exist...pass".format( target_path)) return False zip_file = zipfile.ZipFile(target_path) target_file_path = target_path + "_files/" if os.path.isdir(target_file_path): logger.debug("[Pre][Unzip] Target files {} is exist...continue".format( target_file_path)) return target_file_path else: os.mkdir(target_file_path) for names in zip_file.namelist(): zip_file.extract(names, target_file_path) # 对其中部分文件中为js的时候,将js代码格式化便于阅读 if names.endswith(".js"): file_path = os.path.join(target_file_path, names) file = codecs.open(file_path, 'r+', encoding='utf-8', errors='ignore') file_content = file.read() file.close() new_file = codecs.open(file_path, 'w+', encoding='utf-8', errors='ignore') opts = jsbeautifier.default_options() opts.indent_size = 2 new_file.write(jsbeautifier.beautify(file_content, opts)) new_file.close() zip_file.close() return target_file_path
def navigate_quadicons(q_names, q_type, page_name, nav_limit, ui_worker_pid, prod_tail, soft_assert, acc_topbars=[]): pages = [] count = 0 if nav_limit == 0: count = -1 assert len(q_names) > 0 while (count < nav_limit): for q in q_names: for page in paginator.pages(): quadicon = Quadicon(str(q), q_type) if sel.is_displayed(quadicon): pages.extend(analyze_page_stat(perf_click(ui_worker_pid, prod_tail, True, sel.click, quadicon), soft_assert)) for topbar in acc_topbars: try: if not list_acc.is_active(topbar): list_acc.click(topbar) links = list_acc.get_active_links(topbar) for link in range(len(links)): # Every click makes the previous list of links invalid links = list_acc.get_active_links(topbar) if link <= len(links): # Do not navigate to any link containing: dnn = ['parent', 'Capacity & Utilization', 'Timelines', 'Show tree of all VMs by Resource Pool in this Cluster', 'Show host drift history', 'Show VMs'] if any_in(dnn, links[link].title): logger.debug('DNN Skipping: {}'.format(links[link].title)) else: pages.extend(analyze_page_stat(perf_click(ui_worker_pid, prod_tail, True, links[link].click), soft_assert)) except NoSuchElementException: logger.warning('NoSuchElementException - page_name:{}, Quadicon:{},' ' topbar:{}'.format(page_name, q, topbar)) soft_assert(False, 'NoSuchElementException - page_name:{}, Quadicon:{},' ' topbar:{}'.format(page_name, q, topbar)) break count += 1 break pages.extend(analyze_page_stat(perf_click(ui_worker_pid, prod_tail, True, sel.force_navigate, page_name), soft_assert)) # If nav_limit == 0 , every item is navigated to if not nav_limit == 0 and count == nav_limit: break return pages
def get_any_class_methodcall(self, method_name, call_params, unserchain=[], define_param=(), deepth=0): """ 可以调用任意类的任意个方法,跟踪分析 :param method_name: :param call_params: :param unserchain: :param define_param: :param deepth: :return: """ deepth += 1 define_param = (*call_params, *define_param) call_nodes = self.dataflow_db.objects.filter(node_type='newMethod') logger.debug( "[PhpUnSerChain] trigger any class method. try to found all method in class with {}." .format(call_params)) for node in call_nodes: # 为了不影响数据,要先生成新的 newunserchain = [node] class_locate = node.node_locate new_locate = node.node_locate + '.' + node.source_node method_nodes = self.dataflow_db.objects.filter( node_locate__startswith=new_locate) params_count = self.dataflow_db.objects.filter( node_locate__startswith=new_locate, node_type='newMethodparams') if params_count != len(define_param): continue status = self.deep_search_chain(method_nodes, class_locate, newunserchain, define_param=define_param, deepth=deepth) if status: unserchain.extend(newunserchain) return True return False
def detect_observed_field(loc): """Detect observed fields; sleep if needed Used after filling most form fields, this function will inspect the filled field for one of the known CFME observed field attribues, and if found, sleep long enough for the observed field's AJAX request to go out, and then block until no AJAX requests are in flight. Observed fields occasionally declare their own wait interval before firing their AJAX request. If found, that interval will be used instead of the default. """ if is_displayed(loc): el = element(loc) else: # Element not visible, sort out return # Default wait period, based on the default UI wait (700ms) # plus a little padding to let the AJAX fire before we wait_for_ajax default_wait = .8 # Known observed field attributes observed_field_markers = ( 'data-miq_observe', 'data-miq_observe_date', 'data-miq_observe_checkbox', ) for attr in observed_field_markers: try: observed_field_attr = el.get_attribute(attr) break except NoSuchAttributeException: pass else: # Failed to detect an observed text field, short out return try: attr_dict = json.loads(observed_field_attr) interval = float(attr_dict.get('interval', default_wait)) # Pad the detected interval, as with default_wait interval += .1 except (TypeError, ValueError): # ValueError and TypeError happens if the attribute value couldn't be decoded as JSON # ValueError also happens if interval couldn't be coerced to float # In either case, we've detected an observed text field and should wait interval = default_wait logger.debug(' Observed field detected, pausing %.1f seconds' % interval) sleep(interval) wait_for_ajax()
def _fill_multibox_str(multi, string): """ Filler function for MultiBoxSelect Designed for `string`. Selects item with the name. Args: multi: :py:class:`MultiBoxSelect` to fill string: String to select Returns: :py:class:`bool` with success. """ stype = type(multi) logger.debug(' Filling in %s with value %s' % (str(stype), string)) return multi.add(string)
def pytest_sessionfinish(session, exitstatus): """Loop through the appliance stack and close ssh connections""" for ssh_client in store.ssh_clients_to_close: logger.debug('Closing ssh connection on %r', ssh_client) try: ssh_client.close() except: logger.debug('Closing ssh connection on %r failed, but ignoring', ssh_client) for session in ssh._client_session: with diaper: session.close() yield
def wx_login(self, code): '''是否每次都要获取用户信息''' ret = wx_oauth.GetUserInfo(code) if (ret['state'] != 0): self.write(str(ret)) return False userInfo = json.loads(ret['userinfo']) tokenInfo = json.loads(ret['token_data']) userId = daos.userDao.QueryWeChat(userInfo['openid']) if not userId: parentUserId = self.get_argument('userid', default=None) logger.debug('new user login, super user id %s' % parentUserId) # 没有用户绑定, 生成绑定用户, 此时应该把头像存到本地服务器, 目前先不做,暂时用微信的,如果失效再更新吧 userId = daos.userDao.GenerateUserByWeChat(userInfo, tokenInfo, parentId=parentUserId) else: '''更新用户信息 .....''' logger.debug('update wechat info to user. %s' % str(userInfo)) if not userId: logger.debug('failed to generate user..... ' % str(userInfo)) return False expires = int(time.time()) + self.expiresTime logger.debug('%s 登录成功,有效时间 %s 秒' % (userInfo['nickname'].encode("UTF-8"), str(expires))) # 这里的过期时间到浏览器查看总是比当前时间早7-8个小时?, 所以每次登录完以后,cookie就失效,导致无法访问其他页面 self.set_secure_cookie(self.secure_username, userId, expires=expires) #self.set_secure_cookie(self.secure_username, userId, expires_days=1) #self.set_secure_cookie(self.secure_username, userId) return True
def test_filter_save_cancel(rails_delete_filter): # bind filter_name to the function for fixture cleanup test_filter_save_cancel.filter_name = fauxfactory.gen_alphanumeric() logger.debug('Set filter_name to: {}'.format(test_filter_save_cancel.filter_name)) # Try save filter assert search.save_filter("fill_count(Infrastructure Provider.VMs, >)", test_filter_save_cancel.filter_name, cancel=True) assert_no_cfme_exception() assert search.reset_filter() # Exception depends on system state - Load button will be disabled if there are no saved filters with pytest.raises((DisabledButtonException, NoSuchElementException)): search.load_filter(saved_filter=test_filter_save_cancel.filter_name)
def test_SanYaoSu_3(self): '''姓名和身份证匹配,手机号不匹配''' yaosupage = SanYaoSuPage(self.driver, self.url, u'登录-知了背调') yaosupage.input_bei_diao_xing_ming(unicode(self.test_sanyaosu_datas[1][0])) yaosupage.input_id_number(str(int(self.test_sanyaosu_datas[1][1]))) yaosupage.input_phone_number(str(int(self.test_sanyaosu_datas[1][2]))) yaosupage.click_start_button() time.sleep(2) try: self.assertIn("该手机号与本人身份不匹配", yaosupage.get_ren_gong_text()) logger.debug(yaosupage.get_ren_gong_text()) except: screenshot.Cut_img(self.driver, "test_SanYaoSu_3") logger.error("没有进入人工审核第一步")
def recover_jobs(self): logger.info("Rerun the unfailed and unfinished jobs...") try: rejobs = Batchjob.query.filter( ~Batchjob.status.in_(['done', 'failed', 'stopped'])) rejobs = rejobs.order_by(Batchjob.create_time).all() for rejob in rejobs: logger.info("Rerun job: " + rejob.id) logger.debug(str(rejob)) job = BatchJob(rejob.id, rejob.username, None, rejob) self.job_map[job.job_id] = job self.process_job(job) except Exception as err: logger.error(traceback.format_exc())
def get_number_of_vms(do_not_navigate=False): """ Returns the total number of VMs visible to the user, including those archived or orphaned """ logger.info("Getting number of vms") if not do_not_navigate: navigate_to(Vm, 'VMsOnly') if not paginator.page_controls_exist(): logger.debug("No page controls") return 0 total = paginator.rec_total() logger.debug("Number of VMs: %s", total) return int(total)
def switch_to_window(self, partial_url='', partial_title=''): all_windows = self.driver.window_handles if len(all_windows) == 1: logger.warning('只有一个窗口,无法进行切换!') elif len(all_windows) == 2: other_window = all_windows[1 - all_windows.index(self.current_window)] self.driver.switch_to.window(other_window) else: for window in all_windows: self.driver.switch_to.window(window) if partial_url in self.driver.current_url or partial_title in self.driver.title: break logger.debug(self.driver.current_url, self.driver.title)
def clear_cloud_providers(validate=True): sel.force_navigate('clouds_providers') logger.debug('Checking for existing cloud providers...') total = paginator.rec_total() if total is not None and int(total) > 0: logger.info(' Providers exist, so removing all cloud providers') paginator.results_per_page('100') sel.click(paginator.check_all()) toolbar.select('Configuration', 'Remove Cloud Providers from the VMDB', invokes_alert=True) sel.handle_alert() if validate: wait_for_no_cloud_providers()
def get_number_of_vms(do_not_navigate=False): """ Returns the total number of VMs visible to the user, including those archived or orphaned """ logger.info("Getting number of vms") if not do_not_navigate: sel.force_navigate('infra_vms') if not paginator.page_controls_exist(): logger.debug("No page controls") return 0 total = paginator.rec_total() logger.debug("Number of VMs: {}".format(total)) return int(total)
def test_can_delete_filter(): filter_name = fauxfactory.gen_alphanumeric() logger.debug('Set filter_name to: {}'.format(filter_name)) assert search.save_filter("fill_count(Infrastructure Provider.VMs, >, 0)", filter_name) assert_no_cfme_exception() search.reset_filter() assert_no_cfme_exception() search.load_filter(filter_name) assert_no_cfme_exception() if not search.delete_filter(): raise pytest.fail( "Cannot delete filter! Probably the delete button is not present!") assert_no_cfme_exception()
def check_kunlunignore(filename): is_not_ignore = True for ignore_reg in IGNORE_LIST: # ignore_reg_obj = re.match(ignore_reg, filename, re.I) if re.search(ignore_reg, filename, re.I): logger.debug('[INIT][IGNORE] File {} filter by {}'.format( filename, ignore_reg)) return False return True
def delete_provisioned_vm(vm_tuple): """Deletes the Vm specified in the vm_tuple. Expects a the tuple to contain a VM name in index 0, and its provider in index 1. Returns True if successful""" vm_name, provider_name = vm_tuple logger.debug('Cleaning up: {}'.format(vm_name)) provider_details = (cfme_providers[provider_name]) provider = get_mgmt_provider_class(provider_details) try: provider.delete_vm(vm_name) return True except Exception as e: # VM potentially was not yet provisioned logger.error('Could not delete VM: {} Exception: {}'.format(vm_name, e)) return False
def provision_vm(tuple_list): """Create a provision request for a VM This method expects a list of tuples, each of which have the format: (vm_name, template_guid, vlan)""" starttime = time.time() data_dict = {'action': 'create', 'resources': []} for vm_name, template_guid, vlan in tuple_list: data_dict['resources'].append({ 'template_fields': { 'guid': template_guid }, 'vm_fields': { 'number_of_sockets': 1, 'cores_per_socket': 1, 'vm_name': vm_name, 'vm_memory': '1024', 'vlan': vlan, 'vm_auto_start': True, 'provision_type': 'native_clone' }, 'requester': { 'user_name': 'admin', 'owner_first_name': 'FirstName', 'owner_last_name': 'LastName', 'owner_email': '*****@*****.**', 'auto_approve': True }, 'additional_values': { 'request_id': '1001' }, 'ems_custom_attributes': {}, 'miq_custom_attributes': {} }) data_json = json.dumps(data_dict) appliance = cfme_appliance['ip_address'] response = requests.post("https://" + appliance + "/api/provision_requests/", data=data_json, auth=(cfme_appliance['rest_api']['username'], cfme_appliance['rest_api']['password']), verify=False, headers={"content-type": "application/json"}, allow_redirects=False) if response.status_code != 200: logger.debug(response.text) logger.debug('Queued Provision VM {} in: {}s'.format( vm_name, round(time.time() - starttime, 2)))
def forward(self, sock, remote, fix): content_encoding = None left_length = 0 try: fdset = [sock, remote] while True: r, w, e = select.select(fdset, [], []) if sock in r: data = sock.recv(32768) if len(data) <= 0: break remote.sendall(data) if remote in r: data = remote.recv(32768) if len(data) <= 0: break if fix: if None == content_encoding: headers,body = data.split(b'\r\n\r\n', maxsplit=1) headers = headers.decode('iso-8859-1') match = re.search(r'Content-Encoding: (\S+)\r\n', headers) if match: content_encoding = match.group(1) else: content_encoding = '' match = re.search(r'Content-Length: (\d+)\r\n', headers) if match: content_length = int(match.group(1)) else: content_length = 0 left_length = content_length - len(body) else: left_length -= len(body) if left_length <= 0: content_encoding = None if 'gzip' == content_encoding: body = zlib.decompress(body ,15+32) for old in setting.config['content_fix'][self.host]: body = body.replace(old.encode('utf8'), setting.config['content_fix'][self.host][old].encode('utf8')) if None != content_encoding: headers = re.sub(r'Content-Encoding: (\S+)\r\n', r'', headers) headers = re.sub(r'Content-Length: (\d+)\r\n', r'Content-Length: '+str(len(body))+r'\r\n', headers) data = headers.encode('iso-8859-1') + b'\r\n\r\n' + body sock.sendall(data) except socket.error as e: logger.debug('Forward: %s' % e) finally: sock.close() remote.close()
def receive_task(self): # if len(index) == 0: # content = requests.get(m3u8_url).text if self.cfg.m3u8_url == "": return content = requests.get(self.cfg.m3u8_url).text lines = content.split('\n') for line in lines: # print(line) if line.endswith(".ts"): # logger.info(line) # i = line.replace("cHZnNjcxLWF2LzE2LzE=/", "").replace(".ts", "") i = line.replace(self.cfg.suffix, "").replace(".ts", "") if int(i) > self.pre_index or int(i) == 0: self.index_pool.put(int(i)) self.pre_index = int(i) # index.append(int(i)) # if len(index) == 0: # time.sleep(1) # continue # avoid video stream sever lost response if HTTP was frequent if self.index_pool.empty(): self.try_cnt += 1 if self.try_cnt % 20 == 0: logger.info( 'Video [{}]: Empty index response from online video. May please connect to video server to fix this network error ' 'or check the deployment server network connected.'.format( self.cfg.index)) time.sleep(1) return False # n = index[0] # index.remove(index[0]) current_index = self.index_pool.get() # url = "https://222.190.243.176:8081/proxy/video_ts/live/cHZnNjcxLWF2LzE2LzE=/" + str(n) + ".ts" # url = vcfg[URL] + str(n) + ".ts" url = self.cfg.url + str(current_index) + ".ts" # headers = {"User-Agent": "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"} # headers = {"User-Agent": "AppleCoreMedia/1.0.0.19B88(Macintosh;U;Intel Mac OS X 10_15_1;en_us)"} logger.debug('Send Request: [{}]'.format(url)) response = requests.get(url, headers=self.cfg.headers) logger.debug('Reponse status code: [{}]'.format(response.status_code)) if response.status_code == 404: logger.debug('Stream Not found: [{}]'.format(response.status_code)) return False # f = open(ts_localpath + "/" + "%03d.ts" % n, "wb") format_index = str("%03d.ts" % current_index) f = open(self.stream_save_path / format_index, "wb") f.write(response.content) f.close() # caches the stream index which has been completed by HTTP request. # note that the Queue is p self.pass_index(format_index) logger.debug("%03d.ts Download~" % current_index)
def get_any_methodcall(self, method_name, call_params, unserchain=[], define_param=(), deepth=0): """ 可以调用任意类的某个方法,跟踪分析 :param method_name: :param call_params: :param unserchain: :param define_param: :param deepth: :return: """ deepth += 1 define_param = (*call_params, *define_param) method_node_name = 'Method-{}'.format(method_name) call_nodes = self.dataflow_db.objects.filter( node_type='newMethod', source_node__startswith=method_node_name) logger.debug("[PhpUnSerChain] trigger {}{}. try to found it.".format( method_node_name, call_params)) for node in call_nodes: logger.debug("[PhpUnSerChain] Found New () in {}".format( method_node_name, node.node_locate)) # 为了不影响数据,要先生成新的 newunserchain = [node] class_locate = node.node_locate new_locate = node.node_locate + '.' + node.source_node method_nodes = self.dataflow_db.objects.filter( node_locate__startswith=new_locate) status = self.deep_search_chain(method_nodes, class_locate, newunserchain, define_param=define_param, deepth=deepth) if status: unserchain.extend(newunserchain) return True return False
def update_and_new_project_vendor(project_id, name, version, language, source=None, ext=None): hash = md5("{},{},{},{}".format(project_id, name, language, source)) vendor = ProjectVendors.objects.filter(project_id=project_id, name=name, language=language).first() if vendor: # 兼容性处理,如果source未指定,先更新source进去 if not vendor.source: vendor.version = version vendor.source = source vendor.ext = ext vendor.version = version vendor.hash = hash try: vendor.save() except IntegrityError: logger.warn("[Model Save] vendor model not changed") else: vendor = ProjectVendors.objects.filter(project_id=project_id, hash=hash).first() if vendor: if vendor.version != version and version != 'unknown': logger.debug("[Vendors] Component {} update to version {}".format( name, version)) vendor.version = version try: vendor.save() except IntegrityError: logger.warn("[Model Save] vendor model not changed") else: v = ProjectVendors(project_id=project_id, name=name, version=version, language=language, ext=ext) v.save() return True
def select(root, sub=None, invokes_alert=False): """ Clicks on a button by calling the :py:meth:`click_n_move` method. Args: root: The root button's name as a string. sub: The sub button's name as a string. (optional) invokes_alert: If ``True``, then the behaviour is little bit different. After the last click, no ajax wait and no move away is done to be able to operate the alert that appears after click afterwards. Defaults to ``False``. Returns: ``True`` if everything went smoothly Raises: :py:class:`cfme.exceptions.ToolbarOptionGreyed` """ # wait for ajax on select to prevent pickup up a toolbar button in the middle of a page change sel.wait_for_ajax() if isinstance(root, dict): root = version.pick(root) if sub is not None and isinstance(sub, dict): sub = version.pick(sub) if not is_greyed(root): try: if sub is None and invokes_alert: # We arrived into a place where alert will pop up so no moving and no ajax sel.click(root_loc(root), wait_ajax=False) else: select_n_move(root_loc(root)) except sel.NoSuchElementException: raise ToolbarOptionUnavailable( "Toolbar button '{}' was not found.".format(root)) except sel.StaleElementReferenceException: logger.debug('Stale toolbar button "{}", relocating'.format(root)) select(root, sub, invokes_alert) else: raise ToolbarOptionGreyed("Toolbar button {} is greyed!".format(root)) if sub: sel.wait_for_ajax() if not is_greyed(root, sub): try: if invokes_alert: # We arrived into a place where alert will pop up so no moving and no ajax sel.click(sub_loc(sub), wait_ajax=False) else: select_n_move(sub_loc(sub)) except sel.NoSuchElementException: raise ToolbarOptionUnavailable( "Toolbar button '{}/{}' was not found.".format(root, sub)) else: raise ToolbarOptionGreyed("Toolbar option {}/{} is greyed!".format( root, sub)) return True