def fetch(self, args): """ fetch a remote control file or packages """ if not len(args): self.help() url = args.pop(0) if not utils.is_url(url): logging.info("Not a remote url, nothing to fetch (%s)", url) self.help() if len(args): name = args.pop(0) else: name = "" logging.info("Fetching file %s:%s", url, name) pkg_dir = os.path.join(output_dir, 'packages') install_dir = os.path.join(FETCHDIRTEST, name) pkgmgr = packages.PackageManager(output_dir, run_function_dargs={'timeout':3600}) pkgmgr.install_pkg(name, 'test', pkg_dir, install_dir, repo_url=url) raise SystemExit(0)
def _install_using_packaging(self, host, autodir): repos = self.get_fetch_location() if not repos: raise error.PackageInstallError("No repos to install an " "autotest client from") pkgmgr = packages.PackageManager(autodir, hostname=host.hostname, repo_urls=repos, do_locking=False, run_function=host.run, run_function_dargs=dict(timeout=600)) # The packages dir is used to store all the packages that # are fetched on that client. (for the tests,deps etc. # too apart from the client) pkg_dir = os.path.join(autodir, 'packages') # clean up the autodir except for the packages directory host.run('cd %s && ls | grep -v "^packages$"' ' | xargs rm -rf && rm -rf .[^.]*' % autodir) pkgmgr.install_pkg('autotest', 'client', pkg_dir, autodir, preserve_install_dir=True) self._create_test_output_dir(host, autodir) logging.info("Installation of autotest completed") self.installed = True
def fetch(self, args): """ fetch a remote control file or packages """ if not len(args): self.help() url = args.pop(0) if not utils.is_url(url): logging.info("Not a remote url, nothing to fetch (%s)", url) self.help() if len(args): name = args.pop(0) else: name = "" logging.info("Fetching file %s:%s", url, name) autodir = os.path.abspath(os.environ['AUTODIR']) tmpdir = os.path.join(autodir, 'tmp') tests_out_dir = GLOBAL_CONFIG.get_config_value('COMMON', 'test_output_dir', default=tmpdir) pkg_dir = os.path.join(tests_out_dir, 'packages') install_dir = os.path.join(FETCHDIRTEST, name) pkgmgr = packages.PackageManager(tests_out_dir, run_function_dargs={'timeout': 3600}) pkgmgr.install_pkg(name, 'test', pkg_dir, install_dir, repo_url=url) raise SystemExit(0)
def _init_packages(self): """ Perform the packages support initialization. """ tmpdir = settings.get_value('COMMON', 'test_output_dir', default=self.autodir) self.pkgmgr = packages.PackageManager( tmpdir, run_function_dargs={'timeout': 3600})
def __init__(self, options): """ Since setup_job is a client job but run on a server, it takes no control file as input. So client_job.__init__ is by-passed. :param options: an object passed in from command line OptionParser. See all options defined on client/autotest. """ base_job.base_job.__init__(self, options=options) self._cleanup_debugdir_files() self._cleanup_results_dir() self.pkgmgr = packages.PackageManager( self.autodir, run_function_dargs={'timeout': 3600})
def _do_run(self, control_file, results_dir, host, atrun, timeout, client_disconnect_timeout): try: atrun.verify_machine() except: logging.error("Verify failed on %s. Reinstalling autotest", host.hostname) self.install(host) atrun.verify_machine() debug = os.path.join(results_dir, 'debug') try: os.makedirs(debug) except Exception: pass delete_file_list = [ atrun.remote_control_file, atrun.remote_control_state, atrun.manual_control_file, atrun.manual_control_state ] cmd = ';'.join('rm -f ' + control for control in delete_file_list) host.run(cmd, ignore_status=True) tmppath = utils.get(control_file) # build up the initialization prologue for the control file prologue_lines = [] # Add the additional user arguments prologue_lines.append("args = %r\n" % self.job.args) # If the packaging system is being used, add the repository list. repos = None try: repos = self.get_fetch_location() pkgmgr = packages.PackageManager('autotest', hostname=host.hostname, repo_urls=repos) prologue_lines.append('job.add_repository(%s)\n' % repos) except global_config.ConfigError, e: # If repos is defined packaging is enabled so log the error if repos: logging.error(e)
def __init__(self, control, args, resultdir, label, user, machines, client=False, parse_job='', ssh_user='******', ssh_port=22, ssh_pass='', group_name='', tag='', control_filename=SERVER_CONTROL_FILENAME): """ Create a server side job object. @param control: The pathname of the control file. @param args: Passed to the control file. @param resultdir: Where to throw the results. @param label: Description of the job. @param user: Username for the job (email address). @param client: True if this is a client-side control file. @param parse_job: string, if supplied it is the job execution tag that the results will be passed through to the TKO parser with. @param ssh_user: The SSH username. [root] @param ssh_port: The SSH port number. [22] @param ssh_pass: The SSH passphrase, if needed. @param group_name: If supplied, this will be written out as host_group_name in the keyvals file for the parser. @param tag: The job execution tag from the scheduler. [optional] @param control_filename: The filename where the server control file should be written in the results directory. """ super(base_server_job, self).__init__(resultdir=resultdir) path = os.path.dirname(__file__) self.control = control self._uncollected_log_file = os.path.join(self.resultdir, 'uncollected_logs') debugdir = os.path.join(self.resultdir, 'debug') if not os.path.exists(debugdir): os.mkdir(debugdir) if user: self.user = user else: self.user = getpass.getuser() self.args = args self.machines = machines self._client = client self.warning_loggers = set() self.warning_manager = warning_manager() self._ssh_user = ssh_user self._ssh_port = ssh_port self._ssh_pass = ssh_pass self.tag = tag self.last_boot_tag = None self.hosts = set() self.drop_caches = False self.drop_caches_between_iterations = False self._control_filename = control_filename self.logging = logging_manager.get_logging_manager( manage_stdout_and_stderr=True, redirect_fds=True) subcommand.logging_manager_object = self.logging self.sysinfo = sysinfo.sysinfo(self.resultdir) self.profilers = profilers.profilers(self) job_data = {'label' : label, 'user' : user, 'hostname' : ','.join(machines), 'drone' : platform.node(), 'status_version' : str(self._STATUS_VERSION), 'job_started' : str(int(time.time()))} if group_name: job_data['host_group_name'] = group_name # only write these keyvals out on the first job in a resultdir if 'job_started' not in utils.read_keyval(self.resultdir): job_data.update(get_site_job_data(self)) utils.write_keyval(self.resultdir, job_data) self._parse_job = parse_job self._using_parser = (self._parse_job and len(machines) <= 1) self.pkgmgr = packages.PackageManager( self.autodir, run_function_dargs={'timeout':600}) self.num_tests_run = 0 self.num_tests_failed = 0 self._register_subcommand_hooks() # these components aren't usable on the server self.bootloader = None self.harness = None # set up the status logger self._indenter = status_indenter() self._logger = base_job.status_logger( self, self._indenter, 'status.log', 'status.log', record_hook=server_job_record_hook(self))
def main(): # get options and args options, args = parse_args() server_dir = server_utils.get_server_dir() autotest_dir = os.path.abspath(os.path.join(server_dir, '..')) # extract the pkg locations from global config repo_urls = settings.get_value('PACKAGES', 'fetch_location', type=list, default=[]) upload_paths = settings.get_value('PACKAGES', 'upload_location', type=list, default=[]) if options.repo: upload_paths.append(options.repo) # Having no upload paths basically means you're not using packaging. if not upload_paths: print( "No upload locations found. Please set upload_location under" " PACKAGES in the global_config.ini or provide a location using" " the --repository option.") return client_dir = os.path.join(autotest_dir, "client") # Bail out if the client directory does not exist if not os.path.exists(client_dir): sys.exit(0) dep_dir = os.path.join(client_dir, "deps") prof_dir = os.path.join(client_dir, "profilers") if len(args) == 0 or args[0] not in ['upload', 'remove']: print( "Either 'upload' or 'remove' needs to be specified " "for the package") sys.exit(0) if args[0] == 'upload': remove_flag = False elif args[0] == 'remove': remove_flag = True else: # we should not be getting here assert (False) pkgmgr = packages.PackageManager(autotest_dir, repo_urls=repo_urls, upload_paths=upload_paths, run_function_dargs={'timeout': 600}) if options.all: process_all_packages(pkgmgr, client_dir, remove=remove_flag) if options.client: process_packages(pkgmgr, 'client', 'autotest', client_dir, remove=remove_flag) if options.dep: process_packages(pkgmgr, 'dep', options.dep, dep_dir, remove=remove_flag) if options.test: process_packages(pkgmgr, 'test', options.test, client_dir, remove=remove_flag) if options.prof: process_packages(pkgmgr, 'profiler', options.prof, prof_dir, remove=remove_flag) if options.file: if remove_flag: pkgmgr.remove_pkg(options.file, remove_checksum=True) else: pkgmgr.upload_pkg(options.file, update_checksum=True)
def _init_packages(self): """ Perform the packages support initialization. """ self.pkgmgr = packages.PackageManager( self.autodir, run_function_dargs={'timeout':3600})