def auth_headers_for_method_call_on(self, url, auth_backend): parsed = urlparse.urlparse(url) mocker = Mocker() transport = mocker.mock() auth_data = [] def intercept_request(host, handler, request_body, verbose=0): actual_transport = XMLRPCTransport(parsed.scheme, auth_backend) request = actual_transport.build_http_request( host, handler, request_body) if (request.has_header('Authorization')): auth_data.append(request.get_header('Authorization')) response_body = xmlrpclib.dumps((1, ), methodresponse=True) response = StringIO.StringIO(response_body) response.status = 200 response.__len__ = lambda: len(response_body) transport.request(ARGS, KWARGS) mocker.call(intercept_request) mocker.result(response) with mocker: server_proxy = AuthenticatingServerProxy(url, auth_backend=auth_backend, transport=transport) server_proxy.method() return auth_data
def __init__(self, parser, args): super(pull, self).__init__(parser, args) remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM) self.remote_server = AuthenticatingServerProxy( remote_xml_rpc_url, verbose=args.verbose_xml_rpc, use_datetime=True, allow_none=True, auth_backend=KeyringAuthBackend()) self.use_non_legacy_api_if_possible('remote_server')
def invoke(self): uri = normalize_xmlrpc_url(self.args.HOST) parsed_host = urlparse.urlparse(uri) if parsed_host.username: username = parsed_host.username else: username = getpass.getuser() host = parsed_host.hostname if parsed_host.port: host += ':' + str(parsed_host.port) uri = '%s://%s@%s%s' % (parsed_host.scheme, username, host, parsed_host.path) if self.args.token_file: if parsed_host.password: raise LavaCommandError( "Token specified in url but --token-file also passed.") else: try: token_file = open(self.args.token_file) except IOError as ex: raise LavaCommandError("opening %r failed: %s" % (self.args.token_file, ex)) token = token_file.read().strip() else: if parsed_host.password: token = parsed_host.password else: token = getpass.getpass("Paste token for %s: " % uri) userless_uri = '%s://%s%s' % (parsed_host.scheme, host, parsed_host.path) if not self.args.no_check: sp = AuthenticatingServerProxy(uri, auth_backend=MemoryAuthBackend([ (username, userless_uri, token) ])) try: token_user = sp.system.whoami() except xmlrpclib.ProtocolError as ex: if ex.errcode == 401: raise LavaCommandError( "Token rejected by server for user %s." % username) else: raise except xmlrpclib.Fault as ex: raise LavaCommandError( "Server reported error during check: %s." % ex) if token_user != username: raise LavaCommandError( "whoami() returned %s rather than expected %s -- this is " "a bug." % (token_user, username)) self.auth_backend.add_token(username, userless_uri, token) print 'Token added successfully for user %s.' % username
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) try: job_id = server.scheduler.resubmit_job(self.args.JOB_ID) except xmlrpclib.Fault, e: raise CommandError(str(e))
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) jobs_list = server.scheduler.all_jobs() data_for_renderer = [] for row in jobs_list: if row[6]: job_id = row[6] else: job_id = row[0] if row[4]: req_d = row[4]["hostname"] else: req_d = row[5]["name"] if row[3]: req_d = "" act_d = row[3]["hostname"] else: act_d = "" data_for_renderer.append( dict( zip([ 'id', 'description', 'status', 'actual_device', 'requested_device' ], [job_id, row[1], row[2], act_d, req_d]))) self.renderer.render(data_for_renderer)
def invoke(self): if self.args.output is None and not self.args.stdout: filename = str(self.args.DEVICE_HOSTNAME) + '_config.yaml' if os.path.exists(filename) and not self.args.overwrite: print >> sys.stderr, "File {filename!r} already exists".format( filename=filename) print >> sys.stderr, "You may pass --overwrite to write over it" return -1 stream = open(filename, "wb") elif self.args.stdout: stream = sys.stdout filename = "stdout" else: stream = self.args.output filename = self.args.output.name server = AuthenticatingServerProxy( self.args.SERVER, auth_backend=KeyringAuthBackend()) try: stream.write(server.scheduler.get_pipeline_device_config( self.args.DEVICE_HOSTNAME).data) print "Downloaded device config of {0} to file {1!r}".format( self.args.DEVICE_HOSTNAME, filename) except xmlrpclib.Fault as exc: print >> sys.stderr, exc return -1
def _get_scheduler_rpc(): """Returns the scheduler xmlrpc AuthicatingServerProxy object. """ username = '******' # We assume this user exists always. user = User.objects.get(username=username) rpc2_url = Worker.get_rpc2_url() try: token = AuthToken.objects.filter(user=user)[0] except IndexError: token = AuthToken.objects.create(user=user) token.save() parsed_server = urlparse.urlparse(rpc2_url) server = '{0}://{1}:{2}@{3}'.format(parsed_server.scheme, username, token.secret, parsed_server.hostname) if parsed_server.port: server += ':' + str(parsed_server.port) server += parsed_server.path auth_backend = MemoryAuthBackend([(username, rpc2_url, token.secret)]) server = AuthenticatingServerProxy(server, auth_backend=auth_backend) server = server.scheduler return server
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) job_details = server.scheduler.job_details(self.args.JOB_ID) print "Details of job {0}: \n".format(str(self.args.JOB_ID)) for detail in job_details: print "%s: %s" % (detail, job_details[detail])
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) job_status = server.scheduler.job_status(self.args.JOB_ID) print "Job ID: %s\nJob Status: %s\nBundle SHA1: %s" % \ (str(self.args.JOB_ID), job_status['job_status'], job_status['bundle_sha1'])
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) with open(self.args.JSON_FILE, 'rb') as stream: command_text = stream.read() try: job_id = server.scheduler.submit_job(command_text) except xmlrpclib.Fault, e: raise CommandError(str(e))
def __init__(self, parser, args): super(XMLRPCCommand, self).__init__(parser, args) xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url) self.server = AuthenticatingServerProxy( xml_rpc_url, verbose=args.verbose_xml_rpc, allow_none=True, use_datetime=True, auth_backend=KeyringAuthBackend())
def invoke(self): # ServerProxy for [email protected]/RPC2/ server = AuthenticatingServerProxy( self.args.SERVER, auth_backend=KeyringAuthBackend()) if not os.path.exists(self.args.JSON_FILE): raise CommandError("No such file: %s" % self.args.JSON_FILE) with open(self.args.JSON_FILE, 'rb') as stream: command_text = stream.read() try: job_ids = server.scheduler.submit_job(command_text) except xmlrpclib.Fault, e: raise CommandError(str(e))
def submit_yaml_job(job_yaml_str, server_url): #TOKEN = "" #SERVER_URL="https://yongqin.liu:%[email protected]/RPC2/" % TOKEN # server = xmlrpclib.ServerProxy(SERVER_URL) server = AuthenticatingServerProxy(server_url, auth_backend=KeyringAuthBackend()) try: job_id = server.scheduler.submit_job(job_yaml_str) print "Job %s submitted successfuly." % str(job_id) except xmlrpclib.Fault as e: raise e except: raise
def _get_dashboard(server, token): if not server.endswith("/"): server = ''.join([server, "/"]) # add backward compatible for 'dashboard/'-end URL # Fix it: it's going to be deleted after transition if server.endswith("dashboard/"): server = ''.join([server, "xml-rpc/"]) logging.warning( "Please use whole endpoint URL not just end with 'dashboard/', " "'xml-rpc/' is added automatically now!!!") parsed_server = urlparse.urlparse(server) auth_backend = MemoryAuthBackend([]) if parsed_server.username: if token: userless_server = '%s://%s' % (parsed_server.scheme, parsed_server.hostname) if parsed_server.port: userless_server += ':' + str(parsed_server.port) userless_server += parsed_server.path auth_backend = MemoryAuthBackend([(parsed_server.username, userless_server, token)]) else: logging.warning( "specifying a user without a token is unlikely to work") else: if token: logging.warning( "specifying a token without a user is probably useless") srv = AuthenticatingServerProxy(server, allow_none=True, use_datetime=True, auth_backend=auth_backend) if server.endswith("xml-rpc/"): logging.error( "Please use RPC2 endpoint instead, xml-rpc is no longer supported") raise OperationFailed("xml-rpc endpoint is not supported.") elif server.endswith("RPC2/"): # include lava-server/RPC2/ dashboard = srv.dashboard else: logging.warning( "The url seems not RPC2 or xml-rpc endpoints, please make sure it's a valid one!!!" ) dashboard = srv.dashboard logging.debug("server RPC endpoint URL: %s" % server) return dashboard
def invoke(self): server = AuthenticatingServerProxy( self.args.SERVER, auth_backend=KeyringAuthBackend()) all_devices = server.scheduler.all_devices() data_for_renderer = [] for row in all_devices: if row[3]: job_id = row[3] else: job_id = "" data_for_renderer.append(dict(zip(['hostname', 'device_type_name', 'status', 'job'], [row[0], row[1], row[2], job_id]))) self.renderer.render(data_for_renderer)
def authenticated_server(self): """Returns a connection to a LAVA server. It will ask the user the necessary parameters to establish the connection. """ print >> sys.stdout, "\nServer connection parameters:" server_name_parameter = Parameter("server") rpc_endpoint_parameter = Parameter("rpc_endpoint", depends=server_name_parameter) self.config.get(server_name_parameter) endpoint = self.config.get(rpc_endpoint_parameter) rpc_url = verify_and_create_url(endpoint) server = AuthenticatingServerProxy(rpc_url, auth_backend=KeyringAuthBackend()) return server
def invoke(self): if self.args.output is None: filename = str(self.args.JOB_ID) + '_output.txt' if os.path.exists(filename) and not self.args.overwrite: print >> sys.stderr, "File {filename!r} already exists".format( filename=filename) print >> sys.stderr, "You may pass --overwrite to write over it" return -1 stream = open(filename, "wb") else: stream = self.args.output filename = self.args.output.name server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) stream.write(server.scheduler.job_output(self.args.JOB_ID).data) print "Downloaded job output of {0} to file {1!r}".format( self.args.JOB_ID, filename)
# https://validation.linaro.org/scheduler/device_type/x15?dt_length=100&dt_search=android-lcr-member-x15-n-premerge-ci-153 #x15_build2_job_ids = ["1504877", "1504876", "1504874", "1504872", "1504870", "1504868", "1504866", "1504864", "1504862", "1504860", "1504858", "1504856", "1504854", "1504852", "1504850", "1504848", "1504846", "1504844", "1504842", "1504840", "1504838", "1504836", "1504834", "1504832", "1504830", "1504079", "1504078" ] # master build # https://validation.linaro.org/scheduler/device_type/x15?dt_length=100&dt_search=android-lcr-member-x15-n-premerge-ci-156#dt_ x15_build2_job_ids = [ "1511565", "1511564", "1511562", "1511560", "1511558", "1511556", "1511554", "1511552", "1511550", "1511548", "1511546", "1511544", "1511542", "1511540", "1511538", "1511536", "1511534", "1511532", "1511530", "1511528", "1511526", "1511524", "1511522", "1511520", "1511518", "1511142", "1511141" ] lava_server_url = "https://[email protected]/RPC2/" server = AuthenticatingServerProxy(lava_server_url, auth_backend=KeyringAuthBackend()) badchars = "[^a-zA-Z0-9\._-]" #server.dashboard.get("ced91695e2e2cbb8984a8d93ff4d638410b44f2a") def getBundleIdWithJobId(job_id=None): if job_id is None or job_id == "": return None job_details = server.scheduler.job_details(job_id) job_status = job_details.get("status") if job_status != "Complete": #print "status:%s "% job_details.get("status") return None results_bundle = job_details.get("_results_link") if results_bundle is None:
class pull(XMLRPCCommand): """ Copy bundles and bundle streams from one dashboard to another. This command checks for two environment varialbes: The value of DASHBOARD_URL is used as a replacement for --dashbard-url. The value of REMOTE_DASHBOARD_URL as a replacement for FROM. Their presence automatically makes the corresponding argument optional. """ def __init__(self, parser, args): super(pull, self).__init__(parser, args) remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM) self.remote_server = AuthenticatingServerProxy( remote_xml_rpc_url, verbose=args.verbose_xml_rpc, use_datetime=True, allow_none=True, auth_backend=KeyringAuthBackend()) self.use_non_legacy_api_if_possible('remote_server') @classmethod def register_arguments(cls, parser): group = super(pull, cls).register_arguments(parser) default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL") if default_remote_dashboard_url: group.add_argument( "FROM", nargs="?", help="URL of the remote validation dashboard (currently %(default)s)", default=default_remote_dashboard_url) else: group.add_argument( "FROM", help="URL of the remote validation dashboard)") group.add_argument( "STREAM", nargs="*", help="Streams to pull from (all by default)") @staticmethod def _filesizeformat(num_bytes): """ Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB, 102 num_bytes, etc). """ try: num_bytes = float(num_bytes) except (TypeError, ValueError, UnicodeDecodeError): return "%(size)d byte", "%(size)d num_bytes" % {'size': 0} filesize_number_format = lambda value: "%0.2f" % (round(value, 1),) if num_bytes < 1024: return "%(size)d bytes" % {'size': num_bytes} if num_bytes < 1024 * 1024: return "%s KB" % filesize_number_format(num_bytes / 1024) if num_bytes < 1024 * 1024 * 1024: return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024)) return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024)) def invoke_remote(self): self._check_server_version(self.server, "0.3") print "Checking local and remote streams" remote = self.remote_server.streams() if self.args.STREAM: # Check that all requested streams are available remotely requested_set = frozenset(self.args.STREAM) remote_set = frozenset((stream["pathname"] for stream in remote)) unavailable_set = requested_set - remote_set if unavailable_set: print >> sys.stderr, "Remote stream not found: %s" % ", ".join( unavailable_set) return -1 # Limit to requested streams if necessary remote = [ stream for stream in remote if stream[ "pathname"] in requested_set] local = self.server.streams() missing_pathnames = set([stream["pathname"] for stream in remote]) - set([stream["pathname"] for stream in local]) for stream in remote: if stream["pathname"] in missing_pathnames: self.server.make_stream(stream["pathname"], stream["name"]) local_bundles = [] else: local_bundles = [ bundle for bundle in self.server.bundles(stream["pathname"])] remote_bundles = [ bundle for bundle in self.remote_server.bundles(stream["pathname"])] missing_bundles = set( (bundle["content_sha1"] for bundle in remote_bundles)) missing_bundles -= set( (bundle["content_sha1"] for bundle in local_bundles)) try: missing_bytes = sum( (bundle["content_size"] for bundle in remote_bundles if bundle["content_sha1"] in missing_bundles)) except KeyError as ex: # Older servers did not return content_size so this part is # optional missing_bytes = None if missing_bytes: print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes)) elif missing_bundles: print "Stream %s needs update (no estimate available)" % (stream["pathname"],) else: print "Stream %s is up to date" % (stream["pathname"],) for content_sha1 in missing_bundles: print "Getting %s" % (content_sha1,), sys.stdout.flush() data = self.remote_server.get(content_sha1) print "got %s, storing" % (self._filesizeformat(len(data["content"]))), sys.stdout.flush() try: self.server.put( data["content"], data["content_filename"], stream["pathname"]) except xmlrpclib.Fault as ex: if ex.faultCode == 409: # duplicate print "already present (in another stream)" else: raise else: print "done"
"|regression_\d+" "|arm64-v8a.+_executed" "|arm64-v8a.+_failed" "|armeabi-v7a.+_executed" "|armeabi-v7a.+_failed" ")$") names_ignore = [ "test-attachment", "test-skipped", "regression_4003_XTS", "regression_4003_NO_XTS", "subtests-fail-rate", "tradefed-test-run", ] server = AuthenticatingServerProxy(SERVER_URL, auth_backend=KeyringAuthBackend()) #TOKEN = "n2ab47pbfbu4um0sw5r3zd22q1zdorj7nlnj3qaaaqwdfigahkn6j1kp0ze49jjir84cud7dq4kezhms0jrwy14k1m609e8q50kxmgn9je3zlum0yrlr0njxc87bpss9" #SERVER_URL="https://yongqin.liu:%[email protected]/RPC2/" % TOKEN def get_yaml_result(job_id, server): tests_res = [] # server = xmlrpclib.ServerProxy(SERVER_URL) try: #job_id = server.scheduler.submit_job(job_yaml_str) res = server.results.get_testjob_results_yaml(job_id) #res = server.results.get_testjob_results_csv(job_id) #print "results for job %s:" % str(job_id) #print "%s" % str(yaml.load(res)) for test in yaml.load(res): if test.get("suite") == "lava":
def invoke(self): server = AuthenticatingServerProxy(self.args.SERVER, auth_backend=KeyringAuthBackend()) server.scheduler.cancel_job(self.args.JOB_ID)