def auth_headers_for_method_call_on(self, url, auth_backend):
parsed = urlparse.urlparse(url)
mocker = Mocker()
transport = mocker.mock()
auth_data = []
def intercept_request(host, handler, request_body, verbose=0):
actual_transport = XMLRPCTransport(parsed.scheme, auth_backend)
request = actual_transport.build_http_request(
host, handler, request_body)
if (request.has_header('Authorization')):
auth_data.append(request.get_header('Authorization'))
response_body = xmlrpclib.dumps((1, ), methodresponse=True)
response = StringIO.StringIO(response_body)
response.status = 200
response.__len__ = lambda: len(response_body)
transport.request(ARGS, KWARGS)
mocker.call(intercept_request)
mocker.result(response)
with mocker:
server_proxy = AuthenticatingServerProxy(url,
auth_backend=auth_backend,
transport=transport)
server_proxy.method()
return auth_data
def __init__(self, parser, args):
super(pull, self).__init__(parser, args)
remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM)
self.remote_server = AuthenticatingServerProxy(
remote_xml_rpc_url,
verbose=args.verbose_xml_rpc,
use_datetime=True,
allow_none=True,
auth_backend=KeyringAuthBackend())
self.use_non_legacy_api_if_possible('remote_server')
def invoke(self):
uri = normalize_xmlrpc_url(self.args.HOST)
parsed_host = urlparse.urlparse(uri)
if parsed_host.username:
username = parsed_host.username
else:
username = getpass.getuser()
host = parsed_host.hostname
if parsed_host.port:
host += ':' + str(parsed_host.port)
uri = '%s://%s@%s%s' % (parsed_host.scheme, username, host,
parsed_host.path)
if self.args.token_file:
if parsed_host.password:
raise LavaCommandError(
"Token specified in url but --token-file also passed.")
else:
try:
token_file = open(self.args.token_file)
except IOError as ex:
raise LavaCommandError("opening %r failed: %s" %
(self.args.token_file, ex))
token = token_file.read().strip()
else:
if parsed_host.password:
token = parsed_host.password
else:
token = getpass.getpass("Paste token for %s: " % uri)
userless_uri = '%s://%s%s' % (parsed_host.scheme, host,
parsed_host.path)
if not self.args.no_check:
sp = AuthenticatingServerProxy(uri,
auth_backend=MemoryAuthBackend([
(username, userless_uri, token)
]))
try:
token_user = sp.system.whoami()
except xmlrpclib.ProtocolError as ex:
if ex.errcode == 401:
raise LavaCommandError(
"Token rejected by server for user %s." % username)
else:
raise
except xmlrpclib.Fault as ex:
raise LavaCommandError(
"Server reported error during check: %s." % ex)
if token_user != username:
raise LavaCommandError(
"whoami() returned %s rather than expected %s -- this is "
"a bug." % (token_user, username))
self.auth_backend.add_token(username, userless_uri, token)
print 'Token added successfully for user %s.' % username
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
try:
job_id = server.scheduler.resubmit_job(self.args.JOB_ID)
except xmlrpclib.Fault, e:
raise CommandError(str(e))
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
jobs_list = server.scheduler.all_jobs()
data_for_renderer = []
for row in jobs_list:
if row[6]:
job_id = row[6]
else:
job_id = row[0]
if row[4]:
req_d = row[4]["hostname"]
else:
req_d = row[5]["name"]
if row[3]:
req_d = ""
act_d = row[3]["hostname"]
else:
act_d = ""
data_for_renderer.append(
dict(
zip([
'id', 'description', 'status', 'actual_device',
'requested_device'
], [job_id, row[1], row[2], act_d, req_d])))
self.renderer.render(data_for_renderer)
def invoke(self):
if self.args.output is None and not self.args.stdout:
filename = str(self.args.DEVICE_HOSTNAME) + '_config.yaml'
if os.path.exists(filename) and not self.args.overwrite:
print >> sys.stderr, "File {filename!r} already exists".format(
filename=filename)
print >> sys.stderr, "You may pass --overwrite to write over it"
return -1
stream = open(filename, "wb")
elif self.args.stdout:
stream = sys.stdout
filename = "stdout"
else:
stream = self.args.output
filename = self.args.output.name
server = AuthenticatingServerProxy(
self.args.SERVER, auth_backend=KeyringAuthBackend())
try:
stream.write(server.scheduler.get_pipeline_device_config(
self.args.DEVICE_HOSTNAME).data)
print "Downloaded device config of {0} to file {1!r}".format(
self.args.DEVICE_HOSTNAME, filename)
except xmlrpclib.Fault as exc:
print >> sys.stderr, exc
return -1
def _get_scheduler_rpc():
"""Returns the scheduler xmlrpc AuthicatingServerProxy object.
"""
username = '******' # We assume this user exists always.
user = User.objects.get(username=username)
rpc2_url = Worker.get_rpc2_url()
try:
token = AuthToken.objects.filter(user=user)[0]
except IndexError:
token = AuthToken.objects.create(user=user)
token.save()
parsed_server = urlparse.urlparse(rpc2_url)
server = '{0}://{1}:{2}@{3}'.format(parsed_server.scheme, username,
token.secret, parsed_server.hostname)
if parsed_server.port:
server += ':' + str(parsed_server.port)
server += parsed_server.path
auth_backend = MemoryAuthBackend([(username, rpc2_url, token.secret)])
server = AuthenticatingServerProxy(server, auth_backend=auth_backend)
server = server.scheduler
return server
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
job_details = server.scheduler.job_details(self.args.JOB_ID)
print "Details of job {0}: \n".format(str(self.args.JOB_ID))
for detail in job_details:
print "%s: %s" % (detail, job_details[detail])
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
job_status = server.scheduler.job_status(self.args.JOB_ID)
print "Job ID: %s\nJob Status: %s\nBundle SHA1: %s" % \
(str(self.args.JOB_ID), job_status['job_status'],
job_status['bundle_sha1'])
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
with open(self.args.JSON_FILE, 'rb') as stream:
command_text = stream.read()
try:
job_id = server.scheduler.submit_job(command_text)
except xmlrpclib.Fault, e:
raise CommandError(str(e))
def __init__(self, parser, args):
super(XMLRPCCommand, self).__init__(parser, args)
xml_rpc_url = self._construct_xml_rpc_url(self.args.dashboard_url)
self.server = AuthenticatingServerProxy(
xml_rpc_url,
verbose=args.verbose_xml_rpc,
allow_none=True,
use_datetime=True,
auth_backend=KeyringAuthBackend())
def invoke(self):
# ServerProxy for [email protected]/RPC2/
server = AuthenticatingServerProxy(
self.args.SERVER, auth_backend=KeyringAuthBackend())
if not os.path.exists(self.args.JSON_FILE):
raise CommandError("No such file: %s" % self.args.JSON_FILE)
with open(self.args.JSON_FILE, 'rb') as stream:
command_text = stream.read()
try:
job_ids = server.scheduler.submit_job(command_text)
except xmlrpclib.Fault, e:
raise CommandError(str(e))
def submit_yaml_job(job_yaml_str, server_url):
#TOKEN = ""
#SERVER_URL="https://yongqin.liu:%[email protected]/RPC2/" % TOKEN
# server = xmlrpclib.ServerProxy(SERVER_URL)
server = AuthenticatingServerProxy(server_url,
auth_backend=KeyringAuthBackend())
try:
job_id = server.scheduler.submit_job(job_yaml_str)
print "Job %s submitted successfuly." % str(job_id)
except xmlrpclib.Fault as e:
raise e
except:
raise
def _get_dashboard(server, token):
if not server.endswith("/"):
server = ''.join([server, "/"])
# add backward compatible for 'dashboard/'-end URL
# Fix it: it's going to be deleted after transition
if server.endswith("dashboard/"):
server = ''.join([server, "xml-rpc/"])
logging.warning(
"Please use whole endpoint URL not just end with 'dashboard/', "
"'xml-rpc/' is added automatically now!!!")
parsed_server = urlparse.urlparse(server)
auth_backend = MemoryAuthBackend([])
if parsed_server.username:
if token:
userless_server = '%s://%s' % (parsed_server.scheme,
parsed_server.hostname)
if parsed_server.port:
userless_server += ':' + str(parsed_server.port)
userless_server += parsed_server.path
auth_backend = MemoryAuthBackend([(parsed_server.username,
userless_server, token)])
else:
logging.warning(
"specifying a user without a token is unlikely to work")
else:
if token:
logging.warning(
"specifying a token without a user is probably useless")
srv = AuthenticatingServerProxy(server,
allow_none=True,
use_datetime=True,
auth_backend=auth_backend)
if server.endswith("xml-rpc/"):
logging.error(
"Please use RPC2 endpoint instead, xml-rpc is no longer supported")
raise OperationFailed("xml-rpc endpoint is not supported.")
elif server.endswith("RPC2/"):
# include lava-server/RPC2/
dashboard = srv.dashboard
else:
logging.warning(
"The url seems not RPC2 or xml-rpc endpoints, please make sure it's a valid one!!!"
)
dashboard = srv.dashboard
logging.debug("server RPC endpoint URL: %s" % server)
return dashboard
def invoke(self):
server = AuthenticatingServerProxy(
self.args.SERVER, auth_backend=KeyringAuthBackend())
all_devices = server.scheduler.all_devices()
data_for_renderer = []
for row in all_devices:
if row[3]:
job_id = row[3]
else:
job_id = ""
data_for_renderer.append(dict(zip(['hostname', 'device_type_name', 'status', 'job'],
[row[0], row[1], row[2], job_id])))
self.renderer.render(data_for_renderer)
def authenticated_server(self):
"""Returns a connection to a LAVA server.
It will ask the user the necessary parameters to establish the
connection.
"""
print >> sys.stdout, "\nServer connection parameters:"
server_name_parameter = Parameter("server")
rpc_endpoint_parameter = Parameter("rpc_endpoint",
depends=server_name_parameter)
self.config.get(server_name_parameter)
endpoint = self.config.get(rpc_endpoint_parameter)
rpc_url = verify_and_create_url(endpoint)
server = AuthenticatingServerProxy(rpc_url,
auth_backend=KeyringAuthBackend())
return server
def invoke(self):
if self.args.output is None:
filename = str(self.args.JOB_ID) + '_output.txt'
if os.path.exists(filename) and not self.args.overwrite:
print >> sys.stderr, "File {filename!r} already exists".format(
filename=filename)
print >> sys.stderr, "You may pass --overwrite to write over it"
return -1
stream = open(filename, "wb")
else:
stream = self.args.output
filename = self.args.output.name
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
stream.write(server.scheduler.job_output(self.args.JOB_ID).data)
print "Downloaded job output of {0} to file {1!r}".format(
self.args.JOB_ID, filename)
# https://validation.linaro.org/scheduler/device_type/x15?dt_length=100&dt_search=android-lcr-member-x15-n-premerge-ci-153
#x15_build2_job_ids = ["1504877", "1504876", "1504874", "1504872", "1504870", "1504868", "1504866", "1504864", "1504862", "1504860", "1504858", "1504856", "1504854", "1504852", "1504850", "1504848", "1504846", "1504844", "1504842", "1504840", "1504838", "1504836", "1504834", "1504832", "1504830", "1504079", "1504078" ]
# master build
# https://validation.linaro.org/scheduler/device_type/x15?dt_length=100&dt_search=android-lcr-member-x15-n-premerge-ci-156#dt_
x15_build2_job_ids = [
"1511565", "1511564", "1511562", "1511560", "1511558", "1511556",
"1511554", "1511552", "1511550", "1511548", "1511546", "1511544",
"1511542", "1511540", "1511538", "1511536", "1511534", "1511532",
"1511530", "1511528", "1511526", "1511524", "1511522", "1511520",
"1511518", "1511142", "1511141"
]
lava_server_url = "https://[email protected]/RPC2/"
server = AuthenticatingServerProxy(lava_server_url,
auth_backend=KeyringAuthBackend())
badchars = "[^a-zA-Z0-9\._-]"
#server.dashboard.get("ced91695e2e2cbb8984a8d93ff4d638410b44f2a")
def getBundleIdWithJobId(job_id=None):
if job_id is None or job_id == "":
return None
job_details = server.scheduler.job_details(job_id)
job_status = job_details.get("status")
if job_status != "Complete":
#print "status:%s "% job_details.get("status")
return None
results_bundle = job_details.get("_results_link")
if results_bundle is None:
class pull(XMLRPCCommand):
"""
Copy bundles and bundle streams from one dashboard to another.
This command checks for two environment varialbes:
The value of DASHBOARD_URL is used as a replacement for --dashbard-url.
The value of REMOTE_DASHBOARD_URL as a replacement for FROM.
Their presence automatically makes the corresponding argument optional.
"""
def __init__(self, parser, args):
super(pull, self).__init__(parser, args)
remote_xml_rpc_url = self._construct_xml_rpc_url(self.args.FROM)
self.remote_server = AuthenticatingServerProxy(
remote_xml_rpc_url,
verbose=args.verbose_xml_rpc,
use_datetime=True,
allow_none=True,
auth_backend=KeyringAuthBackend())
self.use_non_legacy_api_if_possible('remote_server')
@classmethod
def register_arguments(cls, parser):
group = super(pull, cls).register_arguments(parser)
default_remote_dashboard_url = os.getenv("REMOTE_DASHBOARD_URL")
if default_remote_dashboard_url:
group.add_argument(
"FROM", nargs="?",
help="URL of the remote validation dashboard (currently %(default)s)",
default=default_remote_dashboard_url)
else:
group.add_argument(
"FROM",
help="URL of the remote validation dashboard)")
group.add_argument(
"STREAM",
nargs="*",
help="Streams to pull from (all by default)")
@staticmethod
def _filesizeformat(num_bytes):
"""
Formats the value like a 'human-readable' file size (i.e. 13 KB, 4.1 MB,
102 num_bytes, etc).
"""
try:
num_bytes = float(num_bytes)
except (TypeError, ValueError, UnicodeDecodeError):
return "%(size)d byte", "%(size)d num_bytes" % {'size': 0}
filesize_number_format = lambda value: "%0.2f" % (round(value, 1),)
if num_bytes < 1024:
return "%(size)d bytes" % {'size': num_bytes}
if num_bytes < 1024 * 1024:
return "%s KB" % filesize_number_format(num_bytes / 1024)
if num_bytes < 1024 * 1024 * 1024:
return "%s MB" % filesize_number_format(num_bytes / (1024 * 1024))
return "%s GB" % filesize_number_format(num_bytes / (1024 * 1024 * 1024))
def invoke_remote(self):
self._check_server_version(self.server, "0.3")
print "Checking local and remote streams"
remote = self.remote_server.streams()
if self.args.STREAM:
# Check that all requested streams are available remotely
requested_set = frozenset(self.args.STREAM)
remote_set = frozenset((stream["pathname"] for stream in remote))
unavailable_set = requested_set - remote_set
if unavailable_set:
print >> sys.stderr, "Remote stream not found: %s" % ", ".join(
unavailable_set)
return -1
# Limit to requested streams if necessary
remote = [
stream for stream in remote if stream[
"pathname"] in requested_set]
local = self.server.streams()
missing_pathnames = set([stream["pathname"]
for stream in remote]) - set([stream["pathname"] for stream in local])
for stream in remote:
if stream["pathname"] in missing_pathnames:
self.server.make_stream(stream["pathname"], stream["name"])
local_bundles = []
else:
local_bundles = [
bundle for bundle in self.server.bundles(stream["pathname"])]
remote_bundles = [
bundle for bundle in self.remote_server.bundles(stream["pathname"])]
missing_bundles = set(
(bundle["content_sha1"] for bundle in remote_bundles))
missing_bundles -= set(
(bundle["content_sha1"] for bundle in local_bundles))
try:
missing_bytes = sum(
(bundle["content_size"]
for bundle in remote_bundles
if bundle["content_sha1"] in missing_bundles))
except KeyError as ex:
# Older servers did not return content_size so this part is
# optional
missing_bytes = None
if missing_bytes:
print "Stream %s needs update (%s)" % (stream["pathname"], self._filesizeformat(missing_bytes))
elif missing_bundles:
print "Stream %s needs update (no estimate available)" % (stream["pathname"],)
else:
print "Stream %s is up to date" % (stream["pathname"],)
for content_sha1 in missing_bundles:
print "Getting %s" % (content_sha1,),
sys.stdout.flush()
data = self.remote_server.get(content_sha1)
print "got %s, storing" % (self._filesizeformat(len(data["content"]))),
sys.stdout.flush()
try:
self.server.put(
data["content"],
data["content_filename"],
stream["pathname"])
except xmlrpclib.Fault as ex:
if ex.faultCode == 409: # duplicate
print "already present (in another stream)"
else:
raise
else:
print "done"
"|regression_\d+"
"|arm64-v8a.+_executed"
"|arm64-v8a.+_failed"
"|armeabi-v7a.+_executed"
"|armeabi-v7a.+_failed"
")$")
names_ignore = [
"test-attachment",
"test-skipped",
"regression_4003_XTS",
"regression_4003_NO_XTS",
"subtests-fail-rate",
"tradefed-test-run",
]
server = AuthenticatingServerProxy(SERVER_URL,
auth_backend=KeyringAuthBackend())
#TOKEN = "n2ab47pbfbu4um0sw5r3zd22q1zdorj7nlnj3qaaaqwdfigahkn6j1kp0ze49jjir84cud7dq4kezhms0jrwy14k1m609e8q50kxmgn9je3zlum0yrlr0njxc87bpss9"
#SERVER_URL="https://yongqin.liu:%[email protected]/RPC2/" % TOKEN
def get_yaml_result(job_id, server):
tests_res = []
# server = xmlrpclib.ServerProxy(SERVER_URL)
try:
#job_id = server.scheduler.submit_job(job_yaml_str)
res = server.results.get_testjob_results_yaml(job_id)
#res = server.results.get_testjob_results_csv(job_id)
#print "results for job %s:" % str(job_id)
#print "%s" % str(yaml.load(res))
for test in yaml.load(res):
if test.get("suite") == "lava":
def invoke(self):
server = AuthenticatingServerProxy(self.args.SERVER,
auth_backend=KeyringAuthBackend())
server.scheduler.cancel_job(self.args.JOB_ID)