def cli(args=sys.argv[1:]):
parser = OptionParser()
parser.add_option("--timestamp", dest="timestamp", help="timestamp of "
"inbound build")
parser.add_option("-a", "--addon", dest="addons",
help="an addon to install; repeat for multiple addons",
metavar="PATH1", default=[], action="append")
parser.add_option("-p", "--profile", dest="profile",
help="path to profile to user", metavar="PATH")
parser.add_option("--bits", dest="bits",
help="force 32 or 64 bit version (only applies to"
" x86_64 boxes)",
choices=("32", "64"), default=mozinfo.bits)
parser.add_option("--persist", dest="persist",
help="the directory in which files are to persist"
" ie. /Users/someuser/Documents")
options, args = parser.parse_args(args)
if not options.timestamp:
print "timestamp must be specified"
sys.exit(1)
options.bits = parse_bits(options.bits)
runner = InboundRunner(addons=options.addons, profile=options.profile,
bits=options.bits, persist=options.persist)
runner.start(get_date(options.date))
try:
runner.wait()
except KeyboardInterrupt:
runner.stop()
def test_invalid_date(self, stdout):
stdout_data = []
stdout.write = lambda text: stdout_data.append(text)
date = utils.get_date("invalid_format")
self.assertIsNone(date)
self.assertIn("Incorrect date format", ''.join(stdout_data))
def test_invalid_date(self, stdout):
stdout_data = []
stdout.write = lambda text: stdout_data.append(text)
date = utils.get_date("invalid_format")
self.assertIsNone(date)
self.assertIn("Incorrect date format", ''.join(stdout_data))
def cli(args=sys.argv[1:]):
"""moznightly command line entry point"""
# parse command line options
parser = OptionParser()
parser.add_option("-d", "--date", dest="date", help="date of the nightly",
metavar="YYYY-MM-DD", default=str(datetime.date.today()))
parser.add_option("-a", "--addons", dest="addons",
help="an addon to install; repeat for multiple addons",
metavar="PATH1", default=[], action="append")
parser.add_option("-p", "--profile", dest="profile",
help="path to profile to user", metavar="PATH")
parser.add_option("-n", "--app", dest="app", help="application name",
type="choice",
metavar="[%s]" % "|".join(NightlyRunner.apps.keys()),
choices=NightlyRunner.apps.keys(),
default="firefox")
parser.add_option("--inbound-branch", dest="inbound_branch",
help="inbound branch name on ftp.mozilla.org",
metavar="[tracemonkey|mozilla-1.9.2]", default=None)
parser.add_option("--bits", dest="bits",
help="force 32 or 64 bit version (only applies to"
" x86_64 boxes)",
choices=("32", "64"), default=mozinfo.bits)
parser.add_option("--persist", dest="persist",
help="the directory in which files are to persist ie."
" /Users/someuser/Documents")
options, args = parser.parse_args(args)
options.bits = parse_bits(options.bits)
# run nightly
runner = NightlyRunner(appname=options.app, addons=options.addons,
profile=options.profile,
inbound_branch=options.inbound_branch,
bits=options.bits, persist=options.persist)
runner.start(get_date(options.date))
try:
runner.wait()
except KeyboardInterrupt:
runner.stop()
def test_valid_date(self):
date = utils.get_date("2014-07-05")
self.assertEquals(date, datetime.date(2014, 7, 5))
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--num-runs",
action="store",
type="int",
dest="num_runs",
default=1,
help="number of runs (default: 1)")
parser.add_option("--output-dir",
action="store",
type="string",
dest="outputdir",
help="output results to json file")
parser.add_option("--no-capture",
action="store_true",
dest="no_capture",
help="run through the test, but don't actually "
"capture anything")
parser.add_option(
"--profile-file",
action="store",
type="string",
dest="profile_file",
help="Collect a performance profile using the built in profiler.")
parser.add_option("--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="measure startup times instead of normal metrics")
parser.add_option("--url-params",
action="store",
dest="url_params",
default="",
help="additional url parameters for test")
parser.add_option(
"--use-apks",
action="store_true",
dest="use_apks",
help=
"use and install android APKs as part of test (instead of specifying appnames)"
)
parser.add_option("--date",
action="store",
dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date",
action="store",
dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date",
action="store",
dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error(
"Must specify at least one argument: the path to the test")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
test = args[0]
start_date = get_date(options.start_date)
end_date = get_date(options.end_date)
days = (end_date - start_date).days
for numdays in range(days + 1):
dates.append(start_date + datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
test = args[0]
dates = [get_date(options.date)]
elif not options.date and len(args) >= 2:
test = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif not options.date or (not options.start_date and not options.end_date):
parser.error(
"Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)"
)
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if options.outputdir:
outputfile = os.path.join(options.outputdir,
"metric-test-%s.json" % time.time())
else:
outputfile = None
if appnames:
for appname in appnames:
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
appname=appname,
profile_file=options.profile_file,
**devicePrefs)
elif apks:
for apk in apks:
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
apk=apk,
profile_file=options.profile_file,
**devicePrefs)
else:
for date in dates:
apk = get_build_for_date(date)
run_test(device,
options.outputdir,
outputfile,
test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats,
apk=apk,
appdate=date,
profile_file=options.profile_file,
**devicePrefs)
def cli():
default_bad_date = str(datetime.date.today())
default_good_date = "2009-01-01"
parser = OptionParser()
parser.add_option("-b",
"--bad",
dest="bad_date",
help="first known bad nightly build, default is today",
metavar="YYYY-MM-DD",
default=None)
parser.add_option("-g",
"--good",
dest="good_date",
help="last known good nightly build",
metavar="YYYY-MM-DD",
default=None)
parser.add_option("--bad-release",
dest="bad_release",
type=int,
help="first known bad nightly build. This option is "
"incompatible with --bad.")
parser.add_option("--good-release",
dest="good_release",
type=int,
help="last known good nightly build. This option is "
"incompatible with --good.")
parser.add_option("-e",
"--addon",
dest="addons",
help="an addon to install; repeat for multiple addons",
metavar="PATH1",
default=[],
action="append")
parser.add_option("-p",
"--profile",
dest="profile",
help="profile to use with nightlies",
metavar="PATH")
parser.add_option(
"-a",
"--arg",
dest="cmdargs",
help="a command-line argument to pass to the application;"
" repeat for multiple arguments",
metavar="ARG1",
default=[],
action="append")
parser.add_option("-n",
"--app",
dest="app",
help="application name (firefox, fennec,"
" thunderbird or b2g)",
metavar="[firefox|fennec|thunderbird|b2g]",
default="firefox")
parser.add_option("--inbound-branch",
dest="inbound_branch",
help="inbound branch name on ftp.mozilla.org",
metavar="[tracemonkey|mozilla-1.9.2]",
default=None)
parser.add_option("--bits",
dest="bits",
help="force 32 or 64 bit version (only applies to"
" x86_64 boxes)",
choices=("32", "64"),
default=mozinfo.bits)
parser.add_option("--persist",
dest="persist",
help="the directory in which files are to persist ie."
" /Users/someuser/Documents")
parser.add_option("--inbound",
action="store_true",
dest="inbound",
help="use inbound instead of nightlies (use --good-rev"
" and --bad-rev options")
parser.add_option("--bad-rev",
dest="first_bad_revision",
help="first known bad revision (use with --inbound)")
parser.add_option("--good-rev",
dest="last_good_revision",
help="last known good revision (use with --inbound)")
parser.add_option("--version",
dest="version",
action="store_true",
help="print the mozregression version number and exits")
(options, args) = parser.parse_args()
if options.version:
print __version__
sys.exit(0)
options.bits = parse_bits(options.bits)
inbound_runner = None
if options.app in ("firefox", "fennec", "b2g"):
inbound_runner = InboundRunner(appname=options.app,
addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
if options.inbound:
if not options.last_good_revision or not options.first_bad_revision:
print "If bisecting inbound, both --good-rev and --bad-rev " \
" must be set"
sys.exit(1)
bisector = Bisector(None,
inbound_runner,
appname=options.app,
last_good_revision=options.last_good_revision,
first_bad_revision=options.first_bad_revision)
app = bisector.bisect_inbound
else:
if not options.bad_release and not options.bad_date:
options.bad_date = default_bad_date
print "No 'bad' date specified, using " + options.bad_date
elif options.bad_release and options.bad_date:
raise Exception("Options '--bad_release' and '--bad_date' "
"are incompatible.")
elif options.bad_release:
options.bad_date = date_of_release(options.bad_release)
if options.bad_date is None:
raise Exception("Unable to find a matching date for release " +
str(options.bad_release))
print "Using 'bad' date " + options.bad_date + " for release " + \
str(options.bad_release)
if not options.good_release and not options.good_date:
options.good_date = default_good_date
print "No 'good' date specified, using " + options.good_date
elif options.good_release and options.good_date:
raise Exception("Options '--good_release' and '--good_date' "
"are incompatible.")
elif options.good_release:
options.good_date = date_of_release(options.good_release)
if options.good_date is None:
raise Exception("Unable to find a matching date for release " +
str(options.good_release))
print "Using 'good' date " + options.good_date + " for release " + \
str(options.good_release)
nightly_runner = NightlyRunner(appname=options.app,
addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
bisector = Bisector(nightly_runner,
inbound_runner,
appname=options.app)
app = lambda: bisector.bisect_nightlies(get_date(options.good_date),
get_date(options.bad_date))
try:
app()
except KeyboardInterrupt:
sys.exit("\nInterrupted.")
except errors.MozRegressionError as exc:
sys.exit(str(exc))
def cli():
parser = OptionParser()
parser.add_option("-b", "--bad", dest="bad_date",
help="first known bad nightly build, default is today",
metavar="YYYY-MM-DD", default=str(datetime.date.today()))
parser.add_option("-g", "--good", dest="good_date",
help="last known good nightly build",
metavar="YYYY-MM-DD", default=None)
parser.add_option("-e", "--addon", dest="addons",
help="an addon to install; repeat for multiple addons",
metavar="PATH1", default=[], action="append")
parser.add_option("-p", "--profile", dest="profile",
help="profile to use with nightlies", metavar="PATH")
parser.add_option("-a", "--arg", dest="cmdargs",
help="a command-line argument to pass to the application;"
" repeat for multiple arguments",
metavar="ARG1", default=[], action="append")
parser.add_option("-n", "--app", dest="app",
help="application name (firefox, fennec,"
" thunderbird or b2g)",
metavar="[firefox|fennec|thunderbird|b2g]",
default="firefox")
parser.add_option("--inbound-branch", dest="inbound_branch",
help="inbound branch name on ftp.mozilla.org",
metavar="[tracemonkey|mozilla-1.9.2]", default=None)
parser.add_option("--bits", dest="bits",
help="force 32 or 64 bit version (only applies to"
" x86_64 boxes)",
choices=("32", "64"), default=mozinfo.bits)
parser.add_option("--persist", dest="persist",
help="the directory in which files are to persist ie."
" /Users/someuser/Documents")
parser.add_option("--inbound", action="store_true", dest="inbound",
help="use inbound instead of nightlies (use --good-rev"
" and --bad-rev options")
parser.add_option("--bad-rev", dest="first_bad_revision",
help="first known bad revision (use with --inbound)")
parser.add_option("--good-rev", dest="last_good_revision",
help="last known good revision (use with --inbound)")
parser.add_option("--version", dest="version", action="store_true",
help="print the mozregression version number and exits")
(options, args) = parser.parse_args()
if options.version:
print __version__
sys.exit(0)
options.bits = parse_bits(options.bits)
inbound_runner = None
if options.app in ("firefox", "fennec", "b2g"):
inbound_runner = InboundRunner(appname=options.app,
addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
if options.inbound:
if not options.last_good_revision or not options.first_bad_revision:
print "If bisecting inbound, both --good-rev and --bad-rev " \
" must be set"
sys.exit(1)
bisector = Bisector(None, inbound_runner, appname=options.app,
last_good_revision=options.last_good_revision,
first_bad_revision=options.first_bad_revision)
app = bisector.bisect_inbound
else:
if not options.good_date:
options.good_date = "2009-01-01"
print "No 'good' date specified, using " + options.good_date
nightly_runner = NightlyRunner(appname=options.app, addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
bisector = Bisector(nightly_runner, inbound_runner,
appname=options.app)
app = lambda: bisector.bisect_nightlies(get_date(options.good_date),
get_date(options.bad_date))
try:
app()
except KeyboardInterrupt:
sys.exit("\nInterrupted.")
except errors.MozRegressionError as exc:
sys.exit(str(exc))
def main(args=sys.argv[1:]):
usage = "usage: %prog <test> [appname1] [appname2] ..."
parser = eideticker.OptionParser(usage=usage)
parser.add_option("--num-runs", action="store",
type = "int", dest = "num_runs",
default=1,
help = "number of runs (default: 1)")
parser.add_option("--output-dir", action="store",
type="string", dest="outputdir",
help="output results to json file")
parser.add_option("--no-capture", action="store_true",
dest = "no_capture",
help = "run through the test, but don't actually "
"capture anything")
parser.add_option("--profile-file", action="store",
type="string", dest = "profile_file",
help="Collect a performance profile using the built in profiler.")
parser.add_option("--get-internal-checkerboard-stats",
action="store_true",
dest="get_internal_checkerboard_stats",
help="get and calculate internal checkerboard stats")
parser.add_option("--startup-test",
action="store_true",
dest="startup_test",
help="measure startup times instead of normal metrics")
parser.add_option("--url-params", action="store",
dest="url_params", default="",
help="additional url parameters for test")
parser.add_option("--use-apks", action="store_true", dest="use_apks",
help="use and install android APKs as part of test (instead of specifying appnames)")
parser.add_option("--date", action="store", dest="date",
metavar="YYYY-MM-DD",
help="get and test nightly build for date")
parser.add_option("--start-date", action="store", dest="start_date",
metavar="YYYY-MM-DD",
help="start date for range of nightlies to test")
parser.add_option("--end-date", action="store", dest="end_date",
metavar="YYYY-MM-DD",
help="end date for range of nightlies to test")
options, args = parser.parse_args()
if len(args) == 0:
parser.error("Must specify at least one argument: the path to the test")
dates = []
appnames = []
apks = []
if options.start_date and options.end_date and len(args) == 1:
test = args[0]
start_date = get_date(options.start_date)
end_date = get_date(options.end_date)
days=(end_date-start_date).days
for numdays in range(days+1):
dates.append(start_date+datetime.timedelta(days=numdays))
elif options.date and len(args) == 1:
test = args[0]
dates = [get_date(options.date)]
elif not options.date and len(args) >= 2:
test = args[0]
if options.use_apks:
apks = args[1:]
else:
appnames = args[1:]
elif not options.date or (not options.start_date and not options.end_date):
parser.error("Must specify date, date range, a set of appnames (e.g. org.mozilla.fennec) or a set of apks (if --use-apks is specified)")
devicePrefs = eideticker.getDevicePrefs(options)
device = eideticker.getDevice(**devicePrefs)
if options.outputdir:
outputfile = os.path.join(options.outputdir, "metric-test-%s.json" % time.time())
else:
outputfile = None
if appnames:
for appname in appnames:
run_test(device, options.outputdir, outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, appname=appname,
profile_file=options.profile_file, **devicePrefs)
elif apks:
for apk in apks:
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
profile_file=options.profile_file, **devicePrefs)
else:
for date in dates:
apk = get_build_for_date(date)
run_test(device, options.outputdir,
outputfile, test,
options.url_params,
options.num_runs,
options.startup_test,
options.no_capture,
options.get_internal_checkerboard_stats, apk=apk,
appdate=date,
profile_file=options.profile_file, **devicePrefs)
def cli(args=sys.argv[1:]):
"""moznightly command line entry point"""
# parse command line options
parser = OptionParser()
parser.add_option("-d",
"--date",
dest="date",
help="date of the nightly",
metavar="YYYY-MM-DD",
default=str(datetime.date.today()))
parser.add_option("-a",
"--addons",
dest="addons",
help="an addon to install; repeat for multiple addons",
metavar="PATH1",
default=[],
action="append")
parser.add_option("-p",
"--profile",
dest="profile",
help="path to profile to user",
metavar="PATH")
parser.add_option("-n",
"--app",
dest="app",
help="application name",
type="choice",
metavar="[%s]" % "|".join(NightlyRunner.apps.keys()),
choices=NightlyRunner.apps.keys(),
default="firefox")
parser.add_option("--inbound-branch",
dest="inbound_branch",
help="inbound branch name on ftp.mozilla.org",
metavar="[tracemonkey|mozilla-1.9.2]",
default=None)
parser.add_option("--bits",
dest="bits",
help="force 32 or 64 bit version (only applies to"
" x86_64 boxes)",
choices=("32", "64"),
default=mozinfo.bits)
parser.add_option("--persist",
dest="persist",
help="the directory in which files are to persist ie."
" /Users/someuser/Documents")
options, args = parser.parse_args(args)
options.bits = parse_bits(options.bits)
# run nightly
runner = NightlyRunner(appname=options.app,
addons=options.addons,
profile=options.profile,
inbound_branch=options.inbound_branch,
bits=options.bits,
persist=options.persist)
runner.start(get_date(options.date))
try:
runner.wait()
except KeyboardInterrupt:
runner.stop()
def test_valid_date(self):
date = utils.get_date("2014-07-05")
self.assertEquals(date, datetime.date(2014, 7, 5))
def cli():
default_bad_date = str(datetime.date.today())
default_good_date = "2009-01-01"
options = parse_args()
inbound_runner = None
if options.app in ("firefox", "fennec", "b2g") and not (mozinfo.os == 'win' and options.bits == 64):
inbound_runner = InboundRunner(appname=options.app,
addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
if options.inbound:
if not options.last_good_revision or not options.first_bad_revision:
sys.exit("If bisecting inbound, both --good-rev and --bad-rev"
" must be set")
bisector = Bisector(None, inbound_runner, appname=options.app,
last_good_revision=options.last_good_revision,
first_bad_revision=options.first_bad_revision)
app = bisector.bisect_inbound
else:
if not options.bad_release and not options.bad_date:
options.bad_date = default_bad_date
print "No 'bad' date specified, using " + options.bad_date
elif options.bad_release and options.bad_date:
sys.exit("Options '--bad_release' and '--bad_date' are"
" incompatible.")
elif options.bad_release:
options.bad_date = date_of_release(options.bad_release)
if options.bad_date is None:
sys.exit("Unable to find a matching date for release "
+ str(options.bad_release))
print "Using 'bad' date " + options.bad_date + " for release " + \
str(options.bad_release)
if not options.good_release and not options.good_date:
options.good_date = default_good_date
print "No 'good' date specified, using " + options.good_date
elif options.good_release and options.good_date:
sys.exit("Options '--good_release' and '--good_date'"
" are incompatible.")
elif options.good_release:
options.good_date = date_of_release(options.good_release)
if options.good_date is None:
sys.exit("Unable to find a matching date for release "
+ str(options.good_release))
print "Using 'good' date " + options.good_date + " for release " + \
str(options.good_release)
nightly_runner = NightlyRunner(appname=options.app, addons=options.addons,
inbound_branch=options.inbound_branch,
profile=options.profile,
cmdargs=options.cmdargs,
bits=options.bits,
persist=options.persist)
bisector = Bisector(nightly_runner, inbound_runner,
appname=options.app)
app = lambda: bisector.bisect_nightlies(get_date(options.good_date),
get_date(options.bad_date))
try:
app()
except KeyboardInterrupt:
sys.exit("\nInterrupted.")
except errors.MozRegressionError as exc:
sys.exit(str(exc))