def upload_repacks(self):
c = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
base_package_name = self.query_base_package_name()
version = self.query_version()
upload_env = self.query_upload_env()
success_count = total_count = 0
buildnum = None
if c.get('release_config_file'):
rc = self.query_release_config()
buildnum = rc['buildnum']
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if c.get('base_post_upload_cmd'):
upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % \
{'version': version, 'locale': locale, 'buildnum': str(buildnum),
'post_upload_extra': ' '.join(c.get('post_upload_extra', []))}
output = self.get_output_from_command_m(
# Ugly hack to avoid |make upload| stderr from showing up
# as get_output_from_command errors
"%s upload AB_CD=%s 2>&1" % (make, locale),
cwd=dirs['abs_locales_dir'],
env=upload_env,
silent=True
)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
self.add_failure(locale, message="%s failed in make upload!" % (locale))
continue
package_name = base_package_name % {'locale': locale}
r = re.compile("(http.*%s)" % package_name)
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
# XXX Move the files to a SIMPLE_NAME format until we can enable
# Simple names in the build system
if self.config.get("simple_name_move"):
# Assume an UPLOAD PATH
upload_target = self.config["upload_env"]["UPLOAD_PATH"]
target_path = os.path.join(upload_target, locale)
self.mkdir_p(target_path)
glob_name = "*.%s.android-arm.*" % locale
for f in glob.glob(os.path.join(upload_target, glob_name)):
glob_extension = f[f.rfind('.'):]
self.move(os.path.join(f),
os.path.join(target_path, "target%s" % glob_extension))
self.log("Converted uploads for %s to simple names" % locale)
success_count += 1
self.summarize_success_count(success_count, total_count,
message="Make Upload for %d of %d locales successful.")
def upload_repacks(self):
c = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
base_package_name = self.query_base_package_name()
version = self.query_version()
upload_env = self.query_upload_env()
success_count = total_count = 0
buildnum = None
if c.get('release_config_file'):
rc = self.query_release_config()
buildnum = rc['buildnum']
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if c.get('base_post_upload_cmd'):
upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))}
output = self.get_output_from_command_m(
# Ugly hack to avoid |make upload| stderr from showing up
# as get_output_from_command errors
"%s upload AB_CD=%s 2>&1" % (make, locale),
cwd=dirs['abs_locales_dir'],
env=upload_env,
silent=True
)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
self.add_failure(locale, message="%s failed in make upload!" % (locale))
continue
package_name = base_package_name % {'locale': locale}
r = re.compile("(http.*%s)" % package_name)
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
# XXX Move the files to a SIMPLE_NAME format until we can enable
# Simple names in the build system
if self.config.get("simple_name_move"):
# Assume an UPLOAD PATH
upload_target = self.config["upload_env"]["UPLOAD_PATH"]
target_path = os.path.join(upload_target, locale)
self.mkdir_p(target_path)
glob_name = "*.%s.android-arm.*" % locale
for f in glob.glob(os.path.join(upload_target, glob_name)):
glob_extension = f[f.rfind('.'):]
self.move(os.path.join(f),
os.path.join(target_path, "target%s" % glob_extension))
self.log("Converted uploads for %s to simple names" % locale)
success_count += 1
self.summarize_success_count(success_count, total_count,
message="Make Upload for %d of %d locales successful.")
def upload_repacks(self):
c = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
base_package_name = self.query_base_package_name()
version = self.query_version()
upload_env = self.query_upload_env()
success_count = total_count = 0
buildnum = None
if c.get('release_config_file'):
rc = self.query_release_config()
buildnum = rc['buildnum']
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if c.get('base_post_upload_cmd'):
upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {
'version': version,
'locale': locale,
'buildnum': str(buildnum),
'post_upload_extra': ' '.join(
c.get('post_upload_extra', []))
}
output = self.get_output_from_command_m(
# Ugly hack to avoid |make upload| stderr from showing up
# as get_output_from_command errors
"%s upload AB_CD=%s 2>&1" % (make, locale),
cwd=dirs['abs_locales_dir'],
env=upload_env,
silent=True)
parser = OutputParser(config=self.config,
log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
self.add_failure(locale,
message="%s failed in make upload!" %
(locale))
continue
package_name = base_package_name % {'locale': locale}
r = re.compile("(http.*%s)" % package_name)
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
success_count += 1
self.summarize_success_count(
success_count,
total_count,
message="Make Upload for %d of %d locales successful.")
def upload_repacks(self):
c = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
base_package_name = self.query_base_package_name()
version = self.query_version()
upload_env = self.query_upload_env()
success_count = total_count = 0
buildnum = None
if c.get("release_config_file"):
rc = self.query_release_config()
buildnum = rc["buildnum"]
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if c.get("base_post_upload_cmd"):
upload_env["POST_UPLOAD_CMD"] = c["base_post_upload_cmd"] % {
"version": version,
"locale": locale,
"buildnum": str(buildnum),
}
output = self.get_output_from_command_m(
# Ugly hack to avoid |make upload| stderr from showing up
# as get_output_from_command errors
"%s upload AB_CD=%s 2>&1" % (make, locale),
cwd=dirs["abs_locales_dir"],
env=upload_env,
silent=True,
)
parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
self.add_failure(locale, message="%s failed in make upload!" % (locale))
continue
package_name = base_package_name % {"locale": locale}
r = re.compile("(http.*%s)" % package_name)
success = False
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
success = True
if not success:
self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale))
print output
continue
success_count += 1
self.summarize_success_count(success_count, total_count, message="Uploaded %d of %d binaries successfully.")
def _query_make_variable(self, variable, make_args=None):
make = self.query_exe("make")
env = self.query_repack_env()
dirs = self.query_abs_dirs()
if make_args is None:
make_args = []
# TODO error checking
output = self.get_output_from_command_m(
[make, "echo-variable-%s" % variable] + make_args, cwd=dirs["abs_locales_dir"], silent=True, env=env
)
parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList)
parser.add_lines(output)
return output.strip()
def _query_make_ident_output(self):
"""Get |make ident| output from the objdir.
Only valid after setup is run.
"""
if self.make_ident_output:
return self.make_ident_output
env = self.query_repack_env()
dirs = self.query_abs_dirs()
output = self.get_output_from_command_m(
["make", "ident"], cwd=dirs["abs_locales_dir"], env=env, silent=True, halt_on_failure=True
)
parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList)
parser.add_lines(output)
self.make_ident_output = output
return output
def _query_make_variable(self, variable, make_args=None):
make = self.query_exe('make')
env = self.query_repack_env()
dirs = self.query_abs_dirs()
if make_args is None:
make_args = []
# TODO error checking
output = self.get_output_from_command_m(
[make, "echo-variable-%s" % variable] + make_args,
cwd=dirs['abs_locales_dir'], silent=True,
env=env
)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
return output.strip()
def upload_repacks(self):
"""calls make upload <locale>"""
config = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
version = self.query_version()
upload_env = self.query_repack_env()
success_count = 0
total_count = 0
cwd = dirs['abs_locales_dir']
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if config.get('base_post_upload_cmd'):
upload_cmd = {'version': version, 'locale': locale}
upload_cmd = config['base_post_upload_cmd'] % upload_cmd
upload_env['POST_UPLOAD_CMD'] = upload_cmd
target = ["upload", "AB_CD=%s" % locale]
output = self._get_output_from_make(target, cwd=cwd, env=upload_env)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
msg = "%s failed in make upload!" % (locale)
self.add_failure(locale, message=msg)
continue
package_name = self.query_base_package_name(locale)
r = re.compile("(http.*%s)" % package_name)
success = False
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
success = True
if not success:
msg = "Failed to detect %s url in make upload!" % locale
self.add_failure(locale, message=msg)
self.debug(output)
continue
success_count += 1
msg = "Uploaded %d of %d binaries successfully." % (success_count,
total_count)
self.summarize_success_count(success_count, total_count, message=msg)
def sign_apk(self, apk, keystore, storepass, keypass, key_alias,
remove_signature=True, error_list=None,
log_level=INFO, error_level=ERROR):
"""
Signs an apk with jarsigner.
"""
jarsigner = self.query_exe('jarsigner')
if remove_signature:
status = self.unsign_apk(apk)
if status:
self.error("Can't remove signature in %s!" % apk)
return -1
if error_list is None:
error_list = JarsignerErrorList[:]
# This needs to run silently, so no run_command() or
# get_output_from_command() (though I could add a
# suppress_command_echo=True or something?)
self.log("(signing %s)" % apk, level=log_level)
try:
p = subprocess.Popen([jarsigner, "-keystore", keystore,
"-storepass", storepass,
"-keypass", keypass,
apk, key_alias],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
bufsize=0)
except OSError:
self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner))
return -2
except ValueError:
self.exception("Popen called with invalid arguments during signing?")
return -3
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=error_list)
loop = True
while loop:
if p.poll() is not None:
"""Avoid losing the final lines of the log?"""
loop = False
for line in p.stdout:
parser.add_lines(line)
if parser.num_errors:
self.log("(failure)", level=error_level)
else:
self.log("(success)", level=log_level)
return parser.num_errors
def sign_apk(self, apk, keystore, storepass, keypass, key_alias,
remove_signature=True, error_list=None,
log_level=INFO, error_level=ERROR):
"""
Signs an apk with jarsigner.
"""
jarsigner = self.query_exe('jarsigner')
if remove_signature:
status = self.unsign_apk(apk)
if status:
self.error("Can't remove signature in %s!" % apk)
return -1
if error_list is None:
error_list = JarsignerErrorList[:]
# This needs to run silently, so no run_command() or
# get_output_from_command() (though I could add a
# suppress_command_echo=True or something?)
self.log("(signing %s)" % apk, level=log_level)
try:
p = subprocess.Popen([jarsigner, "-keystore", keystore,
"-storepass", storepass,
"-keypass", keypass,
apk, key_alias],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
except OSError:
self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner))
return -2
except ValueError:
self.exception("Popen called with invalid arguments during signing?")
return -3
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=error_list)
loop = True
while loop:
if p.poll() is not None:
"""Avoid losing the final lines of the log?"""
loop = False
for line in p.stdout:
parser.add_lines(line)
if parser.num_errors:
self.log("(failure)", level=error_level)
else:
self.log("(success)", level=log_level)
return parser.num_errors
def _query_make_ident_output(self):
"""Get |make ident| output from the objdir.
Only valid after setup is run.
"""
if self.make_ident_output:
return self.make_ident_output
env = self.query_repack_env()
dirs = self.query_abs_dirs()
output = self.get_output_from_command_m(["make", "ident"],
cwd=dirs['abs_locales_dir'],
env=env,
silent=True,
halt_on_failure=True)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
self.make_ident_output = output
return output
def upload_repacks(self):
c = self.config
dirs = self.query_abs_dirs()
locales = self.query_locales()
make = self.query_exe("make")
base_package_name = self.query_base_package_name()
version = self.query_version()
upload_env = self.query_upload_env()
success_count = total_count = 0
buildnum = None
if c.get('release_config_file'):
rc = self.query_release_config()
buildnum = rc['buildnum']
for locale in locales:
if self.query_failure(locale):
self.warning("Skipping previously failed locale %s." % locale)
continue
total_count += 1
if c.get('base_post_upload_cmd'):
upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))}
output = self.get_output_from_command_m(
# Ugly hack to avoid |make upload| stderr from showing up
# as get_output_from_command errors
"%s upload AB_CD=%s 2>&1" % (make, locale),
cwd=dirs['abs_locales_dir'],
env=upload_env,
silent=True
)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=MakefileErrorList)
parser.add_lines(output)
if parser.num_errors:
self.add_failure(locale, message="%s failed in make upload!" % (locale))
continue
package_name = base_package_name % {'locale': locale}
r = re.compile("(http.*%s)" % package_name)
for line in output.splitlines():
m = r.match(line)
if m:
self.upload_urls[locale] = m.groups()[0]
self.info("Found upload url %s" % self.upload_urls[locale])
success_count += 1
self.summarize_success_count(success_count, total_count,
message="Make Upload for %d of %d locales successful.")
def run_command(self, command, cwd=None, error_list=None, parse_at_end=False,
halt_on_failure=False, success_codes=None,
env=None, return_type='status', throw_exception=False):
"""Run a command, with logging and error parsing.
TODO: parse_at_end, context_lines
TODO: retry_interval?
TODO: error_level_override?
TODO: Add a copy-pastable version of |command| if it's a list.
TODO: print env if set
error_list example:
[{'regex': re.compile('^Error: LOL J/K'), level=IGNORE},
{'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'},
{'substr': 'THE WORLD IS ENDING', level=FATAL, contextLines='20:'}
]
"""
if error_list is None:
error_list = []
if success_codes is None:
success_codes = [0]
if cwd:
if not os.path.isdir(cwd):
level = ERROR
if halt_on_failure:
level = FATAL
self.log("Can't run command %s in non-existent directory %s!" % \
(command, cwd), level=level)
return -1
self.info("Running command: %s in %s" % (command, cwd))
else:
self.info("Running command: %s" % command)
if self.config.get('noop'):
self.info("(Dry run; skipping)")
return
shell = True
if isinstance(command, list):
shell = False
p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
cwd=cwd, stderr=subprocess.STDOUT, env=env)
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=error_list)
loop = True
while loop:
if p.poll() is not None:
"""Avoid losing the final lines of the log?"""
loop = False
for line in p.stdout:
parser.add_lines(line)
return_level = INFO
if p.returncode not in success_codes:
return_level = ERROR
if throw_exception:
raise subprocess.CalledProcessError(p.returncode, command)
self.log("Return code: %d" % p.returncode, level=return_level)
if halt_on_failure:
if parser.num_errors or p.returncode not in success_codes:
self.fatal("Halting on failure while running %s" % command,
exit_code=p.returncode)
if return_type == 'num_errors':
return parser.num_errors
return p.returncode
self.info("Copy/paste: %s" % subprocess.list2cmdline(command))
shell = True
if isinstance(command, list):
shell = False
try:
p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE,
cwd=cwd, stderr=subprocess.STDOUT, env=env)
except OSError, e:
level = ERROR
if halt_on_failure:
level = FATAL
self.log('caught OS error %s: %s while running %s' % (e.errno,
e.strerror, command), level=level)
return -1
if output_parser is None:
parser = OutputParser(config=self.config, log_obj=self.log_obj,
error_list=error_list)
else:
parser = output_parser
loop = True
while loop:
if p.poll() is not None:
"""Avoid losing the final lines of the log?"""
loop = False
while True:
line = p.stdout.readline()
if not line:
break
parser.add_lines(line)
return_level = INFO
if p.returncode not in success_codes:
return_level = ERROR
