def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get('release_config_file'): rc = self.query_release_config() buildnum = rc['buildnum'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get('base_post_upload_cmd'): upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % \ {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))} output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs['abs_locales_dir'], env=upload_env, silent=True ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {'locale': locale} r = re.compile("(http.*%s)" % package_name) for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.android-arm.*" % locale for f in glob.glob(os.path.join(upload_target, glob_name)): glob_extension = f[f.rfind('.'):] self.move(os.path.join(f), os.path.join(target_path, "target%s" % glob_extension)) self.log("Converted uploads for %s to simple names" % locale) success_count += 1 self.summarize_success_count(success_count, total_count, message="Make Upload for %d of %d locales successful.")
def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get('release_config_file'): rc = self.query_release_config() buildnum = rc['buildnum'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get('base_post_upload_cmd'): upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))} output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs['abs_locales_dir'], env=upload_env, silent=True ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {'locale': locale} r = re.compile("(http.*%s)" % package_name) for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.android-arm.*" % locale for f in glob.glob(os.path.join(upload_target, glob_name)): glob_extension = f[f.rfind('.'):] self.move(os.path.join(f), os.path.join(target_path, "target%s" % glob_extension)) self.log("Converted uploads for %s to simple names" % locale) success_count += 1 self.summarize_success_count(success_count, total_count, message="Make Upload for %d of %d locales successful.")
def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get('release_config_file'): rc = self.query_release_config() buildnum = rc['buildnum'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get('base_post_upload_cmd'): upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % { 'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join( c.get('post_upload_extra', [])) } output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs['abs_locales_dir'], env=upload_env, silent=True) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {'locale': locale} r = re.compile("(http.*%s)" % package_name) for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) success_count += 1 self.summarize_success_count( success_count, total_count, message="Make Upload for %d of %d locales successful.")
def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get("release_config_file"): rc = self.query_release_config() buildnum = rc["buildnum"] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get("base_post_upload_cmd"): upload_env["POST_UPLOAD_CMD"] = c["base_post_upload_cmd"] % { "version": version, "locale": locale, "buildnum": str(buildnum), } output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs["abs_locales_dir"], env=upload_env, silent=True, ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {"locale": locale} r = re.compile("(http.*%s)" % package_name) success = False for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) success = True if not success: self.add_failure(locale, message="Failed to detect %s url in make upload!" % (locale)) print output continue success_count += 1 self.summarize_success_count(success_count, total_count, message="Uploaded %d of %d binaries successfully.")
def _query_make_variable(self, variable, make_args=None): make = self.query_exe("make") env = self.query_repack_env() dirs = self.query_abs_dirs() if make_args is None: make_args = [] # TODO error checking output = self.get_output_from_command_m( [make, "echo-variable-%s" % variable] + make_args, cwd=dirs["abs_locales_dir"], silent=True, env=env ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) return output.strip()
def _query_make_ident_output(self): """Get |make ident| output from the objdir. Only valid after setup is run. """ if self.make_ident_output: return self.make_ident_output env = self.query_repack_env() dirs = self.query_abs_dirs() output = self.get_output_from_command_m( ["make", "ident"], cwd=dirs["abs_locales_dir"], env=env, silent=True, halt_on_failure=True ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) self.make_ident_output = output return output
def _query_make_variable(self, variable, make_args=None): make = self.query_exe('make') env = self.query_repack_env() dirs = self.query_abs_dirs() if make_args is None: make_args = [] # TODO error checking output = self.get_output_from_command_m( [make, "echo-variable-%s" % variable] + make_args, cwd=dirs['abs_locales_dir'], silent=True, env=env ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) return output.strip()
def upload_repacks(self): """calls make upload <locale>""" config = self.config dirs = self.query_abs_dirs() locales = self.query_locales() version = self.query_version() upload_env = self.query_repack_env() success_count = 0 total_count = 0 cwd = dirs['abs_locales_dir'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if config.get('base_post_upload_cmd'): upload_cmd = {'version': version, 'locale': locale} upload_cmd = config['base_post_upload_cmd'] % upload_cmd upload_env['POST_UPLOAD_CMD'] = upload_cmd target = ["upload", "AB_CD=%s" % locale] output = self._get_output_from_make(target, cwd=cwd, env=upload_env) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: msg = "%s failed in make upload!" % (locale) self.add_failure(locale, message=msg) continue package_name = self.query_base_package_name(locale) r = re.compile("(http.*%s)" % package_name) success = False for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) success = True if not success: msg = "Failed to detect %s url in make upload!" % locale self.add_failure(locale, message=msg) self.debug(output) continue success_count += 1 msg = "Uploaded %d of %d binaries successfully." % (success_count, total_count) self.summarize_success_count(success_count, total_count, message=msg)
def sign_apk(self, apk, keystore, storepass, keypass, key_alias, remove_signature=True, error_list=None, log_level=INFO, error_level=ERROR): """ Signs an apk with jarsigner. """ jarsigner = self.query_exe('jarsigner') if remove_signature: status = self.unsign_apk(apk) if status: self.error("Can't remove signature in %s!" % apk) return -1 if error_list is None: error_list = JarsignerErrorList[:] # This needs to run silently, so no run_command() or # get_output_from_command() (though I could add a # suppress_command_echo=True or something?) self.log("(signing %s)" % apk, level=log_level) try: p = subprocess.Popen([jarsigner, "-keystore", keystore, "-storepass", storepass, "-keypass", keypass, apk, key_alias], stdout=subprocess.PIPE, stderr=subprocess.STDOUT, bufsize=0) except OSError: self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner)) return -2 except ValueError: self.exception("Popen called with invalid arguments during signing?") return -3 parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) loop = True while loop: if p.poll() is not None: """Avoid losing the final lines of the log?""" loop = False for line in p.stdout: parser.add_lines(line) if parser.num_errors: self.log("(failure)", level=error_level) else: self.log("(success)", level=log_level) return parser.num_errors
def sign_apk(self, apk, keystore, storepass, keypass, key_alias, remove_signature=True, error_list=None, log_level=INFO, error_level=ERROR): """ Signs an apk with jarsigner. """ jarsigner = self.query_exe('jarsigner') if remove_signature: status = self.unsign_apk(apk) if status: self.error("Can't remove signature in %s!" % apk) return -1 if error_list is None: error_list = JarsignerErrorList[:] # This needs to run silently, so no run_command() or # get_output_from_command() (though I could add a # suppress_command_echo=True or something?) self.log("(signing %s)" % apk, level=log_level) try: p = subprocess.Popen([jarsigner, "-keystore", keystore, "-storepass", storepass, "-keypass", keypass, apk, key_alias], stdout=subprocess.PIPE, stderr=subprocess.STDOUT) except OSError: self.exception("Error while signing %s (missing %s?):" % (apk, jarsigner)) return -2 except ValueError: self.exception("Popen called with invalid arguments during signing?") return -3 parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) loop = True while loop: if p.poll() is not None: """Avoid losing the final lines of the log?""" loop = False for line in p.stdout: parser.add_lines(line) if parser.num_errors: self.log("(failure)", level=error_level) else: self.log("(success)", level=log_level) return parser.num_errors
def _query_make_ident_output(self): """Get |make ident| output from the objdir. Only valid after setup is run. """ if self.make_ident_output: return self.make_ident_output env = self.query_repack_env() dirs = self.query_abs_dirs() output = self.get_output_from_command_m(["make", "ident"], cwd=dirs['abs_locales_dir'], env=env, silent=True, halt_on_failure=True) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) self.make_ident_output = output return output
def upload_repacks(self): c = self.config dirs = self.query_abs_dirs() locales = self.query_locales() make = self.query_exe("make") base_package_name = self.query_base_package_name() version = self.query_version() upload_env = self.query_upload_env() success_count = total_count = 0 buildnum = None if c.get('release_config_file'): rc = self.query_release_config() buildnum = rc['buildnum'] for locale in locales: if self.query_failure(locale): self.warning("Skipping previously failed locale %s." % locale) continue total_count += 1 if c.get('base_post_upload_cmd'): upload_env['POST_UPLOAD_CMD'] = c['base_post_upload_cmd'] % {'version': version, 'locale': locale, 'buildnum': str(buildnum), 'post_upload_extra': ' '.join(c.get('post_upload_extra', []))} output = self.get_output_from_command_m( # Ugly hack to avoid |make upload| stderr from showing up # as get_output_from_command errors "%s upload AB_CD=%s 2>&1" % (make, locale), cwd=dirs['abs_locales_dir'], env=upload_env, silent=True ) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=MakefileErrorList) parser.add_lines(output) if parser.num_errors: self.add_failure(locale, message="%s failed in make upload!" % (locale)) continue package_name = base_package_name % {'locale': locale} r = re.compile("(http.*%s)" % package_name) for line in output.splitlines(): m = r.match(line) if m: self.upload_urls[locale] = m.groups()[0] self.info("Found upload url %s" % self.upload_urls[locale]) success_count += 1 self.summarize_success_count(success_count, total_count, message="Make Upload for %d of %d locales successful.")
def run_command(self, command, cwd=None, error_list=None, parse_at_end=False, halt_on_failure=False, success_codes=None, env=None, return_type='status', throw_exception=False): """Run a command, with logging and error parsing. TODO: parse_at_end, context_lines TODO: retry_interval? TODO: error_level_override? TODO: Add a copy-pastable version of |command| if it's a list. TODO: print env if set error_list example: [{'regex': re.compile('^Error: LOL J/K'), level=IGNORE}, {'regex': re.compile('^Error:'), level=ERROR, contextLines='5:5'}, {'substr': 'THE WORLD IS ENDING', level=FATAL, contextLines='20:'} ] """ if error_list is None: error_list = [] if success_codes is None: success_codes = [0] if cwd: if not os.path.isdir(cwd): level = ERROR if halt_on_failure: level = FATAL self.log("Can't run command %s in non-existent directory %s!" % \ (command, cwd), level=level) return -1 self.info("Running command: %s in %s" % (command, cwd)) else: self.info("Running command: %s" % command) if self.config.get('noop'): self.info("(Dry run; skipping)") return shell = True if isinstance(command, list): shell = False p = subprocess.Popen(command, shell=shell, stdout=subprocess.PIPE, cwd=cwd, stderr=subprocess.STDOUT, env=env) parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) loop = True while loop: if p.poll() is not None: """Avoid losing the final lines of the log?""" loop = False for line in p.stdout: parser.add_lines(line) return_level = INFO if p.returncode not in success_codes: return_level = ERROR if throw_exception: raise subprocess.CalledProcessError(p.returncode, command) self.log("Return code: %d" % p.returncode, level=return_level) if halt_on_failure: if parser.num_errors or p.returncode not in success_codes: self.fatal("Halting on failure while running %s" % command, exit_code=p.returncode) if return_type == 'num_errors': return parser.num_errors return p.returncode
return -1 if output_parser is None: parser = OutputParser(config=self.config, log_obj=self.log_obj, error_list=error_list) else: parser = output_parser loop = True while loop: if p.poll() is not None: """Avoid losing the final lines of the log?""" loop = False while True: line = p.stdout.readline() if not line: break parser.add_lines(line) return_level = INFO if p.returncode not in success_codes: return_level = ERROR if throw_exception: raise subprocess.CalledProcessError(p.returncode, command) self.log("Return code: %d" % p.returncode, level=return_level) if halt_on_failure: if parser.num_errors or p.returncode not in success_codes: self.fatal("Halting on failure while running %s" % command, exit_code=p.returncode) if return_type == 'num_errors': return parser.num_errors return p.returncode def get_output_from_command(self, command, cwd=None,