def make_upload(self, locale): """wrapper for make upload command""" config = self.config env = self.query_l10n_env() dirs = self.query_abs_dirs() buildid = self._query_buildid() try: env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % {'buildid': buildid, 'branch': config['branch']} except KeyError: # no base_post_upload_cmd in configuration, just skip it pass target = ['upload', 'AB_CD=%s' % (locale)] cwd = dirs['abs_locales_dir'] parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj) retval = self._make(target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser) if locale not in self.package_urls: self.package_urls[locale] = {} self.package_urls[locale].update(parser.matches) if 'partialMarUrl' in self.package_urls[locale]: self.package_urls[locale]['partialInfo'] = self._get_partial_info( self.package_urls[locale]['partialMarUrl']) if retval == SUCCESS: self.info('Upload successful (%s)' % (locale)) ret = SUCCESS else: self.error('failed to upload %s' % (locale)) ret = FAILURE return ret
def make_upload(self, locale): """wrapper for make upload command""" env = self.query_l10n_env() dirs = self.query_abs_dirs() target = ['upload', 'AB_CD=%s' % (locale)] cwd = dirs['abs_locales_dir'] parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj) retval = self._make(target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser) if retval == SUCCESS: self.info('Upload successful (%s)' % locale) ret = SUCCESS else: self.error('failed to upload %s' % locale) ret = FAILURE if ret == FAILURE: # If we failed above, we shouldn't even attempt a SIMPLE_NAME move # even if we are configured to do so return ret # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.*" % locale matches = (glob.glob(os.path.join(upload_target, glob_name)) + glob.glob(os.path.join(upload_target, 'update', glob_name)) + glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)) + glob.glob(os.path.join(upload_target, 'install', 'sea', glob_name)) + glob.glob(os.path.join(upload_target, 'setup.exe')) + glob.glob(os.path.join(upload_target, 'setup-stub.exe'))) targets_exts = ["tar.bz2", "dmg", "langpack.xpi", "checksums", "zip", "installer.exe", "installer-stub.exe"] targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts] targets.extend([(f, f) for f in ('setup.exe', 'setup-stub.exe')]) for f in matches: possible_targets = [ (tail, target_file) for (tail, target_file) in targets if f.endswith(tail) ] if len(possible_targets) == 1: _, target_file = possible_targets[0] # Remove from list of available options for this locale targets.remove(possible_targets[0]) else: # wasn't valid (or already matched) raise RuntimeError("Unexpected matching file name encountered: %s" % f) self.move(os.path.join(f), os.path.join(target_path, target_file)) self.log("Converted uploads for %s to simple names" % locale) return ret
def _do_postupload_upload(self, upload_dir, ssh_key, ssh_user, remote_host, postupload_cmd): ssh = self.query_exe('ssh') remote_path = self.get_output_from_command( [ssh, '-l', ssh_user, '-i', ssh_key, remote_host, 'mktemp -d'] ) if not remote_path.endswith('/'): remote_path += '/' retval = self.rsync_upload_directory(upload_dir, ssh_key, ssh_user, remote_host, remote_path) if retval is not None: self.error("Failed to upload %s to %s@%s:%s!" % (upload_dir, ssh_user, remote_host, remote_path)) self.return_code = 2 else: # post_upload.py parser = MakeUploadOutputParser( config=self.config, log_obj=self.log_obj ) # build filelist filelist = [] for dirpath, dirname, filenames in os.walk(upload_dir): for f in filenames: # use / instead of os.path.join() because this is part of # a post_upload.py call on a fileserver, which is probably # not windows path = '%s/%s' % (dirpath, f) path = path.replace(upload_dir, remote_path) filelist.append(path) cmd = [ssh, '-l', ssh_user, '-i', ssh_key, remote_host, '%s %s %s' % (postupload_cmd, remote_path, ' '.join(filelist)) ] retval = self.run_command(cmd, output_parser=parser) self.package_urls = parser.matches if retval != 0: self.error("failed to run %s!" % postupload_cmd) self.return_code = 2 else: self.info("Upload successful.") # cleanup, whether we ran postupload or not cmd = [ssh, '-l', ssh_user, '-i', ssh_key, remote_host, 'rm -rf %s' % remote_path ] self.run_command(cmd)
def make_upload(self, locale): """wrapper for make upload command""" config = self.config env = self.query_l10n_env() dirs = self.query_abs_dirs() buildid = self._query_buildid() replace_dict = { 'buildid': buildid, 'branch': config['branch'] } try: env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict except KeyError: # no base_post_upload_cmd in configuration, just skip it pass target = ['upload', 'AB_CD=%s' % (locale)] cwd = dirs['abs_locales_dir'] parser = MakeUploadOutputParser(config=self.config, log_obj=self.log_obj) retval = self._make(target=target, cwd=cwd, env=env, halt_on_failure=False, output_parser=parser) if locale not in self.package_urls: self.package_urls[locale] = {} self.package_urls[locale].update(parser.matches) if retval == SUCCESS: self.info('Upload successful (%s)' % locale) ret = SUCCESS else: self.error('failed to upload %s' % locale) ret = FAILURE if ret == FAILURE: # If we failed above, we shouldn't even attempt a SIMPLE_NAME move # even if we are configured to do so return ret # XXX Move the files to a SIMPLE_NAME format until we can enable # Simple names in the build system if self.config.get("simple_name_move"): # Assume an UPLOAD PATH upload_target = self.config["upload_env"]["UPLOAD_PATH"] target_path = os.path.join(upload_target, locale) self.mkdir_p(target_path) glob_name = "*.%s.*" % locale matches = (glob.glob(os.path.join(upload_target, glob_name)) + glob.glob(os.path.join(upload_target, 'update', glob_name)) + glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name))) targets_exts = ["tar.bz2", "langpack.xpi", "complete.mar", "checksums"] targets = ["target.%s" % ext for ext in targets_exts] for f in matches: target_file = next(target_file for target_file in targets if f.endswith(target_file[6:])) if target_file: # Remove from list of available options for this locale targets.remove(target_file) else: # wasn't valid (or already matched) raise RuntimeError("Unexpected matching file name encountered: %s" % f) self.move(os.path.join(f), os.path.join(target_path, target_file)) self.log("Converted uploads for %s to simple names" % locale) return ret