def make_upload(self, locale):
"""wrapper for make upload command"""
config = self.config
env = self.query_l10n_env()
dirs = self.query_abs_dirs()
buildid = self._query_buildid()
try:
env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % {'buildid': buildid,
'branch': config['branch']}
except KeyError:
# no base_post_upload_cmd in configuration, just skip it
pass
target = ['upload', 'AB_CD=%s' % (locale)]
cwd = dirs['abs_locales_dir']
parser = MakeUploadOutputParser(config=self.config,
log_obj=self.log_obj)
retval = self._make(target=target, cwd=cwd, env=env,
halt_on_failure=False, output_parser=parser)
if locale not in self.package_urls:
self.package_urls[locale] = {}
self.package_urls[locale].update(parser.matches)
if 'partialMarUrl' in self.package_urls[locale]:
self.package_urls[locale]['partialInfo'] = self._get_partial_info(
self.package_urls[locale]['partialMarUrl'])
if retval == SUCCESS:
self.info('Upload successful (%s)' % (locale))
ret = SUCCESS
else:
self.error('failed to upload %s' % (locale))
ret = FAILURE
return ret
def make_upload(self, locale):
"""wrapper for make upload command"""
env = self.query_l10n_env()
dirs = self.query_abs_dirs()
target = ['upload', 'AB_CD=%s' % (locale)]
cwd = dirs['abs_locales_dir']
parser = MakeUploadOutputParser(config=self.config,
log_obj=self.log_obj)
retval = self._make(target=target, cwd=cwd, env=env,
halt_on_failure=False, output_parser=parser)
if retval == SUCCESS:
self.info('Upload successful (%s)' % locale)
ret = SUCCESS
else:
self.error('failed to upload %s' % locale)
ret = FAILURE
if ret == FAILURE:
# If we failed above, we shouldn't even attempt a SIMPLE_NAME move
# even if we are configured to do so
return ret
# XXX Move the files to a SIMPLE_NAME format until we can enable
# Simple names in the build system
if self.config.get("simple_name_move"):
# Assume an UPLOAD PATH
upload_target = self.config["upload_env"]["UPLOAD_PATH"]
target_path = os.path.join(upload_target, locale)
self.mkdir_p(target_path)
glob_name = "*.%s.*" % locale
matches = (glob.glob(os.path.join(upload_target, glob_name)) +
glob.glob(os.path.join(upload_target, 'update', glob_name)) +
glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)) +
glob.glob(os.path.join(upload_target, 'install', 'sea', glob_name)) +
glob.glob(os.path.join(upload_target, 'setup.exe')) +
glob.glob(os.path.join(upload_target, 'setup-stub.exe')))
targets_exts = ["tar.bz2", "dmg", "langpack.xpi",
"checksums", "zip",
"installer.exe", "installer-stub.exe"]
targets = [(".%s" % (ext,), "target.%s" % (ext,)) for ext in targets_exts]
targets.extend([(f, f) for f in ('setup.exe', 'setup-stub.exe')])
for f in matches:
possible_targets = [
(tail, target_file)
for (tail, target_file) in targets
if f.endswith(tail)
]
if len(possible_targets) == 1:
_, target_file = possible_targets[0]
# Remove from list of available options for this locale
targets.remove(possible_targets[0])
else:
# wasn't valid (or already matched)
raise RuntimeError("Unexpected matching file name encountered: %s"
% f)
self.move(os.path.join(f),
os.path.join(target_path, target_file))
self.log("Converted uploads for %s to simple names" % locale)
return ret
def _do_postupload_upload(self, upload_dir, ssh_key, ssh_user, remote_host,
postupload_cmd):
ssh = self.query_exe('ssh')
remote_path = self.get_output_from_command(
[ssh, '-l', ssh_user, '-i', ssh_key, remote_host, 'mktemp -d']
)
if not remote_path.endswith('/'):
remote_path += '/'
retval = self.rsync_upload_directory(upload_dir, ssh_key, ssh_user,
remote_host, remote_path)
if retval is not None:
self.error("Failed to upload %s to %s@%s:%s!" % (upload_dir, ssh_user, remote_host, remote_path))
self.return_code = 2
else: # post_upload.py
parser = MakeUploadOutputParser(
config=self.config,
log_obj=self.log_obj
)
# build filelist
filelist = []
for dirpath, dirname, filenames in os.walk(upload_dir):
for f in filenames:
# use / instead of os.path.join() because this is part of
# a post_upload.py call on a fileserver, which is probably
# not windows
path = '%s/%s' % (dirpath, f)
path = path.replace(upload_dir, remote_path)
filelist.append(path)
cmd = [ssh,
'-l', ssh_user,
'-i', ssh_key,
remote_host,
'%s %s %s' % (postupload_cmd, remote_path, ' '.join(filelist))
]
retval = self.run_command(cmd, output_parser=parser)
self.package_urls = parser.matches
if retval != 0:
self.error("failed to run %s!" % postupload_cmd)
self.return_code = 2
else:
self.info("Upload successful.")
# cleanup, whether we ran postupload or not
cmd = [ssh,
'-l', ssh_user,
'-i', ssh_key,
remote_host,
'rm -rf %s' % remote_path
]
self.run_command(cmd)
def make_upload(self, locale):
"""wrapper for make upload command"""
config = self.config
env = self.query_l10n_env()
dirs = self.query_abs_dirs()
buildid = self._query_buildid()
replace_dict = {
'buildid': buildid,
'branch': config['branch']
}
try:
env['POST_UPLOAD_CMD'] = config['base_post_upload_cmd'] % replace_dict
except KeyError:
# no base_post_upload_cmd in configuration, just skip it
pass
target = ['upload', 'AB_CD=%s' % (locale)]
cwd = dirs['abs_locales_dir']
parser = MakeUploadOutputParser(config=self.config,
log_obj=self.log_obj)
retval = self._make(target=target, cwd=cwd, env=env,
halt_on_failure=False, output_parser=parser)
if locale not in self.package_urls:
self.package_urls[locale] = {}
self.package_urls[locale].update(parser.matches)
if retval == SUCCESS:
self.info('Upload successful (%s)' % locale)
ret = SUCCESS
else:
self.error('failed to upload %s' % locale)
ret = FAILURE
if ret == FAILURE:
# If we failed above, we shouldn't even attempt a SIMPLE_NAME move
# even if we are configured to do so
return ret
# XXX Move the files to a SIMPLE_NAME format until we can enable
# Simple names in the build system
if self.config.get("simple_name_move"):
# Assume an UPLOAD PATH
upload_target = self.config["upload_env"]["UPLOAD_PATH"]
target_path = os.path.join(upload_target, locale)
self.mkdir_p(target_path)
glob_name = "*.%s.*" % locale
matches = (glob.glob(os.path.join(upload_target, glob_name)) +
glob.glob(os.path.join(upload_target, 'update', glob_name)) +
glob.glob(os.path.join(upload_target, '*', 'xpi', glob_name)))
targets_exts = ["tar.bz2", "langpack.xpi", "complete.mar", "checksums"]
targets = ["target.%s" % ext for ext in targets_exts]
for f in matches:
target_file = next(target_file for target_file in targets
if f.endswith(target_file[6:]))
if target_file:
# Remove from list of available options for this locale
targets.remove(target_file)
else:
# wasn't valid (or already matched)
raise RuntimeError("Unexpected matching file name encountered: %s"
% f)
self.move(os.path.join(f),
os.path.join(target_path, target_file))
self.log("Converted uploads for %s to simple names" % locale)
return ret