def merge(self, dest_head, source_head, silent):
'''Invoke git merge.'''
command = 'git merge --no-edit --allow-unrelated-histories --no-stat -m "Merge channel \'%s\'" %s %s' \
% (source_head, mkarg(dest_head), mkarg(source_head))
command = self.silence(command, silent)
wait = self.popen2(command, False)
wait()
def exec_and_parse_bugscript(handler, bugscript):
"""Execute and parse the output of the package bugscript, in particular
identifying the headers and pseudo-headers blocks, if present"""
fh, filename = TempFile()
fh.close()
rc = os.system('LC_ALL=C %s %s %s' % (handler, commands.mkarg(bugscript),
commands.mkarg(filename)))
isheaders = False
ispseudoheaders = False
headers = pseudoheaders = text = ''
fp = open(filename)
for line in fp.readlines():
# we identify the blocks for headers and pseudo-h
if line == '-- BEGIN HEADERS --\n':
isheaders = True
elif line == '-- END HEADERS --\n':
isheaders = False
elif line == '-- BEGIN PSEUDOHEADERS --\n':
ispseudoheaders = True
elif line == '-- END PSEUDOHEADERS --\n':
ispseudoheaders = False
else:
if isheaders:
headers += line
elif ispseudoheaders:
pseudoheaders += line
else:
text += line
fp.close()
cleanup_temp_file(filename)
text = text.decode('utf-8', 'replace')
return (rc, headers, pseudoheaders, text)
def merge(self, dest_head, source_head, silent):
'''Invoke git merge.'''
command = 'git merge -n "Merge" %s %s' \
% (mkarg(dest_head), mkarg(source_head))
command = self.silence(command, silent)
wait = self.popen2(command, False)
wait()
def download(self, url, urldata, d):
dldir = d.getVar("DL_DIR", True)
m = __pattern__.match(url)
path = m.group("path")
host = m.group("host")
port = m.group("port")
user = m.group("user")
password = m.group("pass")
if port:
portarg = "-P %s" % port
else:
portarg = ""
if user:
fr = user
if password:
fr += ":%s" % password
fr += "@%s" % host
else:
fr = host
fr += ":%s" % path
import commands
cmd = "scp -B -r %s %s %s/" % (portarg, commands.mkarg(fr), commands.mkarg(dldir))
bb.fetch2.check_network_access(d, cmd, urldata.url)
runfetchcmd(cmd, d)
def download(self, ud, d):
"""Fetch urls"""
urlo = URI(ud.url)
basecmd = "sftp -oPasswordAuthentication=no"
port = ""
if urlo.port:
port = "-P %d" % urlo.port
urlo.port = None
dldir = data.getVar("DL_DIR", d, True)
lpath = os.path.join(dldir, ud.localfile)
user = ""
if urlo.userinfo:
user = urlo.userinfo + "@"
path = urlo.path
# Supoprt URIs relative to the user's home directory, with
# the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>).
if path[:3] == "/~/":
path = path[3:]
remote = "%s%s:%s" % (user, urlo.hostname, path)
cmd = "%s %s %s %s" % (basecmd, port, commands.mkarg(remote), commands.mkarg(lpath))
bb.fetch2.check_network_access(d, cmd, ud.url)
runfetchcmd(cmd, d)
return True
def download(self, urldata, d):
dldir = d.getVar('DL_DIR', True)
m = __pattern__.match(urldata.url)
path = m.group('path')
host = m.group('host')
port = m.group('port')
user = m.group('user')
password = m.group('pass')
if port:
portarg = '-P %s' % port
else:
portarg = ''
if user:
fr = user
if password:
fr += ':%s' % password
fr += '@%s' % host
else:
fr = host
fr += ':%s' % path
import commands
cmd = 'scp -B -r %s %s %s/' % (
portarg,
commands.mkarg(fr),
commands.mkarg(dldir)
)
bb.fetch2.check_network_access(d, cmd, urldata.url)
runfetchcmd(cmd, d)
def unpack(boot_img, output):
""" Unpack the boot image into the output directory.
"""
Utils.format(boot_img)
tool_kit = Toolkit()
# Try unpack tool set to find the suitable one
boot_type = tool_kit.parse_boot_img_type(boot_img)
# Check whether the tools exists
if boot_type is None:
raise ValueError("Unknown boot image type: " + boot_img)
# Execute the unpack command
unpack_tool = tool_kit.get_tools(boot_type, "UNPACK")
cmd = "%s %s %s" % (commands.mkarg(unpack_tool), commands.mkarg(boot_img), commands.mkarg(output))
(status, result) = commands.getstatusoutput(cmd)
if status != 0:
print "\n* Unpack failed"
print result
else:
print "\n* Unpack %s %s --> %s" % (boot_type, boot_img, output)
# Store the used tools to output
Utils.write_boot_img_type(boot_type, output)
def new_branch(self, dest_head, source_head, silent):
'''Make a brand new branch.'''
command = 'git branch %s %s' \
% (mkarg(dest_head), mkarg(source_head))
command = self.silence(command, silent)
wait = self.popen2(command, False)
wait()
def run(self):
i = self.n
while i < len(self.jobs):
(cfg, gwssers, site, testwords, epochs, num) = self.jobs[i]
i = i + NUM_THREADS
# do the tests on all gwssers - do 2 tries, 15 seconds apart
max_epoch_site = -1
for (gws, port) in gwssers:
cmd = ". %s; cd %s/local/google3/enterprise/legacy/checks && "\
"./gws_production_check.py %s %d %s %s %s %d" % (
cfg.getGlobalParam('ENTERPRISE_BASHRC'),
cfg.entHome,
commands.mkarg(gws),
port,
commands.mkarg(site),
commands.mkarg(testwords),
commands.mkarg(string.join(map(str, epochs), ",")),
num)
logging.info("Executing %s" % cmd)
(err, msgs) = E.getstatusoutput(cmd)
max_epoch = None; errors = None
exec("(max_epoch, errors) = %s" % msgs)
if max_epoch > max_epoch_site:
max_epoch_site = max_epoch
if errors:
self.errors[site] = errors
self.max_epochs[site] = max_epoch_site
os.remove(testwords)
def nslookup(name, dnsserver='', prevent_unqualified_dns=True):
"""check whether DNS servers are up to snuff"""
# if it looks like an IP address, don't try to resolve it
if digitsAndDots.match(name):
return (0, "OK")
if name != "localhost" and prevent_unqualified_dns:
name = name + "." # prevent unqualified DNS lookups
# TODO: we really want to call something along the lines of
# google2/io/safe_gethostbyname, this will require some python trickery.
# If dnsserver is an empty string, then mkarg() will escape it with
# quotes and the host call will try to use "''" as a dns server and fail
# So call mkarg only if actually given a non-empty-string dnsserver
if not dnsserver:
dnsserver = ''
if dnsserver != '':
dnsserver = commands.mkarg(dnsserver)
executestring = commands.mkarg(
"host -t a %s %s 2>/dev/null | grep has\ address | wc -l"
% (commands.mkarg(name), dnsserver))
(stat, out) = commands.getstatusoutput('alarm 5 sh -c %s' % executestring)
if stat != 0:
return (1, "TIMEOUT") # E.g. DNS server does not respond
if int(out) == 0:
return (2, "cannot resolve")
return (0, "OK")
def download(self, ud, d):
"""Fetch urls"""
urlo = URI(ud.url)
basecmd = 'sftp -oBatchMode=yes'
port = ''
if urlo.port:
port = '-P %d' % urlo.port
urlo.port = None
dldir = data.getVar('DL_DIR', d, True)
lpath = os.path.join(dldir, ud.localfile)
user = ''
if urlo.userinfo:
user = urlo.userinfo + '@'
path = urlo.path
# Supoprt URIs relative to the user's home directory, with
# the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>).
if path[:3] == '/~/':
path = path[3:]
remote = '%s%s:%s' % (user, urlo.hostname, path)
cmd = '%s %s %s %s' % (basecmd, port, commands.mkarg(remote),
commands.mkarg(lpath))
bb.fetch2.check_network_access(d, cmd, ud.url)
runfetchcmd(cmd, d)
return True
def GetRemoteFiles(machine, dir):
cmd = "ssh %s %s" % (
commands.mkarg(machine),
commands.mkarg("ls -1 %s/media_%s_%s.flv" % (
dir, FILE_PARM, FILE_PARM)))
(err, out) = commands.getstatusoutput(cmd)
if err:
sys.exit("Cannot list remotely: %s" % cmd)
return out.split('\n')
def fmt_screens(commands, stdin=None):
screens = []
for command in commands:
screen = "screen -t %s /bin/bash -c %s" % (mkarg(command),
mkarg("cat %s | (%s)" % (stdin.path, command)
if stdin else command))
screens.append(screen)
return screens
def open(url, desktop=None, wait=0):
"""
Open the 'url' in the current desktop's preferred file browser. If the
optional 'desktop' parameter is specified then attempt to use that
particular desktop environment's mechanisms to open the 'url' instead of
guessing or detecting which environment is being used.
Suggested values for 'desktop' are "standard", "KDE", "GNOME", "Mac OS X",
"Windows" where "standard" employs a DESKTOP_LAUNCH environment variable to
open the specified 'url'. DESKTOP_LAUNCH should be a command, possibly
followed by arguments, and must have any special characters shell-escaped.
The process identifier of the "opener" (ie. viewer, editor, browser or
program) associated with the 'url' is returned by this function. If the
process identifier cannot be determined, None is returned.
An optional 'wait' parameter is also available for advanced usage and, if
'wait' is set to a true value, this function will wait for the launching
mechanism to complete before returning (as opposed to immediately returning
as is the default behaviour).
"""
# Attempt to detect a desktop environment.
detected = get_desktop()
# Give higher priority to user preference
if (desktop is None or desktop == "override") and detected == "override":
global override
arg = override.replace("%url%", commands.mkarg(url))
return _run(arg, 1, wait)
# Desktops whose existence can be easily tested.
elif (desktop is None or desktop == "standard") and is_standard():
arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)])
return _run(arg, 1, wait)
elif (desktop is None or desktop == "Windows") and detected == "Windows":
# NOTE: This returns None in current implementations.
try:
return os.startfile(url)
except OSError:
# some broken configs may return error here but work anyway
return
elif desktop is None:
desktop = detected
cmd = get_command(desktop, url)
if cmd:
return _run(cmd, 0, wait)
else:
# Finish with an error where no suitable desktop was identified.
webbrowser.open(url)
def _Diff(lhs, rhs):
"""Run standard unix 'diff' against two files."""
cmd = '${TEST_DIFF:-diff} %s %s' % (commands.mkarg(lhs), commands.mkarg(rhs))
(status, output) = commands.getstatusoutput(cmd)
if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 1:
# diff outputs must be the same as c++ and shell
raise OutputDifferedError('\nRunning %s\n%s\nTest output differed '
'from golden file\n' % (cmd, output))
elif not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0:
raise DiffFailureError('\nRunning %s\n%s\nFailure diffing test output '
'with golden file\n' % (cmd, output))
def thumbnail_image(self, src_path, dest_path, size):
"""Generate a square thumbnail, with on-disk source and destination paths."""
child = Popen4("%s -geometry %dx%d %s %s" % (
self.CONVERT,
size, size,
commands.mkarg(src_path),
commands.mkarg(dest_path),
))
output = child.fromchild.read()
status = child.wait()
if status:
raise ImageException(output)
def sign_message(body, fromaddr, package='x', pgp_addr=None, sign='gpg', draftpath=None):
'''Sign message with pgp key.'''
''' Return: a signed body.
On failure, return None.
kw need to have the following keys
'''
if not pgp_addr:
pgp_addr = get_email_addr(fromaddr)[1]
# Make the unsigned file first
(unsigned, file1) = TempFile(prefix=tempfile_prefix(package, 'unsigned'), dir=draftpath)
unsigned.write(body)
unsigned.close()
# Now make the signed file
(signed, file2) = TempFile(prefix=tempfile_prefix(package, 'signed'), dir=draftpath)
signed.close()
if sign == 'gpg':
os.unlink(file2)
if 'GPG_AGENT_INFO' not in os.environ:
signcmd = "gpg --local-user '%s' --clearsign " % pgp_addr
else:
signcmd = "gpg --local-user '%s' --use-agent --clearsign " % pgp_addr
signcmd += '--output '+commands.mkarg(file2)+ ' ' + commands.mkarg(file1)
else:
signcmd = "pgp -u '%s' -fast" % pgp_addr
signcmd += '<'+commands.mkarg(file1)+' >'+commands.mkarg(file2)
try:
os.system(signcmd)
x = file(file2, 'r')
signedbody = x.read()
x.close()
if os.path.exists(file1):
os.unlink(file1)
if os.path.exists(file2):
os.unlink(file2)
if not signedbody:
raise NoMessage
body = signedbody
except (NoMessage, IOError, OSError):
fh, tmpfile2 = TempFile(prefix=tempfile_prefix(package), dir=draftpath)
fh.write(body)
fh.close()
ewrite('gpg/pgp failed; input file in %s\n', tmpfile2)
body = None
return body
def doLogReport(self, jobName, jobToken, collection, reportName, reportDate,
withResults, topCount, diagnosticTerms, update):
"""The actual work done in a worker thread to generate a summary report."""
(html_file, valid_file) = liblog.get_report_filenames(self.entConfig,
liblog.SUMMARY_REPORT, reportName, collection)
liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file))
new_html_file = tempfile.mktemp('.report')
new_valid_file = tempfile.mktemp('.report_valid')
args = []
args.append(commands.mkarg(self.entConfig.GetEntHome()))
args.append(commands.mkarg(collection))
args.append(commands.mkarg(reportDate))
args.append(withResults)
args.append(topCount)
args.append(commands.mkarg(diagnosticTerms))
args.append(commands.mkarg(html_file))
args.append(commands.mkarg(valid_file))
args.append(commands.mkarg(new_html_file))
args.append(commands.mkarg(new_valid_file))
cmd = ('. %s && cd %s/enterprise/legacy/logs && '
'alarm %s nice -n 15 ./log_report.py %s' %
(self.cfg.getGlobalParam('ENTERPRISE_BASHRC'),
self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'),
COMMAND_TIMEOUT_PERIOD, string.join(args, ' ')))
logging.info('doLogReport(): CMD = %s' % cmd)
returnCode = E.system(cmd)
self.handleResult(jobName, jobToken, returnCode, liblog.SUMMARY_REPORT,
collection, reportName, update,
html_file, valid_file, new_html_file, new_valid_file)
def do_edit(given_cl, current_cl, cl_file_path):
if given_cl.is_unspecified():
# Show an editor if CL not specified on the command-line
tmp_fd, tmp_path = tempfile.mkstemp(prefix='appspot-', suffix='.txt')
os.write(tmp_fd, editable_change(current_cl))
os.close(tmp_fd)
retcode = subprocess.call(
'%s %s' % (os.getenv('VISUAL', os.getenv('EDITOR', 'vi')),
commands.mkarg(tmp_path)),
shell=True)
try:
if retcode < 0:
raise Exception('editor closed with signal %s' % -retcode)
elif retcode > 0:
raise Exception('editor exited with error value %s' % retcode)
edited_cl = parse_change(open(tmp_path).read())
finally:
os.remove(tmp_path)
if edited_cl.is_unspecified():
print >>sys.stderr, 'cancelled edit'
return
edited_cl.merge_into(current_cl)
else:
given_cl.merge_into(current_cl)
out = open(cl_file_path, 'w')
out.write(editable_change(current_cl))
out.close()
def get_changed_config_files(conffiles, nocompress=False):
confinfo = {}
changed = []
for (filename, md5sum) in conffiles:
try:
fp = file(filename)
except IOError, msg:
confinfo[filename] = msg
continue
filemd5 = commands.getoutput("md5sum " + commands.mkarg(filename)).split()[0]
if filemd5 == md5sum:
continue
changed.append(filename)
thisinfo = "changed:\n"
for line in fp:
if not line:
continue
if line == "\n" and not nocompress:
continue
if line[0] == "#" and not nocompress:
continue
thisinfo += line
confinfo[filename] = thisinfo.decode("utf-8", "replace")
def Submit(executable=None, runHours=3, runMinutes=0, jobname="pyprop", printScript=False, **args):
"""
Set up a job script suitable
"""
#Create jobscript
jscript = submitpbs.SubmitScript()
jscript.jobname = jobname
jscript.walltime = timedelta(hours=runHours, minutes=runMinutes, seconds=0)
jscript.ppn = args.get("ppn", 4)
jscript.proc_memory = args.get("proc_memory", "1000mb")
jscript.nodes = args.get("nodes", "1")
numProcs = args.get("numProcs", jscript.ppn*jscript.nodes)
if installation == "stallo":
jscript.interconnect = args.get("interconnect", "ib")
jscript.workingdir = args.get("workingDir", os.path.abspath(os.curdir) )
jscript.executable = "mpirun -n %s " % (jscript.ppn*jscript.nodes)
jscript.executable += "python %s" % executable
elif installation == "hexagon":
jscript.workingdir = args.get("workingDir", os.path.abspath(os.curdir) )
jscript.executable = "aprun -n %s -N %s " % (numProcs, jscript.ppn)
jscript.executable += "./python-exec %s" % executable
jscript.parameters = commands.mkarg(repr(args))
#Submit this job
if printScript:
print "\n".join(jscript.CreateScript())
else:
jscript.Submit()
def send(self, filename):
elisp = """(progn
(load-file "/usr/share/reportbug/reportbug.el")
(tfheen-reportbug-insert-template "%s"))"""
filename = re.sub('["\\\\]', "\\\\\\g<0>", filename)
elisp = commands.mkarg(elisp % filename)
return ui.system("emacsclient --no-wait --eval %s 2>/dev/null" " || emacs --eval %s" % (elisp, elisp))
def RunAlarmRemoteCmd(mach, cmd, timeout, remote_user=None):
if timeout < 0: # alarm disabled
alarm_cmd = ""
else:
alarm_cmd = "alarm %s " % timeout
# prepend AM user
if remote_user:
remote_string = "%s@" % remote_user
else:
remote_string = ""
entire_cmd = (". /etc/profile; %s ssh -q -P -n -o 'BatchMode=yes' "
"-o 'ForwardAgent=yes' -o 'StrictHostKeyChecking=no' %s%s %s" %
(alarm_cmd, remote_string, mach, commands.mkarg(cmd)))
if verbose:
log("Running %s " % entire_cmd)
try:
(status, output) = commands.getstatusoutput(entire_cmd)
if status:
(status, sig) = (status >> 8, status & 255) # separate high (status)
# and low byte (signal)
else:
(status, sig) = (0, 0)
except OSError, e:
(status, sig, output) = (e.errno, 0, '')
def klipper(args):
import os, commands
args = " ".join([commands.mkarg(str(arg)) for arg in args])
s, o = commands.getstatusoutput("qdbus org.kde.klipper /klipper " + args)
if s == 0:
return o
return ""
def SubmitDelayScan(**args):
delayList = args["delayList"]
outputfile = args["outputfile"]
molecule = args["molecule"]
partitionCount = args["partitionCount"]
partitionSize = int(ceil(len(delayList) / float(partitionCount)))
partitionStep = delayList[1] - delayList[0] # Assume equidistant delayList
plist = []
for i in range(partitionCount):
start = min(i * partitionSize, len(delayList) - 1)
end = min((i + 1) * partitionSize, len(delayList))
delaySlice = slice(delayList[start], delayList[end - 1] + partitionStep, partitionStep)
args["delayList"] = delaySlice
args["outputfile"] = outputfile % i
script = submitpbs.SubmitScript()
script.interconnect = None
script.executable = "./run_delay_scan.py"
script.parameters = commands.mkarg(repr(args))
script.ppn = 1
script.nodes = 1
script.walltime = submitpbs.timedelta(hours=6)
print "\n".join(script.CreateScript())
script.Submit()
def access(machines, file, access):
""" Test the access mode for a file/directory"""
# This is a small hack to increase the spead when checking on
# local machine (in interface happens a lot)
if (len(machines) == 1 and ( LOCALHOST == machines[0] or
machines[0] == getCrtHostName() )):
try:
a = os.F_OK
if 'r' in access: a = a | os.R_OK
if 'w' in access: a = a | os.W_OK
if 'x' in access: a = a | os.X_OK
if not os.access(file, a): return 0
s = os.stat(file)
if 'd' in access and not stat.S_ISDIR(s[stat.ST_MODE]): return 0
if 'f' in access and not (stat.S_ISREG(s[stat.ST_MODE])): return 0
return 1
except OSError:
return 0
cmd = []
for a in access:
cmd.append(ACCESS % (a, commands.mkarg(file)))
return ERR_OK == execute(machines, string.join(cmd), None, true)
def start_service(self, service):
"""
Activates and starts the given service on the given machine
"""
logging.info("ACTIVATE: %s service %s on %s" % (
service, self.version_, self.machine_))
cmd = E.nonblocking_cmd(
"/etc/rc.d/init.d/%s_%s activate" % (service, self.version_))
ret = E.exe("ssh %s %s" % (self.machine_, commands.mkarg(cmd)))
if ret == 0:
logging.info("START: %s service %s on %s" % (
service, self.version_, self.machine_))
cmd = E.nonblocking_cmd(
"/etc/rc.d/init.d/%s_%s start" % (service, self.version_))
ret = E.exe("ssh %s %s" % (self.machine_, commands.mkarg(cmd)))
return ret
def direct_fetch(self, url, upstream_name):
'''Use git directly to do a pull.'''
# for now we assume master, as HEAD is inaccessible. too bad.
refspec = '+master:%s' % (upstream_name)
command = 'git fetch %s %s' % (mkarg(url), refspec)
wait = self.popen2(command, False)
wait()
def CronTabEntry(config, timespec, cmds, log_file, source_env):
cmd = "(" + string.join(cmds, '; ') + ") >& " + log_file
script_dir = config.var('MAIN_GOOGLE3_DIR') + "/enterprise/legacy/util"
str = "( %s; cd %s && ./run_cron_command.py %s ) &> /dev/null" % (
source_env, script_dir, commands.mkarg(cmd))
str = "\n\n" + timespec + '\t ' + str + '\n'
return str
def open(url, desktop=None, wait=0):
"""
Open the 'url' in the current desktop's preferred file browser. If the
optional 'desktop' parameter is specified then attempt to use that
particular desktop environment's mechanisms to open the 'url' instead of
guessing or detecting which environment is being used.
Suggested values for 'desktop' are "standard", "KDE", "GNOME", "XFCE",
"Mac OS X", "Windows" where "standard" employs a DESKTOP_LAUNCH environment
variable to open the specified 'url'. DESKTOP_LAUNCH should be a command,
possibly followed by arguments, and must have any special characters
shell-escaped.
The process identifier of the "opener" (ie. viewer, editor, browser or
program) associated with the 'url' is returned by this function. If the
process identifier cannot be determined, None is returned.
An optional 'wait' parameter is also available for advanced usage and, if
'wait' is set to a true value, this function will wait for the launching
mechanism to complete before returning (as opposed to immediately returning
as is the default behaviour).
"""
# Decide on the desktop environment in use.
desktop_in_use = use_desktop(desktop)
if desktop_in_use == "standard":
arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)])
return _run(arg, 1, wait)
elif desktop_in_use == "Windows":
# NOTE: This returns None in current implementations.
return os.startfile(url)
elif desktop_in_use == "KDE4":
cmd = ["kioclient", "exec", url]
elif desktop_in_use == "KDE":
cmd = ["kfmclient", "exec", url]
elif desktop_in_use == "GNOME":
cmd = ["gnome-open", url]
elif desktop_in_use == "XFCE":
cmd = ["exo-open", url]
elif desktop_in_use == "Mac OS X":
cmd = ["open", url]
elif desktop_in_use == "X11" and os.environ.has_key("BROWSER"):
cmd = [os.environ["BROWSER"], url]
# Finish with an error where no suitable desktop was identified.
else:
raise OSError, "Desktop '%s' not supported (neither DESKTOP_LAUNCH nor os.startfile could be used)" % desktop_in_use
return _run(cmd, 0, wait)
def format_cmdline(args):
return ' '.join(
commands.mkarg(arg).strip() if any(s in arg
for s in (" '\"$")) else arg
for arg in args)
def decryptLicense(self, encrypted_license):
tempdir = E.mktemp(self.working_dir, 'decrypt')
signed_fn = E.joinpaths([tempdir, "license.signed"])
verified_fn = E.joinpaths([tempdir, "license.verified"])
encrypted_fn = E.joinpaths([tempdir, "license.encrypted"])
tgz_fn = E.joinpaths([tempdir, "license.tgz"])
license_fn = E.joinpaths([tempdir, "ent_license.config"])
if not self.checkFiles():
logging.error("decryptor keys don't exist or aren't valid")
return None, None, DECRYPT_SYSTEM_ERROR
try:
# make the temp directory
os.system("rm -rf %s" % commands.mkarg(tempdir))
os.makedirs(tempdir)
# output to a file
try:
open(signed_fn, 'w').write(encrypted_license)
except IOError:
logging.error("writing encrypted license failed")
return None, None, DECRYPT_SYSTEM_ERROR
# verify signature
verify_cmd = ("gpg --no-options --no-default-keyring --no-tty"
" --no-verbose --yes --keyring %s -o %s --decrypt %s"
) % (commands.mkarg(self.license_public_keyring),
commands.mkarg(verified_fn),
commands.mkarg(signed_fn))
# When gpg is given a file with no signature it returns successfully;
# so we must check gpg's output
fi, foe = os.popen4(verify_cmd)
fi.close()
gpg_result = foe.read()
foe.close()
if gpg_result.find("Good signature") == -1:
logging.error("verifying license signature failed")
return None, None, DECRYPT_GPG_VERIFY_FAILED
# remove header
verified_body = open(verified_fn, 'r').read()
sep_poi = string.find(verified_body, "\n\n")
if sep_poi < 0:
logging.error("license doesn't have valid header")
return None, None, DECRYPT_BAD_HEADER
# NOTE: At this point, we've read the header, which is a subset of the
# actual license (which we can't see yet, because it's encrypted). If
# we fail past this point, we return the header in place of the license
# because it will look like an invalid license, and allow us to report
# some basic information about the license.
#
header_body = verified_body[0:sep_poi]
# write out the encrypted body
try:
open(encrypted_fn, 'w').write(verified_body[sep_poi + 2:])
except IOError:
logging.error("writing encrypted license failed")
return None, header_body, DECRYPT_SYSTEM_ERROR
# decrypt it
decrypt_cmd = "gpg --no-options --no-default-keyring --no-tty --yes --keyring %s --secret-keyring %s -o %s --decrypt %s" % (
commands.mkarg(self.box_public_keyring),
commands.mkarg(self.box_private_keyring),
commands.mkarg(tgz_fn), commands.mkarg(encrypted_fn))
if os.system(decrypt_cmd) != 0:
logging.error("decrypting license failed")
return None, header_body, DECRYPT_GPG_DECRYPT_FAILED
# untar it (we only ask for the file we want)
untar_cmd = "tar xvzf %s -C %s ent_license.config" % (
commands.mkarg(tgz_fn), tempdir)
if os.system(untar_cmd) != 0:
logging.error("untarring license failed")
return None, header_body, DECRYPT_TAR_FAILED
# read in the decrypted license
try:
license_body = open(license_fn, 'r').read()
except IOError:
logging.error("reading decrypted license failed")
return None, header_body, DECRYPT_SYSTEM_ERROR
finally:
# make sure to clean up all the temp files
os.system("rm -rf %s" % commands.mkarg(tempdir))
return license_body, header_body, DECRYPT_OK
def send(self, filename):
mua = self.command
if '%s' not in mua:
mua += ' %s'
return ui.system(mua % commands.mkarg(filename)[1:])
def run_update_index_cacheinfo(self, mode, blob_id, filename):
'''Run git update-index --cacheinfo.'''
update_command = 'git update-index --cacheinfo %s %s %s' \
% (mode, blob_id, mkarg(filename))
self.shell_to_string(update_command)
def comment(s): return "echo %s" % commands.mkarg(s)
def run(cmd, file): sts, output = commands.getstatusoutput(cmd + commands.mkarg(file)) if sts: print output print 'Exit status', sts return sts
if __name__ == "__main__":
if len(sys.argv) != 4:
sys.stderr.write(
"Usage: %s <search path> <dimensions> <output directory>\n" %
sys.argv[0])
sys.exit(1)
search_path = sys.argv[1]
dimensions = sys.argv[2]
output_dir = sys.argv[3]
paths = []
paths += map(lambda line: line.strip(), find_files(search_path, "*.jef"))
paths += map(lambda line: line.strip(), find_files(search_path, "*.JEF"))
for path in paths:
directory, file_name = os.path.split(path)
if os.system("python jef2png.py --stitches-only " + dimensions + " " +
commands.mkarg(path) + " " + commands.mkarg(
os.path.join(output_dir, file_name + ".png"))) == 0:
print os.path.join(output_dir, file_name + ".png")
else:
sys.stderr.write("Failed to convert file: " + path + "\n")
sys.exit()
def local_wrapped_cmd(command_line, timeout=DEF_WRAPPED_CMD_TIMEOUT_SEC):
"""Use Popen3 to execute a command locally.
This routine uses the Popen3 class in Python's popen2 module to execute a
command locally and retrieve stdout, stderr, *AND* the exit status code, with
a timeout to prevent runaway commands.
This implementation may seem a bit elaborate, but it is required to prevent
deadlock when the command being executed could output to stdout or stderr
in any order.
Arguments:
command_line: command-line string to execute
timeout: maximum run-time allowed, in seconds; default is 1 minute
Returns:
( [ 'stdout line 0\n', 'stdout line 1\n', ... ],
[ 'stderr line 0\n', 'stderr line 1\n', ... ], exit_code, )
"""
cmd_str = SHELL_GLOB_WRAPPER_FMT % commands.mkarg(command_line)
sub_process = popen2.Popen3(cmd_str, 1)
# limit how long to wait for the subprocess to complete
signal.signal(signal.SIGALRM, _sigalrm_handler)
signal.alarm(timeout)
# create empty lists for threads to append their output
std_out_lines = []
std_err_lines = []
std_out_thread = threading.Thread(name='LocalWrappedCmdStdOut',
target=_wrapped_readlines_thread,
kwargs={
'child_pipe': sub_process.fromchild,
'out_list': std_out_lines,
})
std_err_thread = threading.Thread(name='LocalWrappedCmdStdErr',
target=_wrapped_readlines_thread,
kwargs={
'child_pipe': sub_process.childerr,
'out_list': std_err_lines,
})
# ensure that program will exit even if these threads are running
std_out_thread.setDaemon(1)
std_err_thread.setDaemon(1)
timed_out = 0
try:
std_out_thread.start()
std_err_thread.start()
while std_out_thread.isAlive() or std_err_thread.isAlive():
# wait until both threads are finished (or timeout interrupts them)
pass
except _TimeoutError:
# kill the too-long running child process
os.kill(sub_process.pid, signal.SIGTERM)
timed_out = 1
signal.alarm(0) # shut off the alarm now
exit_code = sub_process.wait()
## NOTE: threads will get an EOF from the pipe and then exit on their own
# wait() returns a variety of items, so "decode" the value into an exit code
if exit_code == None:
exit_code = 0
elif os.WIFEXITED(exit_code):
# child process exit codes are multiplied by 256, so undo this
exit_code = os.WEXITSTATUS(exit_code)
elif timed_out:
# process was killed by the timeout
exit_code = errno.ETIMEDOUT
elif exit_code != 0:
# process was killed by some other signal
exit_code = errno.EINTR
return std_out_lines, std_err_lines, exit_code
def alarm_wrap(cmd, timeout):
"""wraps the given command in alarm cmd with a specified timeout)"""
return 'alarm %s bash -c %s' % (timeout, commands.mkarg(cmd))
def composeCmd(machines,
param,
alarm,
dist,
verbose,
forceRemote,
enthome=None,
ignore=0):
"""
Helper to create googlesh/googldist commands
@param machines - comma separated list of machines
@param param - the parameter for googleX script(command / file)
@param alarm - wrap each command in an alarm (use -a option)
@param dist - true -> googledist; false -> googlesh
@param enthome - the directory of current version of enterprise.
@param ignore - Number of failures to ignore.
@return String - the command to be executed
"""
if enthome == None:
enthome = getEnterpriseHome()
# I have to remove localhost from the list of machines if is a dist
listMachines = []
listMachines.extend(machines)
if dist:
if LOCALHOST in listMachines:
listMachines.remove(LOCALHOST)
if getCrtHostName() in listMachines:
listMachines.remove(getCrtHostName())
# recompose machines space sepparated
strMachines = string.strip(string.join(listMachines))
# Anything left ?
if not strMachines: return None
if (LOCALHOST == strMachines
or strMachines == getCrtHostName()) and not forceRemote:
alarmcmd = ""
if alarm:
if alarm > 1: alarmcmd = LOCAL_ALARM_CMD % alarm
else: alarmcmd = LOCAL_ALARM_CMD % DEFAULTALARM
cmd = EXEC_HERE % (
enthome,
# We need to do this unescaping when running on the same
# machine.
# NOTE: This may be a problem if param has $ that
# is escaped, in which case that argument will be
# evaluated!
string.replace(param, "\\$", "$"))
if alarmcmd:
cmd = '%s %s' % (alarmcmd, commands.mkarg(cmd))
else:
if alarm:
if alarm > 1: aparam = ALARMPARAM % alarm
else: aparam = ALARMPARAM % DEFAULTALARM
else: aparam = ""
if dist:
cmd = EXEC_REMOTE_DIST % (enthome, aparam,
commands.mkarg(strMachines), ignore,
commands.mkarg(param))
else:
cmd = EXEC_REMOTE_CMD % (enthome, aparam,
commands.mkarg(strMachines), ignore,
commands.mkarg(EXEC_HERE %
(enthome, param)))
if verbose:
logging.info("Running : " + cmd)
return cmd
def encryptLicense(self, license, passphrase=None):
tempdir = E.mktemp(self.working_dir, 'encrypt')
# if no passphrase supplied, use the one given in the constructor
if passphrase == None:
passphrase = self.passphrase
# make sure the license is valid (note that this is a fresh license)
if license.isInvalid(fresh_license=1):
logging.error("license is not valid")
return None
if not self.checkFiles():
logging.error("encryptor keys don't exist or aren't valid")
return None
# get the header
header = self.extractHeader(license)
# NOTE: this ent_license.config is important; the old license code expects
# to find this filename. do not change it
license_fn = E.joinpaths([tempdir, "ent_license.config"])
tgz_fn = E.joinpaths([tempdir, "license.tgz"])
encrypted_fn = E.joinpaths([tempdir, "license.encrypted"])
encrypted_w_header_fn = E.joinpaths([tempdir, "license.header"])
signed_fn = E.joinpaths([tempdir, "license.signed"])
try:
# make the temp directory
os.system("rm -rf %s" % commands.mkarg(tempdir))
os.makedirs(tempdir)
# write the raw license body to a file
license_body = {}
license_body.update(license.license)
# overwrite the version number with the current version (just in case)
license_body['ENT_LICENSE_VERSION'] = CURRENT_LICENSE_VERSION
try:
body_str = string.join(
map(lambda (k, v): '%s = %s' % (k, repr(v)),
license_body.items()), '\n')
open(license_fn, 'w').write(str(body_str))
except IOError:
logging.error("writing plain license failed")
return None
# tar the license body
tar_cmd = "tar zvcf %s -C %s %s" % (
commands.mkarg(tgz_fn),
commands.mkarg(os.path.dirname(license_fn)),
commands.mkarg(os.path.basename(license_fn)))
if os.system(tar_cmd) != 0:
logging.error("tarring license failed")
return None
# encrypt it
encrypt_cmd = "gpg --no-options --no-default-keyring --no-secmem-warning --no-tty --yes --default-recipient-self --keyring %s --secret-keyring %s -o %s -se %s" % (
commands.mkarg(self.box_public_keyring),
commands.mkarg(self.box_private_keyring),
commands.mkarg(encrypted_fn), commands.mkarg(tgz_fn))
if os.system(encrypt_cmd) != 0:
logging.error("encrypting license failed")
return None
# write out the header (plus \n\n) to encrypted_w_header_fn
try:
header_str = string.join(
map(lambda (k, v): '%s = %s' % (k, repr(v)),
header.items()), '\n')
header_str = header_str + '\n\n' # end of header
open(encrypted_w_header_fn, 'w').write(header_str)
except IOError:
logging.error("writing license header failed")
return None
# append the encrypted tarred license to encrypted_w_header_fn
cat_cmd = "cat %s >> %s" % (commands.mkarg(encrypted_fn),
commands.mkarg(encrypted_w_header_fn))
if os.system(cat_cmd) != 0:
logging.error("reading encrypted license failed")
return None
# sign encrypted_w_header_fn
sign_cmd = "gpg --no-options --no-default-keyring --no-tty --no-secmem-warning --keyring %s --secret-keyring %s --passphrase-fd 0 --yes -o %s -s %s" % (
commands.mkarg(self.license_public_keyring),
commands.mkarg(
self.license_private_keyring), commands.mkarg(signed_fn),
commands.mkarg(encrypted_w_header_fn))
try:
# use popen so that we can write the passphrase to stdin
sign_fd = os.popen(sign_cmd, 'w')
if passphrase != None:
sign_fd.write(passphrase)
status = sign_fd.close()
except IOError:
status = -1
if status != None:
logging.error("signing license signature failed")
return None
# read the signed body
try:
signed_body = open(signed_fn, 'r').read()
except IOError:
logging.error("reading signed license failed")
return None
finally:
# make sure to clean up all the temp files
os.system("rm -rf %s" % commands.mkarg(tempdir))
return signed_body
def setUserConfig(self, usersToAdd):
''' delete some users and reset the User Config File'''
cmd = 'secure_script_wrapper -p2 %s setUserConfig %s' % (
self.snmpWrapperPath, commands.mkarg(usersToAdd))
logMsg = 'SNMP users configuration changed'
return self.execCmd(cmd, self.machines, logMsg)
#!/usr/bin/python
"""
argv_to_sh.py
Embed an argv vector into a shell string, e.g. for ssh or sudo.
"""
import commands
import sys
# strategy: double quote if it has ' -- otherwise single quote
for arg in sys.argv[1:]:
sys.stdout.write(commands.mkarg(arg))
def find_files(path, pattern):
so, si = popen2.popen2('find ' + commands.mkarg(path) + ' -name "' +
pattern + '"')
si.close()
return so.readlines()
def quote_args(self, files):
'''Join files into a single shell quoted string.'''
return ''.join([mkarg(f) for f in files])
if G.debug: print 'mode =', mode, 'rate =', rate
if mode in ('au', 'aiff', 'wav', 'aifc', 'ul', 'ub', 'sb') and \
rate in validrates:
tempname = filename
if mode in ('ul', 'ub', 'sb'):
raw = 1
elif cache.has_key(filename):
tempname = cache[filename]
else:
tempname = G.tempprefix + ` rand.rand() ` + '.aiff'
cmd = SOX
if G.debug:
cmd = cmd + ' -V'
if mode <> '':
cmd = cmd + ' -t ' + mode
cmd = cmd + ' ' + commands.mkarg(filename)
cmd = cmd + ' -t aiff'
if rate not in validrates:
rate = 32000
if rate:
cmd = cmd + ' -r ' + ` rate `
cmd = cmd + ' ' + tempname
if G.debug: print cmd
sts = os.system(cmd)
if sts:
print cmd
print 'Exit status', sts
stdwin.fleep()
try:
os.unlink(tempname)
except:
def run_read_tree(self, tree_id):
'''Run git read-tree on the given tree.'''
self.shell_to_string('git read-tree %s' % mkarg(tree_id))
def send(cfg, to, problem, subject, msgText, logSubject):
# No mails in install mode
if cfg.getInstallState() == "INSTALL": return
smtpHost = cfg.getGlobalParam("SMTP_SERVER")
fromEmail = cfg.getGlobalParam("OUTGOING_EMAIL_SENDER")
allSubject = None
if not to:
if problem:
to = cfg.getGlobalParam("PROBLEM_EMAIL")
else:
to = cfg.getGlobalParam("NOTIFICATION_EMAIL")
# if to is empty, we still go through the whole process, but bail out right
# before actually sending the email. we want all the side effects to still
# happen (logging, testing hooks, etc..)
# if "*EMAIL" is None, commands.mkarg() will throw a TypeError exception
if not to:
to = ""
else:
to = string.strip(to)
if problem:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get(
"ENT_BOX_ID", ""), cfg.getGlobalParam("EMAIL_PROBLEM_PREFIX"))
else:
allSubject = "%s: %s" % (
cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get(
"ENT_BOX_ID",
""), cfg.getGlobalParam("EMAIL_NOTIFICATION_PREFIX"))
allSubject = allSubject + subject
# Log the subject
if logSubject:
logging.info('Sending mail: To: %s; Subject: %s' %
(repr(to), repr(subject)))
cfg.writeAdminRunnerOpMsg(subject)
dateString = time.strftime("%Y/%m/%d %H:%M:%S",
E.getLocaltime(time.time()))
if msgText:
msgText = msgText + "\n"
else:
msgText = ""
ipAddr = cfg.getGlobalParam("EXTERNAL_WEB_IP")
body = "%s [%s @%s from %s]" % (msgText, M.MSG_MAIL_AUTOGENERATED,
dateString, ipAddr)
# if we have a testing hook, call that
if TEST_MAIL_HANDLER_HOOK != None:
logging.info("calling TEST_MAIL_HANDLER_HOOK instead of mailnotify")
to_list = filter(None, map(string.strip, string.split(to, ",")))
TEST_MAIL_HANDLER_HOOK(smtpHost, to_list, fromEmail, allSubject, body)
return
# if to is empty, bail out
if not (to and smtpHost): return
# We execute this and not call the function directly because mailnotify
# uses signal and we might call this from a secondary thread.
E.execute([E.LOCALHOST],
". %s; cd %s/local/google3/enterprise/legacy/util; "\
"./mailnotify.py %s %s %s %s %s" % (
cfg.getGlobalParam('ENTERPRISE_BASHRC'),
E.getEnterpriseHome(),
commands.mkarg(smtpHost),
commands.mkarg(allSubject),
commands.mkarg(fromEmail),
commands.mkarg(to),
commands.mkarg(body)), None, 0)
def open(url, desktop=None, wait=0, dialog_on_error=False):
"""
Open the 'url' in the current desktop's preferred file browser. If the
optional 'desktop' parameter is specified then attempt to use that
particular desktop environment's mechanisms to open the 'url' instead of
guessing or detecting which environment is being used.
Suggested values for 'desktop' are "standard", "KDE", "GNOME", "XFCE",
"Mac OS X", "Windows" where "standard" employs a DESKTOP_LAUNCH environment
variable to open the specified 'url'. DESKTOP_LAUNCH should be a command,
possibly followed by arguments, and must have any special characters
shell-escaped.
The process identifier of the "opener" (ie. viewer, editor, browser or
program) associated with the 'url' is returned by this function. If the
process identifier cannot be determined, None is returned.
An optional 'wait' parameter is also available for advanced usage and, if
'wait' is set to a true value, this function will wait for the launching
mechanism to complete before returning (as opposed to immediately returning
as is the default behaviour).
"""
# Decide on the desktop environment in use.
import bauble.utils as utils
desktop_in_use = use_desktop(desktop)
cmd = None
if desktop_in_use == "standard":
arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)])
return _run(arg, 1, wait)
elif desktop_in_use == "Windows":
# NOTE: This returns None in current implementations.
return os.startfile(url)
elif desktop_in_use == "KDE":
cmd = ["kfmclient", "exec", url]
elif desktop_in_use == "GNOME":
cmd = ["gnome-open", url]
elif desktop_in_use == "XFCE":
cmd = ["exo-open", url]
elif desktop_in_use == "Mac OS X":
cmd = ["open", url]
elif desktop_in_use == "X11" and "BROWSER" in os.environ:
cmd = [os.environ["BROWSER"], url]
if not cmd:
# if can't detect the desktop environment then see if xdg-open
# is available
exe = utils.which('xdg-open')
if exe:
cmd = [exe, url]
# Finish with an error where no suitable desktop was
# identified.
try:
if not cmd:
# TODO: maybe we should tell the user to define DESKTOP_LAUNCH
raise OSError, _("Could not open %(url)s\n\n"
"Unknown desktop environment: %(desktop)s\n\n") \
% dict(url=url, desktop=desktop_in_use)
except Exception, e:
if dialog_on_error:
utils.message_dialog(utils.utf8(e))
else:
raise
def quote_shell_arg(cmd):
return commands.mkarg(cmd)
def doReconfigureNet(config, machines=None, i_am_master=1,
force_ntp_reconfig=0):
""" reconfigure serveriron, DNS, NTPs, iptables, timezone as needed.
Force NTP server reconfiguration if force_ntp_reconfig=1
"""
# first we need to reconfigure our external IP address
ret1 = ret2 = ret4 = ret5 = ret6 = ret8 = 0
configType = config.var('ENT_CONFIG_TYPE')
ent_home = E.getEnterpriseHome()
if not machines:
machines = config.var('MACHINES')
# Sanity check
if not config.var('EXTERNAL_WEB_IP'):
logging.error('config file is missing EXTERNAL_WEB_IP, probably corrupt')
raise Exception, 'config file is missing EXTERNAL_WEB_IP, probably corrupt'
if config.var('USE_DHCP') is None:
logging.error('config file is missing USE_DHCP, probably corrupt')
raise Exception, 'config file is missing USE_DHCP, probably corrupt'
if config.var('DNS_DHCP') is None:
logging.error('config file is missing DNS_DHCP, probably corrupt')
raise Exception, 'config file is missing DNS_DHCP, probably corrupt'
tries = 3 # default for CLUSTER and PILE configuration
if configType in ["SUPER", "ONEBOX", "MINI", "LITE", "FULL"]:
tries = 1
# reconfigure eth0 to dhcp mode
if config.var('USE_DHCP') == 1:
ret1 = ExecuteWrapper(
machines,
"%s LOCALNET DHCP %s" % (
C.RECONFIGURE_COMMAND % ent_home,
config.var('ONEBOX_NETCARD_SETTINGS'),
), None, 600, num_tries = tries)
logging.info("reconfigure eth0 IP to dhcp mode: %s" % ret1)
else:
ret1 = ExecuteWrapper(
machines,
"%s LOCALNET %s %s %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
config.var('EXTERNAL_WEB_IP'),
config.var('EXTERNAL_NETMASK'),
config.var('EXTERNAL_DEFAULT_ROUTE'),
config.var('ONEBOX_NETCARD_SETTINGS'),
), None, 600, num_tries = tries)
logging.info("reconfigure eth0 IP address: %s" % ret1)
elif "CLUSTER" == configType:
# reconfigure serveriron IP address
cmd = "(sleep 5; %s SERVERIRON %s %s %s %s %s %s %s %s %s >&/dev/null " \
"</dev/null)" % (
C.RECONFIGURE_COMMAND % ent_home,
"%s/local/conf/defaults/" % ent_home,
commands.mkarg(config.var("EXTERNAL_SWITCH_IP")),
commands.mkarg(config.var("EXTERNAL_WEB_IP")),
commands.mkarg(config.var("EXTERNAL_CRAWL_IP")),
commands.mkarg(config.var("EXTERNAL_NETMASK")),
commands.mkarg(config.var("EXTERNAL_DEFAULT_ROUTE")),
commands.mkarg(str(config.var("EXTERNAL_WEB_PORT"))),
commands.mkarg(str(config.var("SERVERIRON_AUTONEGOTIATION"))),
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))),
)
t = threading.Thread(target=E.execute,
args=([E.getCrtHostName()], cmd, None, 60))
t.start()
logging.info("serveriron IP address reconfigured")
# we don't want to change ANYTHING on PILE clusters
if "PILE" != configType:
# DNS needs to be set everywhere
dns_server = "\"\""
dns_searchpath = "\"\""
if config.var('BOT_DNS_SERVERS') != None:
dns_server = config.var('BOT_DNS_SERVERS')
if config.var('DNS_SEARCH_PATH') != None:
dns_searchpath = config.var('DNS_SEARCH_PATH')
if config.var('DNS_DHCP') != 1:
ret8 = ExecuteWrapper(
machines,
"%s DNSMODE STATIC" % (
C.RECONFIGURE_COMMAND % ent_home
), None, 600, num_tries = tries)
logging.info("setting DNS mode to STATIC: %s" % ret8)
ret2 = ExecuteWrapper(
machines,
"%s DNS %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
commands.mkarg(dns_server),
commands.mkarg(dns_searchpath),
), None, 600, num_tries = tries)
logging.info("reconfigure DNS: %s" % ret2)
else:
ret8 = ExecuteWrapper(
machines,
"%s DNSMODE DHCP" % (
C.RECONFIGURE_COMMAND % ent_home
), None, 600, num_tries = tries)
logging.info("setting DNS mode to DHCP: %s" % ret8)
# NTP is special: all machines but the master must set their
# NTP server to the master, the master to the external one.
# However, It can take 3 minutes for the stratum level of the master
# node to be set. Before that, non-master nodes using the master
# node to do "ntpdate" may return "no server suitable for synchronization
# found" error.
# To fix the problem, we just use the external ntp servers for all nodes.
# Later, periodic script will set the non-master nodes to use the master
# node. (periodic script will only set it once as long as master does
# not switch)
ntpServers = "\"\""
if config.var('NTP_SERVERS') != None:
ntp_server_list = copy.copy(config.var('NTP_SERVERS'))
AddDefaultNTPServer(ntp_server_list)
ntpServers = string.join(ntp_server_list, ",")
if i_am_master:
ret4 = ExecuteWrapper(
machines,
"%s NTP %d %s" % (
C.RECONFIGURE_COMMAND % ent_home,
force_ntp_reconfig,
commands.mkarg(ntpServers)),
None, 600, num_tries = tries)
logging.info("reconfigure NTP: %s" % ret4)
# whenever we print dates, we want to include the timezone. Since
# the timezone may change on the machine (through config interface),
# and java VM does not pick this change up automatically,
# we keep track of the timezone here
ret5 = ExecuteWrapper(
machines,
"%s TIMEZONE %s" % (
C.RECONFIGURE_COMMAND % ent_home,
commands.mkarg(config.var('TIMEZONE'))),
None, 600, num_tries = tries)
logging.info("reconfigure TIMEZONE: %s" % ret5)
# iptables
if configType in ["SUPER", "ONEBOX", "LITE", "FULL"]:
iptables_file = "%s/local/conf/defaults/iptables.onebox" % ent_home
elif "MINI" == configType:
iptables_file = "%s/local/conf/defaults/iptables.mini" % ent_home
elif "CLUSTER" == configType:
iptables_file = "%s/local/conf/defaults/iptables.cluster" % ent_home
else:
iptables_file = "%s/local/conf/defaults/iptables.open" % ent_home
ret6 = ExecuteWrapper(
machines,
"%s IPTABLES %s %s %s" % (
C.RECONFIGURE_COMMAND % ent_home,
iptables_file,
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))),
commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_BORGMON"))),
), None, 600, num_tries = tries)
logging.info("reconfigure IPTABLES: %s" % ret6)
# return value from each reconfigurenet call is:
# 0 : no change
# 1 : changed
# 2+: error (but we report no errors)
# _our_ return value is if any of them failed
ret = (ret1 <= 1 and ret2 <= 1 and ret4 <= 1 and
ret5 <= 1 and ret6 <= 1 and ret8 <= 1)
return ret
def get_package_status(package, avail=False):
if not avail and package in statuscache:
return statuscache[package]
versionre = re.compile('Version: ')
packagere = re.compile('Package: ')
priorityre = re.compile('Priority: ')
dependsre = re.compile('(Pre-)?Depends: ')
recsre = re.compile('Recommends: ')
suggestsre = re.compile('Suggests: ')
conffilesre = re.compile('Conffiles:')
maintre = re.compile('Maintainer: ')
statusre = re.compile('Status: ')
originre = re.compile('Origin: ')
bugsre = re.compile('Bugs: ')
descre = re.compile('Description: ')
fullre = re.compile(' ')
srcre = re.compile('Source: ')
sectionre = re.compile('Section: ')
pkgversion = pkgavail = maintainer = status = origin = None
bugs = vendor = priority = desc = src_name = section = None
conffiles = []
fulldesc = []
depends = []
recommends = []
suggests = []
confmode = False
state = ''
try:
x = os.getcwd()
except OSError:
os.chdir('/')
packarg = commands.mkarg(package)
if avail:
output = commands.getoutput(
"COLUMNS=79 dpkg --print-avail %s 2>/dev/null" % packarg)
else:
output = commands.getoutput("COLUMNS=79 dpkg --status %s 2>/dev/null" %
packarg)
# dpkg output is in UTF-8 format
output = output.decode('utf-8', 'replace')
for line in output.split(os.linesep):
line = line.rstrip()
if not line: continue
if confmode:
if line[:2] != ' /':
confmode = False
else:
conffiles = conffiles + [tuple(line.split()[:2])]
if versionre.match(line):
(crud, pkgversion) = line.split(": ", 1)
elif statusre.match(line):
(crud, status) = line.split(": ", 1)
elif priorityre.match(line):
(crud, priority) = line.split(": ", 1)
elif packagere.match(line):
(crud, pkgavail) = line.split(": ", 1)
elif originre.match(line):
(crud, origin) = line.split(": ", 1)
elif bugsre.match(line):
(crud, bugs) = line.split(": ", 1)
elif descre.match(line):
(crud, desc) = line.split(": ", 1)
elif dependsre.match(line):
(crud, thisdepends) = line.split(": ", 1)
# Remove versioning crud
thisdepends = [[y.split()[0] for y in x.split('|')]
for x in (thisdepends.split(', '))]
depends.extend(thisdepends)
elif recsre.match(line):
(crud, thisdepends) = line.split(": ", 1)
# Remove versioning crud
thisdepends = [[y.split()[0] for y in x.split('|')]
for x in (thisdepends.split(', '))]
recommends.extend(thisdepends)
elif suggestsre.match(line):
(crud, thisdepends) = line.split(": ", 1)
# Remove versioning crud
thisdepends = [[y.split()[0] for y in x.split('|')]
for x in (thisdepends.split(', '))]
suggests.extend(thisdepends)
elif conffilesre.match(line):
confmode = True
elif maintre.match(line):
crud, maintainer = line.split(": ", 1)
elif srcre.match(line):
crud, src_name = line.split(": ", 1)
src_name = src_name.split()[0]
elif sectionre.match(line):
crud, section = line.split(": ", 1)
elif desc and line[0] == ' ':
fulldesc.append(line)
installed = False
if status:
state = status.split()[2]
installed = (state not in ('config-files', 'not-installed'))
reportinfo = None
if bugs:
reportinfo = debbugs.parse_bts_url(bugs)
elif origin:
if debbugs.SYSTEMS.has_key(origin):
vendor = debbugs.SYSTEMS[origin]['name']
reportinfo = (debbugs.SYSTEMS[origin].get('type', 'debbugs'),
debbugs.SYSTEMS[origin]['btsroot'])
else:
vendor = origin.capitalize()
else:
vendor = ''
info = (pkgversion, pkgavail, tuple(depends), tuple(recommends),
tuple(conffiles), maintainer, installed, origin,
vendor, reportinfo, priority, desc, src_name,
os.linesep.join(fulldesc), state, tuple(suggests), section)
if not avail:
statuscache[package] = info
return info
def escape(str):
"""
Shell escape the given string
"""
return commands.mkarg(str)[1:]
def SyncRequestDirs(self):
sync_problem = []
if self._peeks:
logging.info("Peeking commands from other machines")
self._sync_cycle = self._sync_cycle + 1
err = 0
if self._sync_cycle > PEEK_CYCLE_FREQUENCY:
# We sync one by one to avoid confilicts
for machine, dir in self._peeks.items():
# First get a directory listing of all files waiting for us
if machine == 'None':
ls_cmd = self._fileutil_cmd % ("ls %s" % dir)
_, remote_files = commands.getstatusoutput(ls_cmd)
status = 0 # in fileutil case we are OK since nofiles == error
logging.info("Executing %s" % ls_cmd)
else:
remote_cmd = "ls -1 %s/*" % dir
logging.info("Executing %s on %s" %
(remote_cmd, machine))
(status, sig,
remote_files) = prodlib.RunAlarmRemoteCmd(
machine, remote_cmd, 60)
# endif
if not remote_files:
continue
# endif
if status:
logging.error("Error executing [%s] "
"status: %s / sig: %s / output %s" %
(remote_cmd, status, sig, remote_files))
err = 1
sync_problem.append(machine)
continue
# endif
# We have a list of remote files that need to be copied
# first copy them, then upon success, blow them away remotely
cmds = []
remote_files = string.split(remote_files, '\n')
for file in remote_files:
# We can assume that the files returned by ls are unique
# index the commands by the filename, so we know
# exactly which files to blow away later
if machine == 'None':
cmds.append(
(file, self._fileutil_cmd %
("cp %s %s%s" % (file, self._request_dir,
os.path.basename(file)))))
logging.info("Copying file %s" % (file))
else:
cmds.append((file, "scp %s:%s %s" %
(machine, file, self._request_dir)))
logging.info("Copying files from %s" % (machine))
# endif
# endfor
# We now have a list of commands, execute them in parallel
if machine == 'None':
results = prodlib.ForkCommands(
cmds,
timeout=60,
numslots=20,
machname=lambda x, mach=machine: 'localhost')
else:
results = prodlib.ForkCommands(
cmds,
timeout=60,
numslots=20,
machname=lambda x, mach=machine: mach)
# endif
# Maintain a list of files that were successfully copied
success_files = []
for (file, file_results) in results.items():
(status, output, errors) = file_results
if status:
logging.error(
"Error copying [%s] "
"status: %s / errors: %s / output %s" %
(file, status, errors, output))
# Avoid adding this machine too many times
if not err:
err = 1
sync_problem.append(machine)
else:
# Verify that the file exists locally
filename = os.path.basename(file)
dest_filename = '%s/%s' % (self._request_dir,
filename)
if (os.path.exists(dest_filename)
and os.stat(dest_filename)[stat.ST_SIZE]):
success_files.append(file)
else:
if os.path.exists(dest_filename):
os.unlink(
dest_filename) # remove empty files..
# endif
# Avoid adding this machine too many times
if not err:
err = 1
sync_problem.append(machine)
# endif
# endif
# endfor
# Now, blow away all the files that were successfully copied
if machine == 'None':
command = self._fileutil_cmd % (
"rm -f %s" % string.join(success_files))
status, output = commands.getstatusoutput(command)
sig = 0
else:
command = "rm -f %s" % string.join(success_files)
logging.info("Executing %s on %s" % (command, machine))
(status, sig, output) = prodlib.RunAlarmRemoteCmd(
machine, command, 60)
# endif
if status:
logging.error("Error executing [%s] "
"status: %s / sig: %s / output %s" %
(command, status, sig, output))
if not err:
err = 1
sync_problem.append(machine)
# endif
# endif
# endfor
# Reset the cycle if no error occured
if not err:
self._sync_cycle = 0
# endif
# endif
# endif
# Replicate the working and request dirs to all machines
if self._replication_machines:
# It is unsafe to execute prodlib.ForkExecPool from here because
# we get messed up on Popen3 / select, instead we do a googlesh
err = 0
for f in [self._working_dir, self._request_dir]:
if os.system(
". /etc/profile ; googlesh -m'%s' "
"-p10 -a120 -d0 -q -b %s" %
(string.join(self._replication_machines, " "),
commands.mkarg(
"rsync --delete -e \"ssh -c none\" -aH %s:%s %s" %
(self._local_machine, f, f)))):
err = 1
# endif
# endfor
if err:
sync_problem.extend(self._replication_machines)
# endif
# endif
# Update the status for this guy
self._sync_problems = repr(sync_problem)
#!/usr/bin/env python
import commands, os, sys
# Patch out the JMP (&FFFC) reset call EORed with 0x0e in the final file.
# 0x04 is 0xea ^ 0x0e where 0xea is the opcode for NOP.
t = open("temp/Lunar3\xc3").read()
t = t[:0x261e] + "\x04\x04\x04" + t[0x2621:]
open("temp/Lunar3\xc3", "wb").write(t)
os.system("UEFtrans.py " + commands.mkarg(sys.argv[1]) + " insert 3 temp/Lunar3\xc3")
def escape(str):
return commands.mkarg(str)[1:]
def escape(arg):
arg = commands.mkarg(arg)
if arg[0] == ' ':
arg = arg[1:]
return arg
if last != None:
sys.stdout.write("\rEncoded %i of %i (%1.2f seconds)" % (
frame - first, last - first, elapsed))
else:
sys.stdout.write("\rEncoded %i (%1.2f seconds, %1.2f%%)" % (
frame - first, elapsed, percent))
avi.end_list() # movi
frames = frame - first + 1
for p in placeholders:
avi.f.seek(p)
avi.write_int(frames)
avi.close()
print
print "Ways to process the output file:"
print
#print "mencoder %s -o %s -ovc lavc -oac lavc -lavcopts vcodec=mpeg4:acodec=libmp3lame -srate 44100" % (avi_file, avi_file.replace(".avi", ".mp4"))
avi_file = commands.mkarg(avi_file)
wav_file = avi_file.replace(".avi", ".wav")
wav_44100_file = wav_file.replace(".wav", "-44100.wav")
mp4_file = avi_file.replace(".avi", ".mp4")
print "ffmpeg -i %s -map 0:1 %s" % (avi_file, wav_file)
print "sox %s %s rate 44100" % (wav_file, wav_44100_file)
print "ffmpeg -i %s -i %s -map 0:0 -map 1:0 -strict experimental %s" % (avi_file, wav_44100_file, mp4_file)
print
print "ffmpeg -i %s %s" % (avi_file, avi_file.replace(".avi", ".webm"))
sys.exit()
def remote_cmd(machine, cmd, ssh_port=22):
""" runs a command remotely """
return "ssh -p %d -l root %s %s" % (ssh_port, machine, commands.mkarg(cmd))
directories = {}
for path in old_files:
path = os.path.abspath(path)
directory, file_name = os.path.split(path)
directory = directory.split(os.sep)
if directory[:len(current_dir_pieces)] == current_dir_pieces:
pieces = directory[len(current_dir_pieces):]
dir_path = target_dir
for piece in pieces:
dir_path = os.path.join(dir_path, piece)
if not os.path.exists(dir_path):
os.mkdir(dir_path)
path = path[len(current_dir):].lstrip(os.sep)
new_path = path[:-len(suffix)]
diff_path = os.path.join(dir_path,
file_name[:-len(suffix)] + new_suffix)
os.chdir(current_dir)
os.system("diff -u " + commands.mkarg(path) + " " +
commands.mkarg(new_path) + " > " +
commands.mkarg(diff_path))
sys.exit()
