def merge(self, dest_head, source_head, silent): '''Invoke git merge.''' command = 'git merge --no-edit --allow-unrelated-histories --no-stat -m "Merge channel \'%s\'" %s %s' \ % (source_head, mkarg(dest_head), mkarg(source_head)) command = self.silence(command, silent) wait = self.popen2(command, False) wait()
def exec_and_parse_bugscript(handler, bugscript): """Execute and parse the output of the package bugscript, in particular identifying the headers and pseudo-headers blocks, if present""" fh, filename = TempFile() fh.close() rc = os.system('LC_ALL=C %s %s %s' % (handler, commands.mkarg(bugscript), commands.mkarg(filename))) isheaders = False ispseudoheaders = False headers = pseudoheaders = text = '' fp = open(filename) for line in fp.readlines(): # we identify the blocks for headers and pseudo-h if line == '-- BEGIN HEADERS --\n': isheaders = True elif line == '-- END HEADERS --\n': isheaders = False elif line == '-- BEGIN PSEUDOHEADERS --\n': ispseudoheaders = True elif line == '-- END PSEUDOHEADERS --\n': ispseudoheaders = False else: if isheaders: headers += line elif ispseudoheaders: pseudoheaders += line else: text += line fp.close() cleanup_temp_file(filename) text = text.decode('utf-8', 'replace') return (rc, headers, pseudoheaders, text)
def merge(self, dest_head, source_head, silent): '''Invoke git merge.''' command = 'git merge -n "Merge" %s %s' \ % (mkarg(dest_head), mkarg(source_head)) command = self.silence(command, silent) wait = self.popen2(command, False) wait()
def download(self, url, urldata, d): dldir = d.getVar("DL_DIR", True) m = __pattern__.match(url) path = m.group("path") host = m.group("host") port = m.group("port") user = m.group("user") password = m.group("pass") if port: portarg = "-P %s" % port else: portarg = "" if user: fr = user if password: fr += ":%s" % password fr += "@%s" % host else: fr = host fr += ":%s" % path import commands cmd = "scp -B -r %s %s %s/" % (portarg, commands.mkarg(fr), commands.mkarg(dldir)) bb.fetch2.check_network_access(d, cmd, urldata.url) runfetchcmd(cmd, d)
def download(self, ud, d): """Fetch urls""" urlo = URI(ud.url) basecmd = "sftp -oPasswordAuthentication=no" port = "" if urlo.port: port = "-P %d" % urlo.port urlo.port = None dldir = data.getVar("DL_DIR", d, True) lpath = os.path.join(dldir, ud.localfile) user = "" if urlo.userinfo: user = urlo.userinfo + "@" path = urlo.path # Supoprt URIs relative to the user's home directory, with # the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>). if path[:3] == "/~/": path = path[3:] remote = "%s%s:%s" % (user, urlo.hostname, path) cmd = "%s %s %s %s" % (basecmd, port, commands.mkarg(remote), commands.mkarg(lpath)) bb.fetch2.check_network_access(d, cmd, ud.url) runfetchcmd(cmd, d) return True
def download(self, urldata, d): dldir = d.getVar('DL_DIR', True) m = __pattern__.match(urldata.url) path = m.group('path') host = m.group('host') port = m.group('port') user = m.group('user') password = m.group('pass') if port: portarg = '-P %s' % port else: portarg = '' if user: fr = user if password: fr += ':%s' % password fr += '@%s' % host else: fr = host fr += ':%s' % path import commands cmd = 'scp -B -r %s %s %s/' % ( portarg, commands.mkarg(fr), commands.mkarg(dldir) ) bb.fetch2.check_network_access(d, cmd, urldata.url) runfetchcmd(cmd, d)
def unpack(boot_img, output): """ Unpack the boot image into the output directory. """ Utils.format(boot_img) tool_kit = Toolkit() # Try unpack tool set to find the suitable one boot_type = tool_kit.parse_boot_img_type(boot_img) # Check whether the tools exists if boot_type is None: raise ValueError("Unknown boot image type: " + boot_img) # Execute the unpack command unpack_tool = tool_kit.get_tools(boot_type, "UNPACK") cmd = "%s %s %s" % (commands.mkarg(unpack_tool), commands.mkarg(boot_img), commands.mkarg(output)) (status, result) = commands.getstatusoutput(cmd) if status != 0: print "\n* Unpack failed" print result else: print "\n* Unpack %s %s --> %s" % (boot_type, boot_img, output) # Store the used tools to output Utils.write_boot_img_type(boot_type, output)
def new_branch(self, dest_head, source_head, silent): '''Make a brand new branch.''' command = 'git branch %s %s' \ % (mkarg(dest_head), mkarg(source_head)) command = self.silence(command, silent) wait = self.popen2(command, False) wait()
def run(self): i = self.n while i < len(self.jobs): (cfg, gwssers, site, testwords, epochs, num) = self.jobs[i] i = i + NUM_THREADS # do the tests on all gwssers - do 2 tries, 15 seconds apart max_epoch_site = -1 for (gws, port) in gwssers: cmd = ". %s; cd %s/local/google3/enterprise/legacy/checks && "\ "./gws_production_check.py %s %d %s %s %s %d" % ( cfg.getGlobalParam('ENTERPRISE_BASHRC'), cfg.entHome, commands.mkarg(gws), port, commands.mkarg(site), commands.mkarg(testwords), commands.mkarg(string.join(map(str, epochs), ",")), num) logging.info("Executing %s" % cmd) (err, msgs) = E.getstatusoutput(cmd) max_epoch = None; errors = None exec("(max_epoch, errors) = %s" % msgs) if max_epoch > max_epoch_site: max_epoch_site = max_epoch if errors: self.errors[site] = errors self.max_epochs[site] = max_epoch_site os.remove(testwords)
def nslookup(name, dnsserver='', prevent_unqualified_dns=True): """check whether DNS servers are up to snuff""" # if it looks like an IP address, don't try to resolve it if digitsAndDots.match(name): return (0, "OK") if name != "localhost" and prevent_unqualified_dns: name = name + "." # prevent unqualified DNS lookups # TODO: we really want to call something along the lines of # google2/io/safe_gethostbyname, this will require some python trickery. # If dnsserver is an empty string, then mkarg() will escape it with # quotes and the host call will try to use "''" as a dns server and fail # So call mkarg only if actually given a non-empty-string dnsserver if not dnsserver: dnsserver = '' if dnsserver != '': dnsserver = commands.mkarg(dnsserver) executestring = commands.mkarg( "host -t a %s %s 2>/dev/null | grep has\ address | wc -l" % (commands.mkarg(name), dnsserver)) (stat, out) = commands.getstatusoutput('alarm 5 sh -c %s' % executestring) if stat != 0: return (1, "TIMEOUT") # E.g. DNS server does not respond if int(out) == 0: return (2, "cannot resolve") return (0, "OK")
def download(self, ud, d): """Fetch urls""" urlo = URI(ud.url) basecmd = 'sftp -oBatchMode=yes' port = '' if urlo.port: port = '-P %d' % urlo.port urlo.port = None dldir = data.getVar('DL_DIR', d, True) lpath = os.path.join(dldir, ud.localfile) user = '' if urlo.userinfo: user = urlo.userinfo + '@' path = urlo.path # Supoprt URIs relative to the user's home directory, with # the tilde syntax. (E.g. <sftp://example.com/~/foo.diff>). if path[:3] == '/~/': path = path[3:] remote = '%s%s:%s' % (user, urlo.hostname, path) cmd = '%s %s %s %s' % (basecmd, port, commands.mkarg(remote), commands.mkarg(lpath)) bb.fetch2.check_network_access(d, cmd, ud.url) runfetchcmd(cmd, d) return True
def GetRemoteFiles(machine, dir): cmd = "ssh %s %s" % ( commands.mkarg(machine), commands.mkarg("ls -1 %s/media_%s_%s.flv" % ( dir, FILE_PARM, FILE_PARM))) (err, out) = commands.getstatusoutput(cmd) if err: sys.exit("Cannot list remotely: %s" % cmd) return out.split('\n')
def fmt_screens(commands, stdin=None): screens = [] for command in commands: screen = "screen -t %s /bin/bash -c %s" % (mkarg(command), mkarg("cat %s | (%s)" % (stdin.path, command) if stdin else command)) screens.append(screen) return screens
def open(url, desktop=None, wait=0): """ Open the 'url' in the current desktop's preferred file browser. If the optional 'desktop' parameter is specified then attempt to use that particular desktop environment's mechanisms to open the 'url' instead of guessing or detecting which environment is being used. Suggested values for 'desktop' are "standard", "KDE", "GNOME", "Mac OS X", "Windows" where "standard" employs a DESKTOP_LAUNCH environment variable to open the specified 'url'. DESKTOP_LAUNCH should be a command, possibly followed by arguments, and must have any special characters shell-escaped. The process identifier of the "opener" (ie. viewer, editor, browser or program) associated with the 'url' is returned by this function. If the process identifier cannot be determined, None is returned. An optional 'wait' parameter is also available for advanced usage and, if 'wait' is set to a true value, this function will wait for the launching mechanism to complete before returning (as opposed to immediately returning as is the default behaviour). """ # Attempt to detect a desktop environment. detected = get_desktop() # Give higher priority to user preference if (desktop is None or desktop == "override") and detected == "override": global override arg = override.replace("%url%", commands.mkarg(url)) return _run(arg, 1, wait) # Desktops whose existence can be easily tested. elif (desktop is None or desktop == "standard") and is_standard(): arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)]) return _run(arg, 1, wait) elif (desktop is None or desktop == "Windows") and detected == "Windows": # NOTE: This returns None in current implementations. try: return os.startfile(url) except OSError: # some broken configs may return error here but work anyway return elif desktop is None: desktop = detected cmd = get_command(desktop, url) if cmd: return _run(cmd, 0, wait) else: # Finish with an error where no suitable desktop was identified. webbrowser.open(url)
def _Diff(lhs, rhs): """Run standard unix 'diff' against two files.""" cmd = '${TEST_DIFF:-diff} %s %s' % (commands.mkarg(lhs), commands.mkarg(rhs)) (status, output) = commands.getstatusoutput(cmd) if os.WIFEXITED(status) and os.WEXITSTATUS(status) == 1: # diff outputs must be the same as c++ and shell raise OutputDifferedError('\nRunning %s\n%s\nTest output differed ' 'from golden file\n' % (cmd, output)) elif not os.WIFEXITED(status) or os.WEXITSTATUS(status) != 0: raise DiffFailureError('\nRunning %s\n%s\nFailure diffing test output ' 'with golden file\n' % (cmd, output))
def thumbnail_image(self, src_path, dest_path, size): """Generate a square thumbnail, with on-disk source and destination paths.""" child = Popen4("%s -geometry %dx%d %s %s" % ( self.CONVERT, size, size, commands.mkarg(src_path), commands.mkarg(dest_path), )) output = child.fromchild.read() status = child.wait() if status: raise ImageException(output)
def sign_message(body, fromaddr, package='x', pgp_addr=None, sign='gpg', draftpath=None): '''Sign message with pgp key.''' ''' Return: a signed body. On failure, return None. kw need to have the following keys ''' if not pgp_addr: pgp_addr = get_email_addr(fromaddr)[1] # Make the unsigned file first (unsigned, file1) = TempFile(prefix=tempfile_prefix(package, 'unsigned'), dir=draftpath) unsigned.write(body) unsigned.close() # Now make the signed file (signed, file2) = TempFile(prefix=tempfile_prefix(package, 'signed'), dir=draftpath) signed.close() if sign == 'gpg': os.unlink(file2) if 'GPG_AGENT_INFO' not in os.environ: signcmd = "gpg --local-user '%s' --clearsign " % pgp_addr else: signcmd = "gpg --local-user '%s' --use-agent --clearsign " % pgp_addr signcmd += '--output '+commands.mkarg(file2)+ ' ' + commands.mkarg(file1) else: signcmd = "pgp -u '%s' -fast" % pgp_addr signcmd += '<'+commands.mkarg(file1)+' >'+commands.mkarg(file2) try: os.system(signcmd) x = file(file2, 'r') signedbody = x.read() x.close() if os.path.exists(file1): os.unlink(file1) if os.path.exists(file2): os.unlink(file2) if not signedbody: raise NoMessage body = signedbody except (NoMessage, IOError, OSError): fh, tmpfile2 = TempFile(prefix=tempfile_prefix(package), dir=draftpath) fh.write(body) fh.close() ewrite('gpg/pgp failed; input file in %s\n', tmpfile2) body = None return body
def doLogReport(self, jobName, jobToken, collection, reportName, reportDate, withResults, topCount, diagnosticTerms, update): """The actual work done in a worker thread to generate a summary report.""" (html_file, valid_file) = liblog.get_report_filenames(self.entConfig, liblog.SUMMARY_REPORT, reportName, collection) liblog.MakeGoogleDir(self.entConfig, os.path.dirname(html_file)) new_html_file = tempfile.mktemp('.report') new_valid_file = tempfile.mktemp('.report_valid') args = [] args.append(commands.mkarg(self.entConfig.GetEntHome())) args.append(commands.mkarg(collection)) args.append(commands.mkarg(reportDate)) args.append(withResults) args.append(topCount) args.append(commands.mkarg(diagnosticTerms)) args.append(commands.mkarg(html_file)) args.append(commands.mkarg(valid_file)) args.append(commands.mkarg(new_html_file)) args.append(commands.mkarg(new_valid_file)) cmd = ('. %s && cd %s/enterprise/legacy/logs && ' 'alarm %s nice -n 15 ./log_report.py %s' % (self.cfg.getGlobalParam('ENTERPRISE_BASHRC'), self.cfg.getGlobalParam('MAIN_GOOGLE3_DIR'), COMMAND_TIMEOUT_PERIOD, string.join(args, ' '))) logging.info('doLogReport(): CMD = %s' % cmd) returnCode = E.system(cmd) self.handleResult(jobName, jobToken, returnCode, liblog.SUMMARY_REPORT, collection, reportName, update, html_file, valid_file, new_html_file, new_valid_file)
def do_edit(given_cl, current_cl, cl_file_path): if given_cl.is_unspecified(): # Show an editor if CL not specified on the command-line tmp_fd, tmp_path = tempfile.mkstemp(prefix='appspot-', suffix='.txt') os.write(tmp_fd, editable_change(current_cl)) os.close(tmp_fd) retcode = subprocess.call( '%s %s' % (os.getenv('VISUAL', os.getenv('EDITOR', 'vi')), commands.mkarg(tmp_path)), shell=True) try: if retcode < 0: raise Exception('editor closed with signal %s' % -retcode) elif retcode > 0: raise Exception('editor exited with error value %s' % retcode) edited_cl = parse_change(open(tmp_path).read()) finally: os.remove(tmp_path) if edited_cl.is_unspecified(): print >>sys.stderr, 'cancelled edit' return edited_cl.merge_into(current_cl) else: given_cl.merge_into(current_cl) out = open(cl_file_path, 'w') out.write(editable_change(current_cl)) out.close()
def get_changed_config_files(conffiles, nocompress=False): confinfo = {} changed = [] for (filename, md5sum) in conffiles: try: fp = file(filename) except IOError, msg: confinfo[filename] = msg continue filemd5 = commands.getoutput("md5sum " + commands.mkarg(filename)).split()[0] if filemd5 == md5sum: continue changed.append(filename) thisinfo = "changed:\n" for line in fp: if not line: continue if line == "\n" and not nocompress: continue if line[0] == "#" and not nocompress: continue thisinfo += line confinfo[filename] = thisinfo.decode("utf-8", "replace")
def Submit(executable=None, runHours=3, runMinutes=0, jobname="pyprop", printScript=False, **args): """ Set up a job script suitable """ #Create jobscript jscript = submitpbs.SubmitScript() jscript.jobname = jobname jscript.walltime = timedelta(hours=runHours, minutes=runMinutes, seconds=0) jscript.ppn = args.get("ppn", 4) jscript.proc_memory = args.get("proc_memory", "1000mb") jscript.nodes = args.get("nodes", "1") numProcs = args.get("numProcs", jscript.ppn*jscript.nodes) if installation == "stallo": jscript.interconnect = args.get("interconnect", "ib") jscript.workingdir = args.get("workingDir", os.path.abspath(os.curdir) ) jscript.executable = "mpirun -n %s " % (jscript.ppn*jscript.nodes) jscript.executable += "python %s" % executable elif installation == "hexagon": jscript.workingdir = args.get("workingDir", os.path.abspath(os.curdir) ) jscript.executable = "aprun -n %s -N %s " % (numProcs, jscript.ppn) jscript.executable += "./python-exec %s" % executable jscript.parameters = commands.mkarg(repr(args)) #Submit this job if printScript: print "\n".join(jscript.CreateScript()) else: jscript.Submit()
def send(self, filename): elisp = """(progn (load-file "/usr/share/reportbug/reportbug.el") (tfheen-reportbug-insert-template "%s"))""" filename = re.sub('["\\\\]', "\\\\\\g<0>", filename) elisp = commands.mkarg(elisp % filename) return ui.system("emacsclient --no-wait --eval %s 2>/dev/null" " || emacs --eval %s" % (elisp, elisp))
def RunAlarmRemoteCmd(mach, cmd, timeout, remote_user=None): if timeout < 0: # alarm disabled alarm_cmd = "" else: alarm_cmd = "alarm %s " % timeout # prepend AM user if remote_user: remote_string = "%s@" % remote_user else: remote_string = "" entire_cmd = (". /etc/profile; %s ssh -q -P -n -o 'BatchMode=yes' " "-o 'ForwardAgent=yes' -o 'StrictHostKeyChecking=no' %s%s %s" % (alarm_cmd, remote_string, mach, commands.mkarg(cmd))) if verbose: log("Running %s " % entire_cmd) try: (status, output) = commands.getstatusoutput(entire_cmd) if status: (status, sig) = (status >> 8, status & 255) # separate high (status) # and low byte (signal) else: (status, sig) = (0, 0) except OSError, e: (status, sig, output) = (e.errno, 0, '')
def klipper(args): import os, commands args = " ".join([commands.mkarg(str(arg)) for arg in args]) s, o = commands.getstatusoutput("qdbus org.kde.klipper /klipper " + args) if s == 0: return o return ""
def SubmitDelayScan(**args): delayList = args["delayList"] outputfile = args["outputfile"] molecule = args["molecule"] partitionCount = args["partitionCount"] partitionSize = int(ceil(len(delayList) / float(partitionCount))) partitionStep = delayList[1] - delayList[0] # Assume equidistant delayList plist = [] for i in range(partitionCount): start = min(i * partitionSize, len(delayList) - 1) end = min((i + 1) * partitionSize, len(delayList)) delaySlice = slice(delayList[start], delayList[end - 1] + partitionStep, partitionStep) args["delayList"] = delaySlice args["outputfile"] = outputfile % i script = submitpbs.SubmitScript() script.interconnect = None script.executable = "./run_delay_scan.py" script.parameters = commands.mkarg(repr(args)) script.ppn = 1 script.nodes = 1 script.walltime = submitpbs.timedelta(hours=6) print "\n".join(script.CreateScript()) script.Submit()
def access(machines, file, access): """ Test the access mode for a file/directory""" # This is a small hack to increase the spead when checking on # local machine (in interface happens a lot) if (len(machines) == 1 and ( LOCALHOST == machines[0] or machines[0] == getCrtHostName() )): try: a = os.F_OK if 'r' in access: a = a | os.R_OK if 'w' in access: a = a | os.W_OK if 'x' in access: a = a | os.X_OK if not os.access(file, a): return 0 s = os.stat(file) if 'd' in access and not stat.S_ISDIR(s[stat.ST_MODE]): return 0 if 'f' in access and not (stat.S_ISREG(s[stat.ST_MODE])): return 0 return 1 except OSError: return 0 cmd = [] for a in access: cmd.append(ACCESS % (a, commands.mkarg(file))) return ERR_OK == execute(machines, string.join(cmd), None, true)
def start_service(self, service): """ Activates and starts the given service on the given machine """ logging.info("ACTIVATE: %s service %s on %s" % ( service, self.version_, self.machine_)) cmd = E.nonblocking_cmd( "/etc/rc.d/init.d/%s_%s activate" % (service, self.version_)) ret = E.exe("ssh %s %s" % (self.machine_, commands.mkarg(cmd))) if ret == 0: logging.info("START: %s service %s on %s" % ( service, self.version_, self.machine_)) cmd = E.nonblocking_cmd( "/etc/rc.d/init.d/%s_%s start" % (service, self.version_)) ret = E.exe("ssh %s %s" % (self.machine_, commands.mkarg(cmd))) return ret
def direct_fetch(self, url, upstream_name): '''Use git directly to do a pull.''' # for now we assume master, as HEAD is inaccessible. too bad. refspec = '+master:%s' % (upstream_name) command = 'git fetch %s %s' % (mkarg(url), refspec) wait = self.popen2(command, False) wait()
def CronTabEntry(config, timespec, cmds, log_file, source_env): cmd = "(" + string.join(cmds, '; ') + ") >& " + log_file script_dir = config.var('MAIN_GOOGLE3_DIR') + "/enterprise/legacy/util" str = "( %s; cd %s && ./run_cron_command.py %s ) &> /dev/null" % ( source_env, script_dir, commands.mkarg(cmd)) str = "\n\n" + timespec + '\t ' + str + '\n' return str
def open(url, desktop=None, wait=0): """ Open the 'url' in the current desktop's preferred file browser. If the optional 'desktop' parameter is specified then attempt to use that particular desktop environment's mechanisms to open the 'url' instead of guessing or detecting which environment is being used. Suggested values for 'desktop' are "standard", "KDE", "GNOME", "XFCE", "Mac OS X", "Windows" where "standard" employs a DESKTOP_LAUNCH environment variable to open the specified 'url'. DESKTOP_LAUNCH should be a command, possibly followed by arguments, and must have any special characters shell-escaped. The process identifier of the "opener" (ie. viewer, editor, browser or program) associated with the 'url' is returned by this function. If the process identifier cannot be determined, None is returned. An optional 'wait' parameter is also available for advanced usage and, if 'wait' is set to a true value, this function will wait for the launching mechanism to complete before returning (as opposed to immediately returning as is the default behaviour). """ # Decide on the desktop environment in use. desktop_in_use = use_desktop(desktop) if desktop_in_use == "standard": arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)]) return _run(arg, 1, wait) elif desktop_in_use == "Windows": # NOTE: This returns None in current implementations. return os.startfile(url) elif desktop_in_use == "KDE4": cmd = ["kioclient", "exec", url] elif desktop_in_use == "KDE": cmd = ["kfmclient", "exec", url] elif desktop_in_use == "GNOME": cmd = ["gnome-open", url] elif desktop_in_use == "XFCE": cmd = ["exo-open", url] elif desktop_in_use == "Mac OS X": cmd = ["open", url] elif desktop_in_use == "X11" and os.environ.has_key("BROWSER"): cmd = [os.environ["BROWSER"], url] # Finish with an error where no suitable desktop was identified. else: raise OSError, "Desktop '%s' not supported (neither DESKTOP_LAUNCH nor os.startfile could be used)" % desktop_in_use return _run(cmd, 0, wait)
def format_cmdline(args): return ' '.join( commands.mkarg(arg).strip() if any(s in arg for s in (" '\"$")) else arg for arg in args)
def decryptLicense(self, encrypted_license): tempdir = E.mktemp(self.working_dir, 'decrypt') signed_fn = E.joinpaths([tempdir, "license.signed"]) verified_fn = E.joinpaths([tempdir, "license.verified"]) encrypted_fn = E.joinpaths([tempdir, "license.encrypted"]) tgz_fn = E.joinpaths([tempdir, "license.tgz"]) license_fn = E.joinpaths([tempdir, "ent_license.config"]) if not self.checkFiles(): logging.error("decryptor keys don't exist or aren't valid") return None, None, DECRYPT_SYSTEM_ERROR try: # make the temp directory os.system("rm -rf %s" % commands.mkarg(tempdir)) os.makedirs(tempdir) # output to a file try: open(signed_fn, 'w').write(encrypted_license) except IOError: logging.error("writing encrypted license failed") return None, None, DECRYPT_SYSTEM_ERROR # verify signature verify_cmd = ("gpg --no-options --no-default-keyring --no-tty" " --no-verbose --yes --keyring %s -o %s --decrypt %s" ) % (commands.mkarg(self.license_public_keyring), commands.mkarg(verified_fn), commands.mkarg(signed_fn)) # When gpg is given a file with no signature it returns successfully; # so we must check gpg's output fi, foe = os.popen4(verify_cmd) fi.close() gpg_result = foe.read() foe.close() if gpg_result.find("Good signature") == -1: logging.error("verifying license signature failed") return None, None, DECRYPT_GPG_VERIFY_FAILED # remove header verified_body = open(verified_fn, 'r').read() sep_poi = string.find(verified_body, "\n\n") if sep_poi < 0: logging.error("license doesn't have valid header") return None, None, DECRYPT_BAD_HEADER # NOTE: At this point, we've read the header, which is a subset of the # actual license (which we can't see yet, because it's encrypted). If # we fail past this point, we return the header in place of the license # because it will look like an invalid license, and allow us to report # some basic information about the license. # header_body = verified_body[0:sep_poi] # write out the encrypted body try: open(encrypted_fn, 'w').write(verified_body[sep_poi + 2:]) except IOError: logging.error("writing encrypted license failed") return None, header_body, DECRYPT_SYSTEM_ERROR # decrypt it decrypt_cmd = "gpg --no-options --no-default-keyring --no-tty --yes --keyring %s --secret-keyring %s -o %s --decrypt %s" % ( commands.mkarg(self.box_public_keyring), commands.mkarg(self.box_private_keyring), commands.mkarg(tgz_fn), commands.mkarg(encrypted_fn)) if os.system(decrypt_cmd) != 0: logging.error("decrypting license failed") return None, header_body, DECRYPT_GPG_DECRYPT_FAILED # untar it (we only ask for the file we want) untar_cmd = "tar xvzf %s -C %s ent_license.config" % ( commands.mkarg(tgz_fn), tempdir) if os.system(untar_cmd) != 0: logging.error("untarring license failed") return None, header_body, DECRYPT_TAR_FAILED # read in the decrypted license try: license_body = open(license_fn, 'r').read() except IOError: logging.error("reading decrypted license failed") return None, header_body, DECRYPT_SYSTEM_ERROR finally: # make sure to clean up all the temp files os.system("rm -rf %s" % commands.mkarg(tempdir)) return license_body, header_body, DECRYPT_OK
def send(self, filename): mua = self.command if '%s' not in mua: mua += ' %s' return ui.system(mua % commands.mkarg(filename)[1:])
def run_update_index_cacheinfo(self, mode, blob_id, filename): '''Run git update-index --cacheinfo.''' update_command = 'git update-index --cacheinfo %s %s %s' \ % (mode, blob_id, mkarg(filename)) self.shell_to_string(update_command)
def comment(s): return "echo %s" % commands.mkarg(s)
def run(cmd, file): sts, output = commands.getstatusoutput(cmd + commands.mkarg(file)) if sts: print output print 'Exit status', sts return sts
if __name__ == "__main__": if len(sys.argv) != 4: sys.stderr.write( "Usage: %s <search path> <dimensions> <output directory>\n" % sys.argv[0]) sys.exit(1) search_path = sys.argv[1] dimensions = sys.argv[2] output_dir = sys.argv[3] paths = [] paths += map(lambda line: line.strip(), find_files(search_path, "*.jef")) paths += map(lambda line: line.strip(), find_files(search_path, "*.JEF")) for path in paths: directory, file_name = os.path.split(path) if os.system("python jef2png.py --stitches-only " + dimensions + " " + commands.mkarg(path) + " " + commands.mkarg( os.path.join(output_dir, file_name + ".png"))) == 0: print os.path.join(output_dir, file_name + ".png") else: sys.stderr.write("Failed to convert file: " + path + "\n") sys.exit()
def local_wrapped_cmd(command_line, timeout=DEF_WRAPPED_CMD_TIMEOUT_SEC): """Use Popen3 to execute a command locally. This routine uses the Popen3 class in Python's popen2 module to execute a command locally and retrieve stdout, stderr, *AND* the exit status code, with a timeout to prevent runaway commands. This implementation may seem a bit elaborate, but it is required to prevent deadlock when the command being executed could output to stdout or stderr in any order. Arguments: command_line: command-line string to execute timeout: maximum run-time allowed, in seconds; default is 1 minute Returns: ( [ 'stdout line 0\n', 'stdout line 1\n', ... ], [ 'stderr line 0\n', 'stderr line 1\n', ... ], exit_code, ) """ cmd_str = SHELL_GLOB_WRAPPER_FMT % commands.mkarg(command_line) sub_process = popen2.Popen3(cmd_str, 1) # limit how long to wait for the subprocess to complete signal.signal(signal.SIGALRM, _sigalrm_handler) signal.alarm(timeout) # create empty lists for threads to append their output std_out_lines = [] std_err_lines = [] std_out_thread = threading.Thread(name='LocalWrappedCmdStdOut', target=_wrapped_readlines_thread, kwargs={ 'child_pipe': sub_process.fromchild, 'out_list': std_out_lines, }) std_err_thread = threading.Thread(name='LocalWrappedCmdStdErr', target=_wrapped_readlines_thread, kwargs={ 'child_pipe': sub_process.childerr, 'out_list': std_err_lines, }) # ensure that program will exit even if these threads are running std_out_thread.setDaemon(1) std_err_thread.setDaemon(1) timed_out = 0 try: std_out_thread.start() std_err_thread.start() while std_out_thread.isAlive() or std_err_thread.isAlive(): # wait until both threads are finished (or timeout interrupts them) pass except _TimeoutError: # kill the too-long running child process os.kill(sub_process.pid, signal.SIGTERM) timed_out = 1 signal.alarm(0) # shut off the alarm now exit_code = sub_process.wait() ## NOTE: threads will get an EOF from the pipe and then exit on their own # wait() returns a variety of items, so "decode" the value into an exit code if exit_code == None: exit_code = 0 elif os.WIFEXITED(exit_code): # child process exit codes are multiplied by 256, so undo this exit_code = os.WEXITSTATUS(exit_code) elif timed_out: # process was killed by the timeout exit_code = errno.ETIMEDOUT elif exit_code != 0: # process was killed by some other signal exit_code = errno.EINTR return std_out_lines, std_err_lines, exit_code
def alarm_wrap(cmd, timeout): """wraps the given command in alarm cmd with a specified timeout)""" return 'alarm %s bash -c %s' % (timeout, commands.mkarg(cmd))
def composeCmd(machines, param, alarm, dist, verbose, forceRemote, enthome=None, ignore=0): """ Helper to create googlesh/googldist commands @param machines - comma separated list of machines @param param - the parameter for googleX script(command / file) @param alarm - wrap each command in an alarm (use -a option) @param dist - true -> googledist; false -> googlesh @param enthome - the directory of current version of enterprise. @param ignore - Number of failures to ignore. @return String - the command to be executed """ if enthome == None: enthome = getEnterpriseHome() # I have to remove localhost from the list of machines if is a dist listMachines = [] listMachines.extend(machines) if dist: if LOCALHOST in listMachines: listMachines.remove(LOCALHOST) if getCrtHostName() in listMachines: listMachines.remove(getCrtHostName()) # recompose machines space sepparated strMachines = string.strip(string.join(listMachines)) # Anything left ? if not strMachines: return None if (LOCALHOST == strMachines or strMachines == getCrtHostName()) and not forceRemote: alarmcmd = "" if alarm: if alarm > 1: alarmcmd = LOCAL_ALARM_CMD % alarm else: alarmcmd = LOCAL_ALARM_CMD % DEFAULTALARM cmd = EXEC_HERE % ( enthome, # We need to do this unescaping when running on the same # machine. # NOTE: This may be a problem if param has $ that # is escaped, in which case that argument will be # evaluated! string.replace(param, "\\$", "$")) if alarmcmd: cmd = '%s %s' % (alarmcmd, commands.mkarg(cmd)) else: if alarm: if alarm > 1: aparam = ALARMPARAM % alarm else: aparam = ALARMPARAM % DEFAULTALARM else: aparam = "" if dist: cmd = EXEC_REMOTE_DIST % (enthome, aparam, commands.mkarg(strMachines), ignore, commands.mkarg(param)) else: cmd = EXEC_REMOTE_CMD % (enthome, aparam, commands.mkarg(strMachines), ignore, commands.mkarg(EXEC_HERE % (enthome, param))) if verbose: logging.info("Running : " + cmd) return cmd
def encryptLicense(self, license, passphrase=None): tempdir = E.mktemp(self.working_dir, 'encrypt') # if no passphrase supplied, use the one given in the constructor if passphrase == None: passphrase = self.passphrase # make sure the license is valid (note that this is a fresh license) if license.isInvalid(fresh_license=1): logging.error("license is not valid") return None if not self.checkFiles(): logging.error("encryptor keys don't exist or aren't valid") return None # get the header header = self.extractHeader(license) # NOTE: this ent_license.config is important; the old license code expects # to find this filename. do not change it license_fn = E.joinpaths([tempdir, "ent_license.config"]) tgz_fn = E.joinpaths([tempdir, "license.tgz"]) encrypted_fn = E.joinpaths([tempdir, "license.encrypted"]) encrypted_w_header_fn = E.joinpaths([tempdir, "license.header"]) signed_fn = E.joinpaths([tempdir, "license.signed"]) try: # make the temp directory os.system("rm -rf %s" % commands.mkarg(tempdir)) os.makedirs(tempdir) # write the raw license body to a file license_body = {} license_body.update(license.license) # overwrite the version number with the current version (just in case) license_body['ENT_LICENSE_VERSION'] = CURRENT_LICENSE_VERSION try: body_str = string.join( map(lambda (k, v): '%s = %s' % (k, repr(v)), license_body.items()), '\n') open(license_fn, 'w').write(str(body_str)) except IOError: logging.error("writing plain license failed") return None # tar the license body tar_cmd = "tar zvcf %s -C %s %s" % ( commands.mkarg(tgz_fn), commands.mkarg(os.path.dirname(license_fn)), commands.mkarg(os.path.basename(license_fn))) if os.system(tar_cmd) != 0: logging.error("tarring license failed") return None # encrypt it encrypt_cmd = "gpg --no-options --no-default-keyring --no-secmem-warning --no-tty --yes --default-recipient-self --keyring %s --secret-keyring %s -o %s -se %s" % ( commands.mkarg(self.box_public_keyring), commands.mkarg(self.box_private_keyring), commands.mkarg(encrypted_fn), commands.mkarg(tgz_fn)) if os.system(encrypt_cmd) != 0: logging.error("encrypting license failed") return None # write out the header (plus \n\n) to encrypted_w_header_fn try: header_str = string.join( map(lambda (k, v): '%s = %s' % (k, repr(v)), header.items()), '\n') header_str = header_str + '\n\n' # end of header open(encrypted_w_header_fn, 'w').write(header_str) except IOError: logging.error("writing license header failed") return None # append the encrypted tarred license to encrypted_w_header_fn cat_cmd = "cat %s >> %s" % (commands.mkarg(encrypted_fn), commands.mkarg(encrypted_w_header_fn)) if os.system(cat_cmd) != 0: logging.error("reading encrypted license failed") return None # sign encrypted_w_header_fn sign_cmd = "gpg --no-options --no-default-keyring --no-tty --no-secmem-warning --keyring %s --secret-keyring %s --passphrase-fd 0 --yes -o %s -s %s" % ( commands.mkarg(self.license_public_keyring), commands.mkarg( self.license_private_keyring), commands.mkarg(signed_fn), commands.mkarg(encrypted_w_header_fn)) try: # use popen so that we can write the passphrase to stdin sign_fd = os.popen(sign_cmd, 'w') if passphrase != None: sign_fd.write(passphrase) status = sign_fd.close() except IOError: status = -1 if status != None: logging.error("signing license signature failed") return None # read the signed body try: signed_body = open(signed_fn, 'r').read() except IOError: logging.error("reading signed license failed") return None finally: # make sure to clean up all the temp files os.system("rm -rf %s" % commands.mkarg(tempdir)) return signed_body
def setUserConfig(self, usersToAdd): ''' delete some users and reset the User Config File''' cmd = 'secure_script_wrapper -p2 %s setUserConfig %s' % ( self.snmpWrapperPath, commands.mkarg(usersToAdd)) logMsg = 'SNMP users configuration changed' return self.execCmd(cmd, self.machines, logMsg)
#!/usr/bin/python """ argv_to_sh.py Embed an argv vector into a shell string, e.g. for ssh or sudo. """ import commands import sys # strategy: double quote if it has ' -- otherwise single quote for arg in sys.argv[1:]: sys.stdout.write(commands.mkarg(arg))
def find_files(path, pattern): so, si = popen2.popen2('find ' + commands.mkarg(path) + ' -name "' + pattern + '"') si.close() return so.readlines()
def quote_args(self, files): '''Join files into a single shell quoted string.''' return ''.join([mkarg(f) for f in files])
if G.debug: print 'mode =', mode, 'rate =', rate if mode in ('au', 'aiff', 'wav', 'aifc', 'ul', 'ub', 'sb') and \ rate in validrates: tempname = filename if mode in ('ul', 'ub', 'sb'): raw = 1 elif cache.has_key(filename): tempname = cache[filename] else: tempname = G.tempprefix + ` rand.rand() ` + '.aiff' cmd = SOX if G.debug: cmd = cmd + ' -V' if mode <> '': cmd = cmd + ' -t ' + mode cmd = cmd + ' ' + commands.mkarg(filename) cmd = cmd + ' -t aiff' if rate not in validrates: rate = 32000 if rate: cmd = cmd + ' -r ' + ` rate ` cmd = cmd + ' ' + tempname if G.debug: print cmd sts = os.system(cmd) if sts: print cmd print 'Exit status', sts stdwin.fleep() try: os.unlink(tempname) except:
def run_read_tree(self, tree_id): '''Run git read-tree on the given tree.''' self.shell_to_string('git read-tree %s' % mkarg(tree_id))
def send(cfg, to, problem, subject, msgText, logSubject): # No mails in install mode if cfg.getInstallState() == "INSTALL": return smtpHost = cfg.getGlobalParam("SMTP_SERVER") fromEmail = cfg.getGlobalParam("OUTGOING_EMAIL_SENDER") allSubject = None if not to: if problem: to = cfg.getGlobalParam("PROBLEM_EMAIL") else: to = cfg.getGlobalParam("NOTIFICATION_EMAIL") # if to is empty, we still go through the whole process, but bail out right # before actually sending the email. we want all the side effects to still # happen (logging, testing hooks, etc..) # if "*EMAIL" is None, commands.mkarg() will throw a TypeError exception if not to: to = "" else: to = string.strip(to) if problem: allSubject = "%s: %s" % ( cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get( "ENT_BOX_ID", ""), cfg.getGlobalParam("EMAIL_PROBLEM_PREFIX")) else: allSubject = "%s: %s" % ( cfg.getGlobalParam("ENT_LICENSE_INFORMATION").get( "ENT_BOX_ID", ""), cfg.getGlobalParam("EMAIL_NOTIFICATION_PREFIX")) allSubject = allSubject + subject # Log the subject if logSubject: logging.info('Sending mail: To: %s; Subject: %s' % (repr(to), repr(subject))) cfg.writeAdminRunnerOpMsg(subject) dateString = time.strftime("%Y/%m/%d %H:%M:%S", E.getLocaltime(time.time())) if msgText: msgText = msgText + "\n" else: msgText = "" ipAddr = cfg.getGlobalParam("EXTERNAL_WEB_IP") body = "%s [%s @%s from %s]" % (msgText, M.MSG_MAIL_AUTOGENERATED, dateString, ipAddr) # if we have a testing hook, call that if TEST_MAIL_HANDLER_HOOK != None: logging.info("calling TEST_MAIL_HANDLER_HOOK instead of mailnotify") to_list = filter(None, map(string.strip, string.split(to, ","))) TEST_MAIL_HANDLER_HOOK(smtpHost, to_list, fromEmail, allSubject, body) return # if to is empty, bail out if not (to and smtpHost): return # We execute this and not call the function directly because mailnotify # uses signal and we might call this from a secondary thread. E.execute([E.LOCALHOST], ". %s; cd %s/local/google3/enterprise/legacy/util; "\ "./mailnotify.py %s %s %s %s %s" % ( cfg.getGlobalParam('ENTERPRISE_BASHRC'), E.getEnterpriseHome(), commands.mkarg(smtpHost), commands.mkarg(allSubject), commands.mkarg(fromEmail), commands.mkarg(to), commands.mkarg(body)), None, 0)
def open(url, desktop=None, wait=0, dialog_on_error=False): """ Open the 'url' in the current desktop's preferred file browser. If the optional 'desktop' parameter is specified then attempt to use that particular desktop environment's mechanisms to open the 'url' instead of guessing or detecting which environment is being used. Suggested values for 'desktop' are "standard", "KDE", "GNOME", "XFCE", "Mac OS X", "Windows" where "standard" employs a DESKTOP_LAUNCH environment variable to open the specified 'url'. DESKTOP_LAUNCH should be a command, possibly followed by arguments, and must have any special characters shell-escaped. The process identifier of the "opener" (ie. viewer, editor, browser or program) associated with the 'url' is returned by this function. If the process identifier cannot be determined, None is returned. An optional 'wait' parameter is also available for advanced usage and, if 'wait' is set to a true value, this function will wait for the launching mechanism to complete before returning (as opposed to immediately returning as is the default behaviour). """ # Decide on the desktop environment in use. import bauble.utils as utils desktop_in_use = use_desktop(desktop) cmd = None if desktop_in_use == "standard": arg = "".join([os.environ["DESKTOP_LAUNCH"], commands.mkarg(url)]) return _run(arg, 1, wait) elif desktop_in_use == "Windows": # NOTE: This returns None in current implementations. return os.startfile(url) elif desktop_in_use == "KDE": cmd = ["kfmclient", "exec", url] elif desktop_in_use == "GNOME": cmd = ["gnome-open", url] elif desktop_in_use == "XFCE": cmd = ["exo-open", url] elif desktop_in_use == "Mac OS X": cmd = ["open", url] elif desktop_in_use == "X11" and "BROWSER" in os.environ: cmd = [os.environ["BROWSER"], url] if not cmd: # if can't detect the desktop environment then see if xdg-open # is available exe = utils.which('xdg-open') if exe: cmd = [exe, url] # Finish with an error where no suitable desktop was # identified. try: if not cmd: # TODO: maybe we should tell the user to define DESKTOP_LAUNCH raise OSError, _("Could not open %(url)s\n\n" "Unknown desktop environment: %(desktop)s\n\n") \ % dict(url=url, desktop=desktop_in_use) except Exception, e: if dialog_on_error: utils.message_dialog(utils.utf8(e)) else: raise
def quote_shell_arg(cmd): return commands.mkarg(cmd)
def doReconfigureNet(config, machines=None, i_am_master=1, force_ntp_reconfig=0): """ reconfigure serveriron, DNS, NTPs, iptables, timezone as needed. Force NTP server reconfiguration if force_ntp_reconfig=1 """ # first we need to reconfigure our external IP address ret1 = ret2 = ret4 = ret5 = ret6 = ret8 = 0 configType = config.var('ENT_CONFIG_TYPE') ent_home = E.getEnterpriseHome() if not machines: machines = config.var('MACHINES') # Sanity check if not config.var('EXTERNAL_WEB_IP'): logging.error('config file is missing EXTERNAL_WEB_IP, probably corrupt') raise Exception, 'config file is missing EXTERNAL_WEB_IP, probably corrupt' if config.var('USE_DHCP') is None: logging.error('config file is missing USE_DHCP, probably corrupt') raise Exception, 'config file is missing USE_DHCP, probably corrupt' if config.var('DNS_DHCP') is None: logging.error('config file is missing DNS_DHCP, probably corrupt') raise Exception, 'config file is missing DNS_DHCP, probably corrupt' tries = 3 # default for CLUSTER and PILE configuration if configType in ["SUPER", "ONEBOX", "MINI", "LITE", "FULL"]: tries = 1 # reconfigure eth0 to dhcp mode if config.var('USE_DHCP') == 1: ret1 = ExecuteWrapper( machines, "%s LOCALNET DHCP %s" % ( C.RECONFIGURE_COMMAND % ent_home, config.var('ONEBOX_NETCARD_SETTINGS'), ), None, 600, num_tries = tries) logging.info("reconfigure eth0 IP to dhcp mode: %s" % ret1) else: ret1 = ExecuteWrapper( machines, "%s LOCALNET %s %s %s %s" % ( C.RECONFIGURE_COMMAND % ent_home, config.var('EXTERNAL_WEB_IP'), config.var('EXTERNAL_NETMASK'), config.var('EXTERNAL_DEFAULT_ROUTE'), config.var('ONEBOX_NETCARD_SETTINGS'), ), None, 600, num_tries = tries) logging.info("reconfigure eth0 IP address: %s" % ret1) elif "CLUSTER" == configType: # reconfigure serveriron IP address cmd = "(sleep 5; %s SERVERIRON %s %s %s %s %s %s %s %s %s >&/dev/null " \ "</dev/null)" % ( C.RECONFIGURE_COMMAND % ent_home, "%s/local/conf/defaults/" % ent_home, commands.mkarg(config.var("EXTERNAL_SWITCH_IP")), commands.mkarg(config.var("EXTERNAL_WEB_IP")), commands.mkarg(config.var("EXTERNAL_CRAWL_IP")), commands.mkarg(config.var("EXTERNAL_NETMASK")), commands.mkarg(config.var("EXTERNAL_DEFAULT_ROUTE")), commands.mkarg(str(config.var("EXTERNAL_WEB_PORT"))), commands.mkarg(str(config.var("SERVERIRON_AUTONEGOTIATION"))), commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))), ) t = threading.Thread(target=E.execute, args=([E.getCrtHostName()], cmd, None, 60)) t.start() logging.info("serveriron IP address reconfigured") # we don't want to change ANYTHING on PILE clusters if "PILE" != configType: # DNS needs to be set everywhere dns_server = "\"\"" dns_searchpath = "\"\"" if config.var('BOT_DNS_SERVERS') != None: dns_server = config.var('BOT_DNS_SERVERS') if config.var('DNS_SEARCH_PATH') != None: dns_searchpath = config.var('DNS_SEARCH_PATH') if config.var('DNS_DHCP') != 1: ret8 = ExecuteWrapper( machines, "%s DNSMODE STATIC" % ( C.RECONFIGURE_COMMAND % ent_home ), None, 600, num_tries = tries) logging.info("setting DNS mode to STATIC: %s" % ret8) ret2 = ExecuteWrapper( machines, "%s DNS %s %s" % ( C.RECONFIGURE_COMMAND % ent_home, commands.mkarg(dns_server), commands.mkarg(dns_searchpath), ), None, 600, num_tries = tries) logging.info("reconfigure DNS: %s" % ret2) else: ret8 = ExecuteWrapper( machines, "%s DNSMODE DHCP" % ( C.RECONFIGURE_COMMAND % ent_home ), None, 600, num_tries = tries) logging.info("setting DNS mode to DHCP: %s" % ret8) # NTP is special: all machines but the master must set their # NTP server to the master, the master to the external one. # However, It can take 3 minutes for the stratum level of the master # node to be set. Before that, non-master nodes using the master # node to do "ntpdate" may return "no server suitable for synchronization # found" error. # To fix the problem, we just use the external ntp servers for all nodes. # Later, periodic script will set the non-master nodes to use the master # node. (periodic script will only set it once as long as master does # not switch) ntpServers = "\"\"" if config.var('NTP_SERVERS') != None: ntp_server_list = copy.copy(config.var('NTP_SERVERS')) AddDefaultNTPServer(ntp_server_list) ntpServers = string.join(ntp_server_list, ",") if i_am_master: ret4 = ExecuteWrapper( machines, "%s NTP %d %s" % ( C.RECONFIGURE_COMMAND % ent_home, force_ntp_reconfig, commands.mkarg(ntpServers)), None, 600, num_tries = tries) logging.info("reconfigure NTP: %s" % ret4) # whenever we print dates, we want to include the timezone. Since # the timezone may change on the machine (through config interface), # and java VM does not pick this change up automatically, # we keep track of the timezone here ret5 = ExecuteWrapper( machines, "%s TIMEZONE %s" % ( C.RECONFIGURE_COMMAND % ent_home, commands.mkarg(config.var('TIMEZONE'))), None, 600, num_tries = tries) logging.info("reconfigure TIMEZONE: %s" % ret5) # iptables if configType in ["SUPER", "ONEBOX", "LITE", "FULL"]: iptables_file = "%s/local/conf/defaults/iptables.onebox" % ent_home elif "MINI" == configType: iptables_file = "%s/local/conf/defaults/iptables.mini" % ent_home elif "CLUSTER" == configType: iptables_file = "%s/local/conf/defaults/iptables.cluster" % ent_home else: iptables_file = "%s/local/conf/defaults/iptables.open" % ent_home ret6 = ExecuteWrapper( machines, "%s IPTABLES %s %s %s" % ( C.RECONFIGURE_COMMAND % ent_home, iptables_file, commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_SSH"))), commands.mkarg(str(config.var("ENT_ENABLE_EXTERNAL_BORGMON"))), ), None, 600, num_tries = tries) logging.info("reconfigure IPTABLES: %s" % ret6) # return value from each reconfigurenet call is: # 0 : no change # 1 : changed # 2+: error (but we report no errors) # _our_ return value is if any of them failed ret = (ret1 <= 1 and ret2 <= 1 and ret4 <= 1 and ret5 <= 1 and ret6 <= 1 and ret8 <= 1) return ret
def get_package_status(package, avail=False): if not avail and package in statuscache: return statuscache[package] versionre = re.compile('Version: ') packagere = re.compile('Package: ') priorityre = re.compile('Priority: ') dependsre = re.compile('(Pre-)?Depends: ') recsre = re.compile('Recommends: ') suggestsre = re.compile('Suggests: ') conffilesre = re.compile('Conffiles:') maintre = re.compile('Maintainer: ') statusre = re.compile('Status: ') originre = re.compile('Origin: ') bugsre = re.compile('Bugs: ') descre = re.compile('Description: ') fullre = re.compile(' ') srcre = re.compile('Source: ') sectionre = re.compile('Section: ') pkgversion = pkgavail = maintainer = status = origin = None bugs = vendor = priority = desc = src_name = section = None conffiles = [] fulldesc = [] depends = [] recommends = [] suggests = [] confmode = False state = '' try: x = os.getcwd() except OSError: os.chdir('/') packarg = commands.mkarg(package) if avail: output = commands.getoutput( "COLUMNS=79 dpkg --print-avail %s 2>/dev/null" % packarg) else: output = commands.getoutput("COLUMNS=79 dpkg --status %s 2>/dev/null" % packarg) # dpkg output is in UTF-8 format output = output.decode('utf-8', 'replace') for line in output.split(os.linesep): line = line.rstrip() if not line: continue if confmode: if line[:2] != ' /': confmode = False else: conffiles = conffiles + [tuple(line.split()[:2])] if versionre.match(line): (crud, pkgversion) = line.split(": ", 1) elif statusre.match(line): (crud, status) = line.split(": ", 1) elif priorityre.match(line): (crud, priority) = line.split(": ", 1) elif packagere.match(line): (crud, pkgavail) = line.split(": ", 1) elif originre.match(line): (crud, origin) = line.split(": ", 1) elif bugsre.match(line): (crud, bugs) = line.split(": ", 1) elif descre.match(line): (crud, desc) = line.split(": ", 1) elif dependsre.match(line): (crud, thisdepends) = line.split(": ", 1) # Remove versioning crud thisdepends = [[y.split()[0] for y in x.split('|')] for x in (thisdepends.split(', '))] depends.extend(thisdepends) elif recsre.match(line): (crud, thisdepends) = line.split(": ", 1) # Remove versioning crud thisdepends = [[y.split()[0] for y in x.split('|')] for x in (thisdepends.split(', '))] recommends.extend(thisdepends) elif suggestsre.match(line): (crud, thisdepends) = line.split(": ", 1) # Remove versioning crud thisdepends = [[y.split()[0] for y in x.split('|')] for x in (thisdepends.split(', '))] suggests.extend(thisdepends) elif conffilesre.match(line): confmode = True elif maintre.match(line): crud, maintainer = line.split(": ", 1) elif srcre.match(line): crud, src_name = line.split(": ", 1) src_name = src_name.split()[0] elif sectionre.match(line): crud, section = line.split(": ", 1) elif desc and line[0] == ' ': fulldesc.append(line) installed = False if status: state = status.split()[2] installed = (state not in ('config-files', 'not-installed')) reportinfo = None if bugs: reportinfo = debbugs.parse_bts_url(bugs) elif origin: if debbugs.SYSTEMS.has_key(origin): vendor = debbugs.SYSTEMS[origin]['name'] reportinfo = (debbugs.SYSTEMS[origin].get('type', 'debbugs'), debbugs.SYSTEMS[origin]['btsroot']) else: vendor = origin.capitalize() else: vendor = '' info = (pkgversion, pkgavail, tuple(depends), tuple(recommends), tuple(conffiles), maintainer, installed, origin, vendor, reportinfo, priority, desc, src_name, os.linesep.join(fulldesc), state, tuple(suggests), section) if not avail: statuscache[package] = info return info
def escape(str): """ Shell escape the given string """ return commands.mkarg(str)[1:]
def SyncRequestDirs(self): sync_problem = [] if self._peeks: logging.info("Peeking commands from other machines") self._sync_cycle = self._sync_cycle + 1 err = 0 if self._sync_cycle > PEEK_CYCLE_FREQUENCY: # We sync one by one to avoid confilicts for machine, dir in self._peeks.items(): # First get a directory listing of all files waiting for us if machine == 'None': ls_cmd = self._fileutil_cmd % ("ls %s" % dir) _, remote_files = commands.getstatusoutput(ls_cmd) status = 0 # in fileutil case we are OK since nofiles == error logging.info("Executing %s" % ls_cmd) else: remote_cmd = "ls -1 %s/*" % dir logging.info("Executing %s on %s" % (remote_cmd, machine)) (status, sig, remote_files) = prodlib.RunAlarmRemoteCmd( machine, remote_cmd, 60) # endif if not remote_files: continue # endif if status: logging.error("Error executing [%s] " "status: %s / sig: %s / output %s" % (remote_cmd, status, sig, remote_files)) err = 1 sync_problem.append(machine) continue # endif # We have a list of remote files that need to be copied # first copy them, then upon success, blow them away remotely cmds = [] remote_files = string.split(remote_files, '\n') for file in remote_files: # We can assume that the files returned by ls are unique # index the commands by the filename, so we know # exactly which files to blow away later if machine == 'None': cmds.append( (file, self._fileutil_cmd % ("cp %s %s%s" % (file, self._request_dir, os.path.basename(file))))) logging.info("Copying file %s" % (file)) else: cmds.append((file, "scp %s:%s %s" % (machine, file, self._request_dir))) logging.info("Copying files from %s" % (machine)) # endif # endfor # We now have a list of commands, execute them in parallel if machine == 'None': results = prodlib.ForkCommands( cmds, timeout=60, numslots=20, machname=lambda x, mach=machine: 'localhost') else: results = prodlib.ForkCommands( cmds, timeout=60, numslots=20, machname=lambda x, mach=machine: mach) # endif # Maintain a list of files that were successfully copied success_files = [] for (file, file_results) in results.items(): (status, output, errors) = file_results if status: logging.error( "Error copying [%s] " "status: %s / errors: %s / output %s" % (file, status, errors, output)) # Avoid adding this machine too many times if not err: err = 1 sync_problem.append(machine) else: # Verify that the file exists locally filename = os.path.basename(file) dest_filename = '%s/%s' % (self._request_dir, filename) if (os.path.exists(dest_filename) and os.stat(dest_filename)[stat.ST_SIZE]): success_files.append(file) else: if os.path.exists(dest_filename): os.unlink( dest_filename) # remove empty files.. # endif # Avoid adding this machine too many times if not err: err = 1 sync_problem.append(machine) # endif # endif # endfor # Now, blow away all the files that were successfully copied if machine == 'None': command = self._fileutil_cmd % ( "rm -f %s" % string.join(success_files)) status, output = commands.getstatusoutput(command) sig = 0 else: command = "rm -f %s" % string.join(success_files) logging.info("Executing %s on %s" % (command, machine)) (status, sig, output) = prodlib.RunAlarmRemoteCmd( machine, command, 60) # endif if status: logging.error("Error executing [%s] " "status: %s / sig: %s / output %s" % (command, status, sig, output)) if not err: err = 1 sync_problem.append(machine) # endif # endif # endfor # Reset the cycle if no error occured if not err: self._sync_cycle = 0 # endif # endif # endif # Replicate the working and request dirs to all machines if self._replication_machines: # It is unsafe to execute prodlib.ForkExecPool from here because # we get messed up on Popen3 / select, instead we do a googlesh err = 0 for f in [self._working_dir, self._request_dir]: if os.system( ". /etc/profile ; googlesh -m'%s' " "-p10 -a120 -d0 -q -b %s" % (string.join(self._replication_machines, " "), commands.mkarg( "rsync --delete -e \"ssh -c none\" -aH %s:%s %s" % (self._local_machine, f, f)))): err = 1 # endif # endfor if err: sync_problem.extend(self._replication_machines) # endif # endif # Update the status for this guy self._sync_problems = repr(sync_problem)
#!/usr/bin/env python import commands, os, sys # Patch out the JMP (&FFFC) reset call EORed with 0x0e in the final file. # 0x04 is 0xea ^ 0x0e where 0xea is the opcode for NOP. t = open("temp/Lunar3\xc3").read() t = t[:0x261e] + "\x04\x04\x04" + t[0x2621:] open("temp/Lunar3\xc3", "wb").write(t) os.system("UEFtrans.py " + commands.mkarg(sys.argv[1]) + " insert 3 temp/Lunar3\xc3")
def escape(str): return commands.mkarg(str)[1:]
def escape(arg): arg = commands.mkarg(arg) if arg[0] == ' ': arg = arg[1:] return arg
if last != None: sys.stdout.write("\rEncoded %i of %i (%1.2f seconds)" % ( frame - first, last - first, elapsed)) else: sys.stdout.write("\rEncoded %i (%1.2f seconds, %1.2f%%)" % ( frame - first, elapsed, percent)) avi.end_list() # movi frames = frame - first + 1 for p in placeholders: avi.f.seek(p) avi.write_int(frames) avi.close() print print "Ways to process the output file:" print #print "mencoder %s -o %s -ovc lavc -oac lavc -lavcopts vcodec=mpeg4:acodec=libmp3lame -srate 44100" % (avi_file, avi_file.replace(".avi", ".mp4")) avi_file = commands.mkarg(avi_file) wav_file = avi_file.replace(".avi", ".wav") wav_44100_file = wav_file.replace(".wav", "-44100.wav") mp4_file = avi_file.replace(".avi", ".mp4") print "ffmpeg -i %s -map 0:1 %s" % (avi_file, wav_file) print "sox %s %s rate 44100" % (wav_file, wav_44100_file) print "ffmpeg -i %s -i %s -map 0:0 -map 1:0 -strict experimental %s" % (avi_file, wav_44100_file, mp4_file) print print "ffmpeg -i %s %s" % (avi_file, avi_file.replace(".avi", ".webm")) sys.exit()
def remote_cmd(machine, cmd, ssh_port=22): """ runs a command remotely """ return "ssh -p %d -l root %s %s" % (ssh_port, machine, commands.mkarg(cmd))
directories = {} for path in old_files: path = os.path.abspath(path) directory, file_name = os.path.split(path) directory = directory.split(os.sep) if directory[:len(current_dir_pieces)] == current_dir_pieces: pieces = directory[len(current_dir_pieces):] dir_path = target_dir for piece in pieces: dir_path = os.path.join(dir_path, piece) if not os.path.exists(dir_path): os.mkdir(dir_path) path = path[len(current_dir):].lstrip(os.sep) new_path = path[:-len(suffix)] diff_path = os.path.join(dir_path, file_name[:-len(suffix)] + new_suffix) os.chdir(current_dir) os.system("diff -u " + commands.mkarg(path) + " " + commands.mkarg(new_path) + " > " + commands.mkarg(diff_path)) sys.exit()