def _flush(): """Let all pending operations finish and flush anything to any disks E.g. iscsi etc pipe() is used to capture the output of the calls """ # Don't litter the screen with output, so get a handle to /dev/null with open(os.devnull, 'wb') as DEVNULL: process.call(["udevadm", "settle"], stdout=DEVNULL, stderr=DEVNULL)
def resolve_document_using_git(git, docref): document = docref_to_dict(docref) kind = docref.kind id = docref.id get = lambda f, n=docref.name: call( git + ["show", "--quiet", "--format=format:%s" % (f,), n], do_check=False) get = lambda a: git_show(git, docref.name, a) if kind == "branches": branches = [trim(a, prefix=" remotes/origin/") for a in read_lines(call(git + ["branch", "-a"]))] document["branches"] = [] for branch in branches: document["branches"].append(docref_to_dict(BranchDocref(branch))) elif kind == "branch": sha = get1( read_lines( call(git + ["rev-parse", "remotes/origin/" + docref.name]))) document["commit"] = docref_to_dict(ShaDocRef("commit", sha)) elif kind == "commit": document.update( {"author": {"name": get("%an"), "email": get("%ae"), "date": get("%ai")}, "committer": {"name": get("%cn"), "email": get("%ce"), "date": get("%ci")}, "message": get("%B"), "tree": docref_to_dict(ShaDocRef("tree", get("%T"))), "parents": [], }) for p in sorted(get("%P").split(" ")): if p == "": continue document["parents"].append( docref_to_dict(ShaDocRef("commit", p))) elif kind == "tree": document["children"] = [] for line in read_lines(call(git + ["ls-tree", docref.name])): child_mode, child_kind, rest = line.split(" ", 2) child_sha, child_basename = rest.split("\t", 1) ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)), "basename": child_basename, "mode": octal_to_symbolic_mode(child_mode)} document["children"].append(ref) document["children"].sort(key=lambda a: a["child"]["sha"]) elif kind == "blob": blob = call(git + ["show", docref.name], do_crlf_fix=False) if is_text(blob): document["encoding"] = "raw" document["raw"] = blob else: document["encoding"] = "base64" document["base64"] = base64.b64encode(blob) else: raise NotImplementedError(kind) return document
def notify(*, away=False, back=False): assert away or back if away: message = "Marked as Away" else: message = "Marked as Back" call( [cfg.terminal_notifier] + ["-message", message] + ["-title", "Time Tracker"] + ["-sender", "uno.glob.timetracker"] )
def reset_and_run_testcase(testcase_id, category, release): """Resets the chromium repo and runs the testcase.""" delete_if_exists(CHROMIUM_OUT) delete_if_exists(CLUSTERFUZZ_CACHE_DIR) process.call('git checkout -f HEAD', cwd=CHROMIUM_SRC) # Clean untracked files. Because untracked files in submodules are not removed # with `git checkout -f HEAD`. process.call('git clean -d -f -f', cwd=CHROMIUM_SRC) version = prepare_binary_and_get_version(release) update_auth_header() stackdriver_logging.send_run(testcase_id, category, version, release, run_testcase(testcase_id))
def copy_dir_if_not_exist(orig, target): """function to copy missing directories from one location to another should only be used when syncing a directory structure from iso to disk like /var/log use case -- upgrade adds some service which logs to /var/log/<service> need to have the directory created, but it's not on iso upgrade """ for f in os.listdir(orig): if os.path.isdir("%s/%s" % (orig, f)): if not os.path.exists("%s/%s" % (target, f)): process.call("cp -av %s/%s %s &>/dev/null" % (orig, f, target), shell=True) else: copy_dir_if_not_exist("%s/%s" % (orig, f), "%s/%s" % (target, f))
def hg(argv, **kwargs): kwargs.setdefault("cwd", hg_path) kwargs.setdefault("do_print", True) prefix = ["hg"] if username is not None and argv[0] in ("commit", "ci"): prefix.extend(["--config", "ui.username=%s" % (username,)]) return call(prefix + argv, **kwargs)
def update(self): cib_str = process.call(CIB.GET_CIB_CMD.split(" ")) self._cib_el = ET.fromstring(cib_str) self._state_el = CIB.get_real_time_state() self._nodes_el = self._cib_el.find(CIB.NODES_XPATH) self._resources_el = self._cib_el.find(CIB.RESOURCES_XPATH) self._constraints_el = self._cib_el.find(CIB.CONSTRAINTS_XPATH)
def couchapp(url, local_path): url = url.encode("ascii") local_path = os.path.abspath(local_path) design_json = call(["couchapp", "push", "--export"], stderr=None, cwd=local_path) # Strangely it seems to print a line before the design document design_json = design_json[design_json.find("{"):] put_update(url, lambda a=None: json.loads(design_json))
def git_show(git, sha, attr): start = "#start#" end = "#end#" out = call(git + ["show", "--format=format:%s%s%s" % (start, attr, end), "--quiet", sha], do_check=False) sindex = out.index(start) + len(start) eindex = out.rindex(end) assert sindex <= eindex, out return out[sindex:eindex]
def run_testcase(testcase_id): """Attempts to reproduce a testcase.""" try: process.call('/python-daemon/clusterfuzz reproduce %s' % testcase_id, cwd=HOME, env={ 'CF_QUIET': '1', 'USER': '******', 'CHROMIUM_SRC': CHROMIUM_SRC, 'GOMA_GCE_SERVICE_ACCOUNT': 'default', 'PATH': '%s:%s' % (os.environ['PATH'], DEPOT_TOOLS) }) success = True except subprocess.CalledProcessError: success = False TESTCASE_CACHE[testcase_id] = success return success
def git_show(git, sha, attr): try: start = "#start#" end = "#end#" out = call(git + ["show", "--format=format:%s%s%s" % (start, attr, end), "--quiet", sha], do_check=False) sindex = out.index(start) + len(start) # TODO: Shouldn't need these magic markers eindex = out.index(end) assert sindex <= eindex, out return out[sindex:eindex] except Exception: raise Exception("Unable to get git attribute %r from %r" % (attr, sha))
def build_master_and_get_version(): """Checks out the latest master build and creates a new binary.""" if not os.path.exists(TOOL_SOURCE): process.call( 'git clone https://github.com/google/clusterfuzz-tools.git', cwd=HOME) process.call('git fetch', cwd=TOOL_SOURCE) process.call('git checkout origin/master -f', cwd=TOOL_SOURCE) process.call('./pants binary tool:clusterfuzz-ci', cwd=TOOL_SOURCE, env={'HOME': HOME}) delete_if_exists(BINARY_LOCATION) shutil.copy(os.path.join(TOOL_SOURCE, 'dist', 'clusterfuzz-ci.pex'), BINARY_LOCATION) # The full SHA is too long and unpleasant to show in logs. So, we use the # first 7 characters of the SHA instead. return process.call('git rev-parse HEAD', capture=True, cwd=TOOL_SOURCE).strip()[:7]
def get_release_file(base_url): for src, dst in [("", "")] + list(PORTS_MIRRORS): url = base_url if url.startswith(src): url = dst + trim(url, prefix=src) try: file_data = get(url).content except Exception, e: try: file_data_bz2 = get(url + ".bz2").content except Exception, e: try: file_data_gz = get(url + ".gz").content except Exception, e: continue else: file_data = call(["gunzip"], stdin_data=file_data_bz2, do_crlf_fix=False)
def reboot(): """Reboot the system """ process.call(["reboot"])
def ubuntu_to_hg(hg_path, username, do_development=False): def hg(argv, **kwargs): kwargs.setdefault("cwd", hg_path) kwargs.setdefault("do_print", True) prefix = ["hg"] if username is not None and argv[0] in ("commit", "ci"): prefix.extend(["--config", "ui.username=%s" % (username,)]) return call(prefix + argv, **kwargs) with with_ubuntu_keyring() as gpg: meta_release_data = get( join(BASE_URL, "meta-release-development")).content meta_release = parse_control_file(meta_release_data) group_by(meta_release, lambda r: r["dist"]) if not os.path.exists(hg_path): os.makedirs(hg_path) hg(["init"]) branches = set([a.split()[0] for a in read_lines(hg(["branches"]))]) ok_branches = set() seen_supported_non_lts = False for release in meta_release: branch = "ubuntu_codename_%s" % (release["dist"],) is_lts = "LTS" in release["version"] is_supported = release["supported"] == "1" if is_supported and not is_lts: seen_supported_non_lts = True is_development = not is_supported and seen_supported_non_lts if not is_supported and not is_development: continue ok_branches.add(branch) if is_development and not do_development: continue done = set() if branch not in branches: hg(["update", "--clean", "--rev", "00"]) hg(["branch", "--force", branch]) else: hg(["update", "--clean", branch]) hg(["--config", "extensions.purge=", "purge", "--all"]) release_gpg_path = os.path.join(hg_path, "Release.gpg") release_path = os.path.join(hg_path, "Release") old_sha1sums = {} release_gpg_data = get(release["release-file"] + ".gpg").content if os.path.exists(release_gpg_path): if release_gpg_data == read_file(release_gpg_path): continue release_data = read_file(release_path) old_sha1sums = get_release_sha1sums(release_data) old_sha1sums["Release"] = hashlib.sha1( release_data).hexdigest() old_sha1sums["Release.gpg"] = hashlib.sha1( release_gpg_data).hexdigest() # for relpath in sorted(old_sha1sums): # if posixpath.dirname(relpath) == "Index": # index_data = read_file(os.path.join(hg_path, relpath)) # child_sha1sums = get_release_sha1sums(index_data) # for relpath2 in sorted(child_sha1sums): # relpath3 = posixpath.join( # posixpath.dirname(relpath), relpath2) # old_sha1sums[relpath3] = child_sha1sums[relpath2] release_data = get(release["release-file"]).content with open(release_gpg_path, "wb") as fh: fh.write(release_gpg_data) done.add("Release") with open(release_path, "wb") as fh: fh.write(release_data) done.add("Release.gpg") gpg(["--verify", release_gpg_path, release_path]) new_sha1sums = get_release_sha1sums(release_data) new_sha1sums["Release.gpg"] = hashlib.sha1( release_gpg_data).hexdigest() new_sha1sums["Release"] = hashlib.sha1( release_data).hexdigest() # for relpath in sorted(new_sha1sums): # if posixpath.basename(relpath) == "Index": # if new_sha1sums[relpath] == old_sha1sums.get(relpath): # index_data = read_file(os.path.join(hg_path, relpath)) # else: # index_data = get( # posixpath.join( # posixpath.dirname(release["Release-File"]), # relpath)).content # sha1sum = hashlib.sha1(index_data).hexdigest() # if sha1sum != new_sha1sums[relpath]: # raise Exception("sha1sum mismatch for %r: " # "got %s expecting %s" # % (url, sha1sum, # new_sha1sums[relpath])) # index_path = os.path.join(hg_path, relpath) # if not os.path.exists(os.path.dirname(index_path)): # os.makedirs(os.path.dirname(index_path)) # with open(index_path, "wb") as fh: # fh.write(index_data) # done.add(relpath) # child_sha1sums = get_release_sha1sums(index_data) # for relpath2 in sorted(child_sha1sums): # relpath3 = posixpath.join( # posixpath.dirname(relpath), relpath2) # new_sha1sums[relpath3] = child_sha1sums[relpath2] for relpath in old_sha1sums: if relpath in new_sha1sums: continue file_path = os.path.join(hg_path, relpath) call(["rm", "-rf", "--one-file-system", file_path]) for relpath in new_sha1sums: if relpath in old_sha1sums: if new_sha1sums[relpath] == old_sha1sums[relpath]: continue if (relpath.endswith(".gz") and trim(relpath, suffix=".gz") in new_sha1sums): continue if (relpath.endswith(".bz2") and trim(relpath, suffix=".bz2") in new_sha1sums): continue if relpath in done: continue file_path = os.path.join(hg_path, relpath) file_data = get_release_file( posixpath.join( posixpath.dirname(release["release-file"]), relpath)) sha1sum = hashlib.sha1(file_data).hexdigest() if sha1sum != new_sha1sums[relpath]: raise Exception("sha1sum mismatch for %r: " "got %s expecting %s" % (url, sha1sum, new_sha1sums[relpath])) if not os.path.exists(os.path.dirname(file_path)): os.makedirs(os.path.dirname(file_path)) with open(file_path, "wb") as fh: fh.write(file_data) hg(["addremove"]) if len(read_lines(hg(["status"]))) > 0: hg(["commit", "-m", "Update from upstream"]) for branch in branches: if branch == "default" or branch in ok_branches: continue hg(["update", "--clean", branch]) hg(["commit", "--close-branch", "-m", "Closing unsupported release"])
def git_to_couchdb(cache_root, git_url, couchdb_url): if git_url is None: git = ["git"] else: cache_dir = os.path.join(cache_root, encode_as_c_identifier(git_url)) git = ["bash", "-c", 'cd "$1" && shift && exec "$@"', "-", cache_dir, "git"] call(["mkdir", "-p", cache_dir]) call(git + ["init"]) for r in read_lines(call(git + ["remote"])): call(git + ["remote", "rm", r]) call(git + ["remote", "add", "origin", git_url]) call(git + ["fetch", "origin"], stdout=None, stderr=None) resolve_document = lambda d: resolve_document_using_git(git, d) fetch_all(resolve_document, couchdb_url, [BRANCHES_DOCREF])
def restart(self): self.logger.debug("Restarting SSH") process.call("service sshd restart &>/dev/null")
file_data = get(url).content except Exception, e: try: file_data_bz2 = get(url + ".bz2").content except Exception, e: try: file_data_gz = get(url + ".gz").content except Exception, e: continue else: file_data = call(["gunzip"], stdin_data=file_data_bz2, do_crlf_fix=False) else: file_data = call(["bzip2", "-d"], stdin_data=file_data_bz2, do_crlf_fix=False) break else: break else: raise Exception("Failed to fetch %r including gz and " "bz2 variants and ports mirror" % (base_url,)) return file_data @contextlib.contextmanager def with_ubuntu_keyring(): keyrings = [ "/usr/share/keyrings/ubuntu-master-keyring.gpg", "/usr/share/keyrings/ubuntu-archive-keyring.gpg",
def poweroff(): """Poweroff the system """ process.call(["poweroff"])
def get_real_time_state(): xml_str = process.call(CIB.GET_REALTIME_STATUS_CMD.split(" ")) return ET.fromstring(xml_str)
def resolve_document_using_git(git, docref): document = docref_to_dict(docref) kind = docref.kind id = docref.id get = lambda a: git_show(git, docref.name, a) if kind == "branches": branches = set() for line in read_lines(call(git + ["branch", "-a"])): if line.startswith(" "): line = trim(line, prefix=" ") elif line.startswith("* "): line = trim(line, prefix="* ") if line == "(no branch)": continue else: raise NotImplementedError(line) if " -> " in line: ba, bb = line.split(" -> ", 1) branches.add(posixpath.basename(ba)) branches.add(posixpath.basename(bb)) else: branches.add(posixpath.basename(line)) branches = list(sorted(b for b in branches if b != "HEAD")) document["branches"] = [] for branch in branches: document["branches"].append(docref_to_dict(BranchDocref(branch))) elif kind == "branch": sha = get1( read_lines( call(git + ["rev-parse", docref.name]))) document["commit"] = docref_to_dict(ShaDocRef("commit", sha)) elif kind == "commit": document.update( {"author": {"name": get("%an"), "email": get("%ae"), "date": get("%ai")}, "committer": {"name": get("%cn"), "email": get("%ce"), "date": get("%ci")}, "message": get("%B"), "tree": docref_to_dict(ShaDocRef("tree", get("%T"))), "parents": [], }) for p in sorted(get("%P").split(" ")): if p == "": continue document["parents"].append( docref_to_dict(ShaDocRef("commit", p))) elif kind == "tree": document["children"] = [] for line in read_lines(call(git + ["ls-tree", docref.name])): child_mode, child_kind, rest = line.split(" ", 2) child_sha, child_basename = rest.split("\t", 1) ref = {"child": docref_to_dict(ShaDocRef(child_kind, child_sha)), "basename": child_basename, "mode": octal_to_symbolic_mode(child_mode)} document["children"].append(ref) document["children"].sort(key=lambda a: a["child"]["sha"]) elif kind == "blob": blob = call(git + ["show", docref.name], do_crlf_fix=False) if is_text(blob): document["encoding"] = "raw" document["raw"] = blob else: document["encoding"] = "base64" document["base64"] = base64.b64encode(blob) else: raise NotImplementedError(kind) return document
def gpg(argv, **kwargs): return call(["gpg", "--homedir", temp_dir] + argv, **kwargs)
def main(argv): parser = optparse.OptionParser(__doc__) parser.add_option("--install", dest="do_install", default=False, action="store_true") parser.add_option("--skip-proxy-install", dest="do_proxy_install", default=True, action="store_false") parser.add_option("--skip-blank-sources-list-install", dest="do_blank", default=True, action="store_false") parser.add_option("--proxy", dest="proxy", default="http://ubuntu.devel.cmedltd.com:3142/") parser.add_option("--no-proxy", dest="proxy", action="store_const", const=None) parser.add_option("--proxy-basename", dest="proxy_basename", default="02-generated-proxy") parser.add_option("--basename", dest="basename", default="generated") options, args = parser.parse_args(argv) custom_json = json.dumps({}) if len(args) > 0: custom_json = args.pop(0) if len(args) > 0: parser.error("Unexpected: %r" % (args,)) default_json = json.dumps(DEFAULT_CONFIG) defaults = json.loads(default_json) custom = json.loads(custom_json) for key, value in defaults.items(): custom.setdefault(key, value) if custom["distribution"] is None: custom["distribution"] = trim( call(["lsb_release", "--short", "--codename"]), suffix="\r\n") output = render_to_sources_list(custom) if options.do_install: assert "\0" not in options.basename and "/" not in options.basename,\ repr(options.basename) file_data = { "/etc/apt/sources.list.d/%s.list" % (options.basename,): output} if options.do_blank: file_data["/etc/apt/sources.list"] = BLANK_SOURCES if options.do_proxy_install: if options.proxy is None: proxy_data = "# No proxy\r\n" else: assert '"' not in options.proxy, repr(options.proxy) proxy_data = ('Acquire::HTTP { Proxy::"%s"; }\r\n' % (options.proxy,)) assert ("\0" not in options.proxy_basename and "/" not in options.proxy_basename), \ repr(options.proxy_basename) file_data["/etc/apt/apt.conf.d/%s" % (options.proxy_basename,)] = \ proxy_data child = subprocess.Popen(["sudo", "python", "-c", """\ assert __name__ == "__main__" import sys assert len(sys.argv) == 1 import json file_data = json.loads(sys.stdin.read()) for file_path, data in sorted(file_data.items()): fh = open(file_path, "wb") try: fh.write(data) finally: fh.close() """], stdin=subprocess.PIPE) child.communicate(json.dumps(file_data)) assert child.returncode == 0, child.returncode sys.stdout.write(output)
def get_supported_jobtypes(): """Returns a hash of supported job types.""" out = process.call(build_command('supported_job_types'), capture=True) result = yaml.load(out) result.pop('Version', None) return result
def __call__(self, cb): self.usercb = cb process.call(self.cline, loc=self.loc, cb=self._cb, context=self.context, **self.kwargs)
def cleanup(self, id): process.call(CIB.CLEANUP_CMD.format(id=id).split(" "))
def get_binary_version(): """Returns the version of the binary.""" out = process.call(build_command('supported_job_types'), capture=True) return yaml.load(out)['Version']
def bbcall(argv, **kwargs): kwargs.setdefault("do_print", True) argv = basebox.get_ssh_argv(config, argv) return call(argv, **kwargs)