Exemple #1
0
def start_mzbench_server():
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{node_rsync, "{0}"}},'.format(os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{node_git, "{0}"}},'.format(os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write('[{{mzbench_api, [{0}{{node_log_port, 0}}, {{node_management_port, 0}}]}}].'.format(node_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(mzbench_script, dirname))
    try:
        time.sleep(3) # give server some time to start
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        logdir = os.path.join(mzbench_dir + 'server/_build/default/rel/mzbench_api/log')
        logfiles = [logfile for logfile in os.listdir(logdir)]
        logfile = sorted([os.path.join(logdir, l) for l in logfiles if l.startswith('erlang')], key=os.path.getmtime, reverse=True)[0]
        with open(logfile) as f:
            for line in f:
                print line.rstrip().replace('\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))
Exemple #2
0
    def target_merge_busybox(self):
        env = {}
        use = ''
        if 'USE' in self.env:
            use = self.env['USE']

        if len(self.package_list) == 0:
            use += ' static'
        use += ' make-symlinks'
        env.update(self.env)
        env['USE'] = use
        env['ROOT'] = self.target_root

        cmdline = '%s/inhibitor-run.sh run_emerge --newuse --nodeps sys-apps/busybox' \
                % self.env['INHIBITOR_SCRIPT_ROOT']

        if self.seed:
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':env},
                failuref    = self.chroot_failure
            )
            util.chroot(
                path        = self.target_root,
                function    = self.path_sync_callback,
                fargs       = {'src':'/bin/busybox', '_':None},
                failuref    = self.chroot_failure
            )

        else:
            util.cmd( cmdline, env = env )
            self.path_sync_callback('/bin/busybox', None)
Exemple #3
0
def pull(args: Args) -> None:
    """Update ``args.repo`` with ``git pull``."""
    if args.pull:
        cmd("git pull --no-edit")
        proc = subprocess.run(args.sudo_prefix + ["git", "pull", "--no-edit"],
                              check=False,
                              cwd=args.repo)
        if proc.returncode != 0:
            # git pull failed, maybe reset?
            if args.reset:
                run_or_fatal(
                    args.sudo_prefix +
                    ["git", "reset", "--hard", args.remote_branch],
                    log=True,
                    cwd=args.repo,
                )
            else:
                fatal(
                    "`git pull` failed. Pass `--reset` to reset the repository."
                )

    info(f"{args.repo} is now at commit:")
    subprocess.run(["git", "log", "HEAD^1..HEAD", "--oneline"],
                   check=False,
                   cwd=args.repo)
Exemple #4
0
    def pack(self):
        emb_root = self.target_root
        if self.seed:
            emb_root = emb_root.pjoin(self.target_root)

        basedir = util.Path( os.path.dirname(self.tarpath) )
        util.mkdir(basedir)

        archive = tarfile.open(self.tarpath, 'w:bz2')
        archive.add(emb_root,
            arcname = '/',
            recursive = True
        )
        archive.close()

        curdir = os.path.realpath(os.curdir)
        os.chdir(emb_root)
        util.cmd('find ./ | cpio -H newc -o | gzip -c -9 > %s' % (self.cpiopath))
        os.chdir(curdir)

        if self.kernel:
            r = util.Path('/')
            if self.seed:
                r = self.target_root
            r = r.pjoin('/tmp/inhibitor/kernelbuild')

            kernel_link = r.pjoin('/boot/kernel')
            kernel_path = os.path.realpath( kernel_link )

            if os.path.lexists(self.kernlinkpath):
                os.unlink(self.kernlinkpath)

            shutil.copy2(kernel_path, basedir.pjoin( os.path.basename(kernel_path) ))
            os.symlink(os.path.basename(kernel_path), self.kernlinkpath)
Exemple #5
0
def run(workdir, proj, bug, onlyFailing, onlyRelevant, onlyTest):
    cmd("{0}/defects4j/framework/bin/defects4j compile -w {0}/data/{1}/{2}b".
        format(workdir, proj, bug))
    try:
        os.makedirs("{0}/result/".format(workdir))
    except OSError as e:
        if e.errno != errno.EEXIST:
            raise

    if onlyFailing:
        for t in failing_test.get_raw(workdir, proj, bug):
            cmd("{0}/defects4j/framework/bin/defects4j test -w {0}/data/{1}/{2}b -t {3}"
                .format(workdir, proj, bug, t))
    elif onlyRelevant:
        cmd("{0}/defects4j/framework/bin/defects4j test -w {0}/data/{1}/{2}b -f"
            .format(workdir, proj, bug))
    elif onlyTest:
        cmd("{0}/defects4j/framework/bin/defects4j test -w {0}/data/{1}/{2}b -t {3}"
            .format(workdir, proj, bug, onlyTest))
    else:
        cmd("{0}/defects4j/framework/bin/defects4j test -w {0}/data/{1}/{2}b".
            format(workdir, proj, bug))

    with open("result/error.log") as log:
        for line in log:
            if "[ERROR]" in line:
                raise ValueError("Error in log: " + line)
Exemple #6
0
def diff_hw_config(hardware_cfg: Path) -> None:
    """Diff changes if we updated the hardware configuration.

    :param hardware_cfg: path to ``hosts/$(hostname)-hardware-configuration.nix``
    """
    # Figure out what would happen if we ran nixos-generate-config.
    info("Diff if `nixos-generate-config` was run:")
    cmd(
        "nixos-generate-config --show-hardware-config "
        + f"| diff --ignore-all-space {hardware_cfg} - "
        + "| delta"
    )
    new_cfg = get_output(["nixos-generate-config", "--show-hardware-config"])
    diff = get_output(
        [
            "diff",
            "--report-identical-files",
            "--new-file",
            "--unified",
            "--ignore-all-space",
            str(hardware_cfg),
            "-",
        ],
        input=new_cfg,
        # 1 just means we found differences
        ok_returncodes=[0, 1],
    )
    delta = subprocess.run(["delta"], input=diff, encoding="utf-8", check=False)
    if delta.returncode != 0:
        warn(f"delta exited with non-zero return code {delta.returncode}")
Exemple #7
0
	def restore(self, source, specificFileToRestore='', restoreTo=''):
		"""source: name of source group to restore
		specificFileToRestore: relative path of a specific file/directory to restore
		restoreTo: path string of restore location"""
		sourceGrp = self.sourceGroups[source]
		if sourceGrp is None: 
			raise Exception("Cannot restore: no such source group "+
							"associated with link - {0}".format(source))
		os.environ['PASSPHRASE'] = getpwd()
		options = []
		
		# duplicity currently doesn't support the 'include' 
		# and 'exclude' options when performing a restore
		# these options are what allow us to backup in a single command above.
		# instead, we use a series of multiple commands with 'file-to-restore' option
		# to perform the same role.
		for source in souceGrp.items:
			path = source.path
			# We must make it a relative path (i.e. remove leading slash)
			if path.startswith('/'):
				path = path[1:]
			cmdList = ["duplicity",
						"restore",
						"--file-to-restore='{0}'".format(path),
						self.getTarget().path,
						'/' if restoreTo is '' else restoreTo
						]
			cmd(cmdList)
	
		# XXX
		#if specificFileToRestore is not '':
		#	options.append('--file-to-restore="{0}"'.format(specificFileToRestore))
				
		del os.environ['PASSPHRASE']
		self.doneSomething()
Exemple #8
0
    def sync(self):
        if os.path.exists(self.builddir):
            shutil.rmtree(self.builddir)
        elif os.path.islink(self.builddir):
            os.unlink(self.builddir)
        os.makedirs(self.builddir)

        exclude_cmd = ''
        if self.exclude:
            for i in self.exclude:
                exclude_cmd += " --exclude='%s'" % i

        if self.include:
            for pattern in self.include:
                paths = [self.src.cachedir.pjoin(pattern)]
                if '*' in pattern:
                    paths = glob.glob(self.src.cachedir.pjoin(pattern))

                for path in paths:
                    dest = path.replace(self.src.cachedir, self.builddir)
                    if not os.path.lexists( os.path.dirname(dest) ):
                        os.makedirs( os.path.dirname(dest) )
                    util.cmd('rsync -a %s %s/ %s/' % (
                        exclude_cmd,
                        path,
                        dest
                    ))
        else:
            util.cmd('rsync -a %s %s/ %s/' % (exclude_cmd, self.src.cachedir, self.builddir))
Exemple #9
0
def checkout(workdir, proj, bug):
    dir = "{0}/data/{1}/{2}b".format(workdir, proj, bug)
    if not os.path.exists(dir):
        os.makedirs(dir)
        cmd("{0}/defects4j/framework/bin/defects4j checkout -p {1} -v {2}b -w {3}"
            .format(workdir, proj, bug, dir))
    else:
        print "Skip checkout because {0} already exists".format(dir)
def check_migrations_test():
    mzbench_data_dir = tempfile.mkdtemp(prefix='mzbench_data_')

    with start_mzbench_server(custom_data_location=mzbench_data_dir):
        for i in range(5):
            run_successful_bench(scripts_dir + 'correct_script.erl')

    try:
        cmd(mzbench_dir + '/bin/migrate.py ' + mzbench_data_dir)
    finally:
        shutil.rmtree(mzbench_data_dir)
def check_migrations_test():
    mzbench_data_dir = tempfile.mkdtemp(prefix='mzbench_data_')

    with start_mzbench_server(custom_data_location=mzbench_data_dir):
        for i in range(5):
            run_successful_bench(scripts_dir + 'correct_script.erl')

    try:
        cmd(mzbench_dir + '/bin/migrate.py ' + mzbench_data_dir)
    finally:
        shutil.rmtree(mzbench_data_dir)
Exemple #12
0
def rm_tabsint_plugin_dependencies(pluginDict):
    """ remove cordova dependencies part of plugins from config """
    try:
        for name, plugin in pluginDict.iteritems():
            if "cordova" in plugin:
                ut.cmd("npm run cordova --silent -- plugin rm " + plugin["cordova"]["package"], force=True)
    except:
        sys.exit(
            "[ERROR]: Error while trying to uninstall plugin: {0}. Try manually uninstalling using 'npm run cordova --silent -- plugin rm {1}'".format(
                plugin["package"], plugin["package"]
            )
        )
Exemple #13
0
    def merge_packages(self):
        cmdline = '%s/inhibitor-run.sh run_emerge --newuse %s' % (
            self.env['INHIBITOR_SCRIPT_ROOT'], ' '.join(self.package_list))

        if self.seed:
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':self.env},
                failuref    = self.chroot_failure
            )
        else:
            util.cmd( cmdline, env = self.env )
Exemple #14
0
def update_hw_config_force(hardware_cfg: Path) -> None:
    """Generate and replace the hardware configuration.

    Performs no safety checks, but doesn't write if ``DRY_RUN`` is true.

    :param hardware_cfg: Path to ``hosts/$(hostname)-hardware-configuration.nix`` file to replace.
    """
    info(("Updating " if hardware_cfg.exists() else "Generating ") + p(hardware_cfg))
    cmd(f"nixos-generate-config --show-hardware-config > {hardware_cfg}")
    new_hardware_config = get_output(
        ["nixos-generate-config", "--show-hardware-config"]
    )
    if not DRY_RUN:
        hardware_cfg.write_text(new_hardware_config)
Exemple #15
0
def add_tabsint_plugin_dependencies(pluginDict):
    """ add dependent cordova plugins for tabsint plugins listed in config.json"""
    try:
        for name, plugin in pluginDict.iteritems():
            if "cordovaPlugins" in plugin:
                for p in plugin["cordovaPlugins"]:
                    ut.cmd("npm run cordova --silent -- plugin add {0}".format(p))
                    ut.log.info(
                        '[BUILD]: Successfully installed the cordova plugin "{0}" required by tabsint plugin "{1}"'.format(
                            p, name
                        )
                    )
    except Exception as e:
        rm_tabsint_plugins()
        sys.exit('[ERROR]: Error while trying to "cordovaPlugins" from with error: {0}'.format(str(e)))
Exemple #16
0
    def unpack_seed(self):
        if not os.path.isdir(self.seed):
            if os.path.exists(self.seed):
                os.unlink(self.seed)
            seedfile = self.seed + ".tar.bz2"
            util.info("Unpacking %s" % seedfile)
            os.makedirs(self.seed)
            try:
                util.cmd("tar -xjpf %s -C %s/" % (seedfile, self.seed))
            except:
                shutil.rmtree(self.seed)
                raise

        util.info("Syncing %s to %s" % (self.seed.dname(), self.target_root.dname()))
        util.cmd("rsync -a --delete %s %s" % (self.seed.dname(), self.target_root.dname()))
Exemple #17
0
def start_mzbench_server(custom_data_location=None):
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{node_rsync, "{0}"}},'.format(
            os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{node_git, "{0}"}},'.format(
            os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    if custom_data_location:
        custom_data_location_param = '{{bench_data_dir, "{0}"}},'.format(
            custom_data_location)
    else:
        custom_data_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write(
            '[{{mzbench_api, [{0} {1} {{node_log_port, 0}}, {{node_log_user_port, 0}}, {{node_management_port, 0}}, {{node_interconnect_port, 0}}]}}].'
            .format(node_location_param, custom_data_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(
        mzbench_script, dirname))
    try:
        time.sleep(3)  # give server some time to start
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        logdir = os.path.join(mzbench_dir +
                              'server/_build/default/rel/mzbench_api/log')
        logfiles = [logfile for logfile in os.listdir(logdir)]
        logfile = sorted([
            os.path.join(logdir, l) for l in logfiles if l.startswith('erlang')
        ],
                         key=os.path.getmtime,
                         reverse=True)[0]
        with open(logfile) as f:
            for line in f:
                print line.rstrip().replace('\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))
Exemple #18
0
	def backup(self, incremental=False):
		includes = []
		for k, group in self.sourceGroups.items():
			for item in group.items:
				includes.append('--include={0}'.format(item.path))
		
		os.environ['PASSPHRASE'] = getpwd()
		cmdList = ["duplicity", 
					"incremental" if incremental else "full",
					"--name='{0}'".format(self.targetGroup.name)]
		cmdList.extend(includes)
		cmdList.extend(["--exclude=**", "/", self.getTarget().geturl()])
		cmd(cmdList)
		
		del os.environ['PASSPHRASE']
		self.doneSomething()
Exemple #19
0
def devtool_list_templates_test():
    templates = os.listdir(mzbench_dir + "worker_templates")
    got_templates = filter(lambda x: x, cmd(mzbench_dir + "bin/mzbench list_templates").split("\n"))
    if sorted(templates) != sorted(got_templates):
        print sorted(templates)
        print sorted(got_templates)
        assert sorted(templates) == sorted(got_templates)
Exemple #20
0
def main(args: Optional[Args] = None) -> None:
    """Entry point."""
    if args is None:
        args = Args.parse_args()

    # Okay, so we don't actually use `os.chdir` here. Why? If we split a panel
    # while rebuilding (or open a new window), tmux starts the new shell in the
    # current process' cwd. Therefore, so we don't end up accidentally mucking
    # around in `/etc/nixos`, we don't change the cwd and instead use
    # `cwd=args.repo` for `subprocess.run` invocations.
    cmd(f"cd {p(args.repo)}")

    if args.fix_full_boot:
        fix_full_boot(args)

    pull(args)
    rebuild(args.rebuild_args, args.sudo_prefix, args.repo)
Exemple #21
0
def devtool_list_templates_test():
    templates = os.listdir(mzbench_dir + 'worker_templates')
    got_templates = filter(
        lambda x: x,
        cmd(mzbench_dir + 'bin/mzbench list_templates').split('\n'))
    if sorted(templates) != sorted(got_templates):
        print sorted(templates)
        print sorted(got_templates)
        assert sorted(templates) == sorted(got_templates)
Exemple #22
0
def get_tabsint_plugin_docs(name, src, version=None):
    """ include tabsint plugin documentation """

    def copy_userguide(path, ext="rst"):
        installPath = "../docs/userguide/src/docs/plugins"
        try:
            os.mkdir(installPath)
        except:
            pass

        try:
            shutil.copy("{0}/index.{1}".format(path, ext), "{0}/plugin-{1}.{2}".format(installPath, name, ext))

            # copy any other ancillary files - must be named the same as the plugin!
            if name in os.listdir(path):
                shutil.copytree("{0}/{1}".format(path, name), "{0}/{1}/".format(installPath, name))
        except Exception as e:
            sys.exit('[ERROR]: Failed to copy docs for plugin "{0}" to userguide. Error: {1} '.format(name, str(e)))

    # git files
    if any(s in src for s in ["https://", ".git"]):
        [repo, subdir] = ut.checkout(src, tag=version)
        docPath = ".tmp/{0}/{1}".format(repo, subdir)

    # local files
    else:
        src = "../" + src
        ut.check_tag(src, version)

        if src.endswith("/"):
            docPath = src[:-1]
        else:
            docPath = src

    # user guide
    if "index.rst" in os.listdir(docPath):
        copy_userguide(docPath)
    elif "index.md" in os.listdir(docPath):
        copy_userguide(docPath, ext="md")

    ut.cmd("rm -rf .tmp", force=True)  # remove temp git directory, if its there

    ut.log.info('[BUILD]: Successfully retrieved docs for tabsint plugin "{0}"'.format(name))
Exemple #23
0
def get_tabsint_plugin(name, src, version=None):
    """ get tabsint plugin from git repository or local folder"""

    def copy_to_plugins(path):
        shutil.rmtree("../www/tabsint_plugins/{0}".format(name), True)
        shutil.copytree(path, "../www/tabsint_plugins/{0}".format(name))

    # git repos
    if any(s in src for s in ["https://", ".git"]):
        [repo, subdir] = ut.checkout(src, tag=version)
        copy_to_plugins(".tmp/{0}/{1}".format(repo, subdir))
        ut.cmd("rm -rf .tmp")  # remove temp git directory

    # local files
    else:
        src = "../" + src  # make path relative to top level directory
        ut.check_tag(src, version)
        copy_to_plugins(src)

    ut.log.info('[BUILD]: Successfully retrieved the source for tabsint plugin "{0}"'.format(name))
Exemple #24
0
def get_raw(workdir, proj, bug):
    output = cmd(
        "{0}/defects4j/framework/bin/defects4j info -p {1} -b {2}".format(
            workdir, proj, bug))
    tests = re.findall(
        """Root cause in triggering tests:
(.*)
--------------------------------------------------------------------------------
List of modified sources:""", output, re.DOTALL)[0].split('\n')
    tests = filter(lambda x: x.startswith(' - '), tests)
    return map(lambda x: x.replace(' - ', ''), tests)
Exemple #25
0
    def render(self, font):
        with self.cache_lock:
            if self.invalid:
                self.cache = self.do_render()
                text_no_markup = re.sub('\<[^\>]+\>|\^[^\(]*\([^\)]*\)', '',
                                        self.cache)
                self.cache_width = int(cmd('textwidth', font, text_no_markup))

                self.invalid = False

        return self.cache
Exemple #26
0
def list_plugins():
    """ retrieve list of plugins and versions"""
    try:
        [code, stdout] = ut.cmd("npm run cordova --silent -- plugins ls", suppress=True)
        plugins = [p for p in stdout.replace("\n", ",").split(",") if p != ""]
        plugins = [p.split(' "')[0] for p in plugins]
        return plugins
    except:
        sys.exit(
            "[ERROR]: Error while trying to get a list of plugins from cordova. Try running 'npm run cordova --silent -- plugins ls', or for deeper investigation see the function 'list_plugins()' in /bin/util.py"
        )
Exemple #27
0
def devtool_run_local_tests():
    run_erl_and_bdl('validate', 'loop_rate')

    run_erl_and_bdl('validate', 'env', ' --env pool_size=20 --env jozin=jozin --env wait_ms=100')

    run_erl_and_bdl('run_local', 'loop_rate')

    run_erl_and_bdl('run_local', 'data_script')

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir + 'syntax_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir + 'semantic_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass
Exemple #28
0
def run(workdir, proj, bug, mem):
    tests = failing_test.get_with_lines(workdir, proj, bug)
    tests = ";".join(tests)

    for filename in os.listdir("{0}/data/info/{1}/{2}".format(
            workdir, proj, bug)):
        src = "{0}/data/info/{1}/{2}/{3}".format(workdir, proj, bug, filename)
        dest = "{0}/result/info/{1}".format(workdir, filename)
        try:
            os.makedirs("{0}/result/info".format(workdir))
        except OSError as e:
            if e.errno != errno.EEXIST:
                raise

        shutil.copy(src, dest)
    if mem == None:
        cmd("java -jar {0}/falo/falo/target/falo-0.1-jar-with-dependencies.jar '{0}' '{1}'"
            .format(workdir, tests))
    else:
        cmd("java -Xmx{2}g -jar {0}/falo/falo/target/falo-0.1-jar-with-dependencies.jar '{0}' '{1}'"
            .format(workdir, tests, mem))
Exemple #29
0
def start_mzbench_server():
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{mzbench_rsync, "{0}"}}'.format(os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{mzbench_git, "{0}"}}'.format(os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write('[{{mzbench_api, [{0}]}}].'.format(node_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(mzbench_script, dirname))
    try:
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        print cmd('cat ' + mzbench_dir + '/server/log/console.log').replace('\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))
Exemple #30
0
def start_mzbench_server():
    if 'MZBENCH_RSYNC' in os.environ:
        node_location_param = '{{mzbench_rsync, "{0}"}}'.format(
            os.environ['MZBENCH_RSYNC'])
    elif 'MZBENCH_REPO' in os.environ:
        node_location_param = '{{mzbench_git, "{0}"}}'.format(
            os.environ['MZBENCH_REPO'])
    else:
        node_location_param = ''

    with open(dirname + "/test_server.config", "w") as config:
        config.write('[{{mzbench_api, [{0}]}}].'.format(node_location_param))

    with open('{0}/test_server.config'.format(dirname), 'r') as f:
        print(f.read())

    cmd('{0} start_server --config {1}/test_server.config'.format(
        mzbench_script, dirname))
    try:
        yield
    except:
        print ''
        print '-------------------- >> begin server logs << ---------------------'
        print cmd('cat ' + mzbench_dir + '/server/log/console.log').replace(
            '\\n', '\n')
        print '-------------------- >> end server logs   << ---------------------'
        print ''
        raise
    finally:
        cmd('{0} stop_server'.format(mzbench_script))
Exemple #31
0
    def merge_kernel(self):
        args = ['--build_name', self.build_name,
            '--kernel_pkg', '\'%s\'' % (self.kernel.kernel_pkg,)]

        cmdline = '%s/kernel.sh %s' % (
            self.env['INHIBITOR_SCRIPT_ROOT'],
            ' '.join(args) )

        env = {}
        env.update(self.env)
        env['ROOT'] = '/tmp/inhibitor/kernelbuild'

        if self.seed:
            util.mkdir( self.target_root.pjoin(env['ROOT']) )
            util.chroot(
                path        = self.target_root,
                function    = util.cmd,
                fargs       = {'cmdline':cmdline, 'env':env},
                failuref    = self.chroot_failure,
            )

            # Grab any modules or firmware and put them into the embedded root fs.
            for d in ('modules', 'firmware'):
                util.chroot(
                    path        = self.target_root,
                    function    = util.cmd,
                    fargs       = {'cmdline': 'rsync -a --delete-after %s %s/' % (
                                        '/tmp/inhibitor/kernelbuild/lib/%s' % (d,),
                                        self.target_root.pjoin('lib'))},
                    failuref    = self.chroot_failure,
                )

        else:
            util.cmd( cmdline, env )

            # Grab any modules or firmware and put them into the embedded root fs.
            for d in ('modules', 'firmware'):
                util.cmd('rsync -a --delete-after %s %s/' % (
                    '/tmp/inhibitor/kernelbuild/lib/%s' % (d,),
                    self.target_root.pjoin('lib')))
Exemple #32
0
def devtool_run_local_tests():
    run_erl_and_bdl('validate', 'loop_rate')

    run_erl_and_bdl('validate', 'env',
                    ' --env pool_size=20 --env jozin=jozin --env wait_ms=100')

    run_erl_and_bdl('run_local', 'loop_rate')

    run_erl_and_bdl('run_local', 'data_script')

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir +
            'syntax_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass

    try:
        cmd(mzbench_dir + 'bin/mzbench run_local ' + scripts_dir +
            'semantic_error.erl')
        assert False
    except subprocess.CalledProcessError:
        pass
def systemRefresh():
    """
    Up
    """
    log('UPDATING SYSTEM SPICE')

    chdir('/home/ec2-user/spicerackclient')
    cmd('git pull')
    cmd('tar -cvf /home/ec2-user/system.tar system')
    chdir('/home/ec2-user')
    
    currentversion = dget('systemversion', 0)
    currentversion = 1 + int(currentversion)
    put('systemversion', currentversion)

    with open('systemversion.txt', 'w') as f:
        f.write(str(currentversion))

    cmd('tar --append --file=system.tar systemversion.txt')

    log('UPDATED SYSTEM SPICE TO VERSION: %s' % currentversion)
    return 'success'
Exemple #34
0
    def init(self):
        self.clean_cache()

        if os.path.isdir(self.gitdir):
            util.cmd('git reset --hard HEAD',   env=self.env, chdir=self.cachedir)
            util.cmd('git clean -f -d -x',      env=self.env, chdir=self.cachedir)
            util.cmd('git checkout master',     env=self.env, chdir=self.cachedir)
            util.cmd('git pull',                env=self.env, chdir=self.cachedir)
        else:
            util.cmd('git clone %s %s' % (self.src, self.cachedir))

        _, branches = util.cmd_out('git branch -l', env=self.env, chdir=self.cachedir)
        if 'inhibitor' in branches:
            util.cmd('git branch -D inhibitor', env=self.env, chdir=self.cachedir)

        if self.rev != 'HEAD':
            util.cmd('git checkout -b inhibitor %s' % self.rev, env=self.env, chdir=self.cachedir)
        else:
            _, self.rev = util.cmd_out('git rev-parse HEAD', env=self.env)
            self.rev = self.rev[:7]
Exemple #35
0
#!/usr/bin/env python
import os, util

util.chdir()

ports = util.get_ports()

if not os.path.exists('rethinkdb_data'):
    util.cmd('rethinkdb create -d "rethinkdb_data"')

util.cmd('rethinkdb serve --cluster-port {cluster_port} --driver-port {driver_port} --no-http-admin'.format(
        driver_port=ports['rethinkdb'], cluster_port=ports['rethinkdb_cluster']))
#!/usr/bin/env python

import os, sys

path = os.path.split(os.path.realpath(__file__))[0]
os.chdir(path)
sys.path.insert(0, path)

import util

util.chdir()
util.cmd(
    "cd ../../ && git submodule update --init && . smc-env && cd examples && env OUTDIR=../webapp-lib/examples make && cd .. && npm run webpack-watch"
)
Exemple #37
0
#!/usr/bin/env python

import util

util.chdir()
util.cmd("cd ../../; npm run webpack-watch-map")
Exemple #38
0
import sys
sys.path.append("ext")
import webbrowser
import os
import util, calc, twitter
from time import strftime

util.cls()
txt = raw_input("> ").lower()
opt = util.cmd(txt)

class Bot():
	while "bye" not in opt: 
		if "hi" in opt:
			print "Hi there! The time is " + strftime("%H:%M:%S")
		elif "today" in opt:
			print "Now is", strftime("%A, %d %b %Y %X")
		elif "irc" in opt:
			os.system("irssi")
		elif "thanks" in opt:
			print "You're welcome"
		elif "google" in opt:
			keyword = util.content(opt,"google")
			if len(keyword) >= 1:
				util.google(keyword)
			else:
				print "No input entered."
		elif "wiki" in opt:
			keyword = util.content(opt,"wiki")
			if len(keyword) >= 1:
				util.wiki(keyword)
Exemple #39
0
#!/usr/bin/env python

import os, sys

path = os.path.split(os.path.realpath(__file__))[0]
os.chdir(path)
sys.path.insert(0, path)

import util

util.chdir()
util.cmd(
    "cd ../../ && git submodule update --init && . smc-env && npm run webpack-watch"
)
Exemple #40
0
 def _cmd_list_network(self):
     return cmd("sudo wpa_cli list_networks")
Exemple #41
0
 def process(self, opt, arg, cli):
     # FIXME cd 命令不会生效,我看了下ipython中也一样,所以就不处理了
     log.log(cmd(' '.join(arg)))
Exemple #42
0
 def _cmd_reconfigure_network(self):
     return cmd("sudo wpa_cli reconfigure")
Exemple #43
0
 def _cmd_add_network(self):
     return cmd("sudo wpa_cli add_network")
Exemple #44
0
def log_compression_test():
    bench_id = run_successful_bench(scripts_dir + 'correct_script.erl')
    log_cmd = 'curl --head -X GET http://localhost:4800/log?id={0}'.format(bench_id)
    assert("content-encoding: deflate" in cmd(log_cmd))
Exemple #45
0
def run_bench(name=None, worker_package_with_default_scenario=None, nodes=None, 
        workers_per_node=None, env={}, email=None, should_fail=False, max_retries=2,
        exclusive_node_usage=False, expected_log_message_regex=None,
        check_log_function=None, post_start=None):

    email_option = ('--email=' + email) if email else ''

    if workers_per_node:
        nodes_option = '--workers_per_node ' + str(workers_per_node)
    else:
        if nodes:
            nodes_option = '--nodes ' + ','.join(nodes)
        else:
            nodes_option = '--nodes 1'

    env_option = ' '.join(('--env={0}={1}'.format(k, v)
        for k, v in env.iteritems()))

    def run():
        if 'worker_branch' in env:
            node_commit_arg = '--node_commit={0}'.format(env['worker_branch'])
        else:
            node_commit_arg = ''

        flags = ' '.join([
            '--host=localhost:4800',
            '--exclusive_node_usage=' + ('true' if exclusive_node_usage else 'false'),
            node_commit_arg,
            nodes_option,
            env_option,
            email_option])

        if name is not None:
            invocation = mzbench_dir + 'bin/mzbench ' + flags + ' start ' + name
        elif worker_package_with_default_scenario is not None:
            invocation = mzbench_dir + 'bin/mzbench ' + flags + ' start_default_scenario_of_worker ' + worker_package_with_default_scenario
        else:
            raise RuntimeError('Neither script filename nor default scenario package provided.')

        start = subprocess.Popen(shlex.split(invocation.encode('ascii')),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        start_out, start_err = start.communicate()

        try:
            bench_id = json.loads(start_out)['id']
        except Exception:
            print 'mzbench returned invalid json: \nCommand: {0}\nOutput: {1}\nStderr: {2}'.format(invocation, start_out, start_err)
            raise

        if (post_start is not None) and wait_status(bench_id, 'running', 50):
            print "Calling post start for {0}".format(bench_id)
            post_start(bench_id)

        wait = subprocess.Popen(shlex.split(
            mzbench_dir + 'bin/mzbench --host=localhost:4800 status --wait {0}'.format(bench_id)),
            stdout=subprocess.PIPE,
            stderr=subprocess.PIPE)
        wait.communicate()

        return (bench_id, wait.returncode == 0)

    attempt = 0

    while attempt < max_retries:

        print 'Attempt #{0}'.format(attempt)

        try:
            (bench_id, success) = run()
        except Exception as e:
            print "Unexpected error: {0}".format(e)
            bench_id, success = (None, False)

        if xor(success, should_fail):
            if not expected_log_message_regex and not check_log_function:
                # no need to check the log
                return bench_id

            log_cmd = mzbench_dir + 'bin/mzbench --host=localhost:4800 log {0}'.format(bench_id)
            log = cmd(log_cmd)

            re_match = None
            if expected_log_message_regex:
                if isinstance(expected_log_message_regex, str) or isinstance(expected_log_message_regex, unicode):
                    regex = re.compile(expected_log_message_regex, re.DOTALL + re.UNICODE)
                else:
                    regex = expected_log_message_regex
                re_match = regex.search(log)

            maybe_error = None
            if check_log_function:
                maybe_error = check_log_function(log)

            if not re_match or maybe_error:
                print
                if maybe_error:
                    print "Log doesn't pass custom check:\n{0}\n\n".format(maybe_error)
                if not re_match:
                    print u"Log doesn't contain expected log message '{0}':\n".format(regex.pattern)
                print log
                raise RuntimeError

            return bench_id

        print 'Attempt #{0} for bench-id {1} unexpectedly {2}, retrying.'.format(attempt, bench_id, 'succeeded' if should_fail else 'failed')
        attempt += 1

    if (max_retries <= attempt):
        print('All {0} attempts failed'.format(max_retries))
        print('Log of the last attempt (bench {0}):'.format(bench_id))

        if bench_id is not None:
            log_cmd = mzbench_dir + 'bin/mzbench --host=localhost:4800 log {0}'.format(bench_id)
            print cmd(log_cmd).replace('\\n', '\n')

        raise RuntimeError('BenchId {0} for test {1} unexpectedly {2}'.format(
                bench_id, name, 'succeeded' if should_fail else 'failed'))
Exemple #46
0
 def do_render(self):
     num_msgs = cmd('notmuch', 'count', '--output=messages', self.query)
     return self.fmt.format(num_msgs) if int(num_msgs) > 0 else 'No mail'
Exemple #47
0
#!/usr/bin/env python

import os, json, socket, sys, util

path = os.path.split(os.path.realpath(__file__))[0]; os.chdir(path); sys.path.insert(0, path)

os.environ['DEVEL']='yes'
os.environ['PGHOST']=os.path.join(path, 'postgres_data/socket')

if 'TMUX' in os.environ: # see https://github.com/sagemathinc/cocalc/issues/563
    del os.environ['TMUX']

util.chdir()

ports    = util.get_ports()
base_url = util.base_url()
hostname = 'localhost'

cmd = "service_hub.py --dev --foreground --hostname={hostname} --port={hub_port} --proxy_port=0 --gap=0 --base_url={base_url} start".format(
    hostname      = hostname,
    base_url      = base_url,
    hub_port      = ports['hub'])

util.cmd(cmd)


Exemple #48
0
def _build():
    bdb = os.path.join(PATCHER, "build.bat")
    util.cmd(bdb)
Exemple #49
0
#!/usr/bin/env python

"""
This is a script for starting postgres for development purposes on a laptop.
"""

import os, sys, time, util

path = os.path.split(os.path.realpath(__file__))[0]; os.chdir(path); sys.path.insert(0, path)

PG_DATA = os.path.join(path, "postgres_data")

if not os.path.exists(PG_DATA):
    util.cmd("pg_ctl init -D '%s'"%PG_DATA)

    # Lock down authentication so it is ONLY via unix socket
    open(os.path.join(PG_DATA,'pg_hba.conf'), 'w').write(
"""
# This is safe since we only enable a socket protected by filesystem permissions:
local all all trust

# You can uncomment this and comment out the above if you want to test password auth.
#local all all md5
""")

    # Make it so the socket is in this subdirectory, so that it is
    # protected by UNIX permissions.  This approach avoids any need
    # for accounts/passwords for development and the Docker image.
    conf = os.path.join(PG_DATA, 'postgresql.conf')
    s = open(conf).read()
    s += '\n'
Exemple #50
0
def add_tabsint_plugins(pluginDict, src=True, docs=True, cordova=True):
    """ add tabsint plugins listed in config.json """
    try:
        for name, plugin in pluginDict.iteritems():
            # defaults
            if "version" not in plugin:
                plugin["version"] = None
            if "debug" not in plugin:
                plugin["debug"] = False

            # version logic
            if plugin["debug"]:
                version = None
                ut.log.info(
                    '[WARNING]: Installing plugin "'
                    + name
                    + '" in debug mode. Ignoring version field and pulling from the HEAD or current working copy'
                )
            elif plugin["version"] == None or plugin["version"] == "":
                rm_tabsint_plugins()
                sys.exit(
                    '[ERROR]: No "version" value specified for the "{0}" plugin. This field is required.'.format(name)
                )
            else:
                version = plugin["version"]

            # src
            if src:
                if "src" in plugin:
                    get_tabsint_plugin(name, plugin["src"], version)
                else:
                    sys.exit(
                        '[ERROR]: Tabsint plugin "{0}" specified, but no source is included in the "src" field'.format(
                            name
                        )
                    )

            # docs
            if docs:
                if "docs" in plugin:
                    get_tabsint_plugin_docs(name, plugin["docs"], version)
                else:
                    ut.log.info('[WARNING]: Tabsint plugin "{0}" has no documentation'.format(name))

            # cordova plugins
            if cordova and "cordova" in plugin:
                if "https://" in plugin["cordova"]["src"]:  # remote git-based cordova plugins
                    if "#:/" in plugin["cordova"]["src"]:
                        url = "#{0}:/".format(version).join(plugin["cordova"]["src"].split("#:/"))
                    else:
                        url = "{0}#{1}".format(plugin["cordova"]["src"], version)
                else:  # local directory-based cordova plugins
                    url = plugin["cordova"]["src"]

                ut.log.debug("[DEBUG]: Downloading cordova plugin from url: {0}".format(url))
                ut.cmd("npm run cordova --silent -- plugin add " + url)
                ut.log.info(
                    '[BUILD]: Successfully installed the cordova plugin associated with tabsint plugin "{0}"'.format(
                        name
                    )
                )

        write_plugin_require(pluginDict)

    except Exception as e:
        rm_tabsint_plugins()
        sys.exit("[ERROR]: Failure while adding tabsint plugins with exception: \n\n {0}".format(str(e)))
Exemple #51
0
 def _cmd_add_ssid(self, network_id, ssid):
     return cmd("sudo wpa_cli set_network {} ssid {}".format(network_id, toHex(ssid)))
Exemple #52
0
 def _cmd_save_config(self):
     return cmd("sudo wpa_cli save_config")
Exemple #53
0
def restart_test():
    bench_id = run_successful_bench(scripts_dir + 'correct_script.erl')
    restarted_id = restart_bench(bench_id)
    cmd(mzbench_dir + 'bin/mzbench status --wait {0}'.format(restarted_id))
Exemple #54
0
 def _cmd_add_psk(self, network_id, psk=None):
     if psk is None or len(psk) == 0:
         return cmd("sudo wpa_cli set_network {} key_mgmt NONE".format(network_id))
     else:
         return cmd("sudo wpa_cli set_network {} psk '\"{}\"' & sudo wpa_cli set_network {} key_mgmt WPA-PSK ".format(network_id, psk, network_id))
Exemple #55
0
#!/usr/bin/env python

import os, sys

path = os.path.split(os.path.realpath(__file__))[0]; os.chdir(path); sys.path.insert(0, path)

import util

util.chdir()
util.cmd("cd ../../; npm run webpack-watch-map")
Exemple #56
0
 def _cmd_enable_network(self, network_id):
     try:
         _id = int(network_id)
         return cmd("sudo wpa_cli enable_network {}".format(_id))
     except Exception as ex:
         pass
Exemple #57
0
"""
Start

"""

import os, json, socket, sys, util

path = os.path.split(os.path.realpath(__file__))[0]; os.chdir(path); sys.path.insert(0, path)

os.environ['DEVEL']='yes'
os.environ['PGHOST']=os.path.join(path, 'postgres_data/socket')

util.chdir()

ports    = util.get_ports()
base_url = util.base_url(ports['hub-share-2'])

print('''\n\nBASE URL: {}\n\n'''.format(base_url))

share_path= os.path.join(os.environ['SMC_ROOT'], 'data/projects/[project_id]')

cmd = "cd ../../ && . smc-env &&  service_hub.py --share_path={share_path} --foreground --hostname=0.0.0.0 --port=0 --share_port={share_port} --proxy_port=0 --gap=0 --base_url={base_url} start".format(
    base_url   = base_url,
    share_port = ports['hub-share-2'],
    share_path = share_path)

util.cmd(cmd)


Exemple #58
0
 def _cmd_select_network(self, network_id):
     return cmd("sudo wpa_cli select_network {}".format(network_id))