Beispiel #1
0
def mirror(name, repo):
    tempfile.tempdir = app.config['tmp']
    tmpdir = tempfile.mkdtemp()
    repo_url = get_repo_url(repo)
    try:
        tar_file = get_repo_name(repo_url) + '.tar'
        app.log(name, 'Try fetching tarball %s' % tar_file)
        # try tarball first
        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
            call(['wget', os.path.join(app.config['tar-url'], tar_file)])
            call(['tar', 'xf', tar_file], stderr=fnull)
            os.remove(tar_file)
            call(['git', 'config', 'remote.origin.url', repo_url])
            call(['git', 'config', 'remote.origin.mirror', 'true'])
            if call(['git', 'config', 'remote.origin.fetch',
                     '+refs/*:refs/*']) != 0:
                raise BaseException('Did not get a valid git repo')
            call(['git', 'fetch', 'origin'])
    except:
        app.log(name, 'Try git clone from', repo_url)
        with open(os.devnull, "w") as fnull:
            if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
                app.exit(name, 'ERROR: failed to clone', repo)

    with app.chdir(tmpdir):
        if call(['git', 'rev-parse']):
            app.exit(name, 'ERROR: problem mirroring git repo at', tmpdir)

    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    try:
        os.rename(tmpdir, gitdir)
        app.log(name, 'Git repo is mirrored at', gitdir)
    except:
        pass
Beispiel #2
0
def mirror(name, repo):
    tempfile.tempdir = app.config['tmp']
    tmpdir = tempfile.mkdtemp()
    repo_url = get_repo_url(repo)
    try:
        os.makedirs(tmpdir)
        tar_file = get_repo_name(repo_url) + '.tar'
        app.log(name, 'Try fetching tarball %s' % tar_file)
        # try tarball first
        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
            call(['wget', os.path.join(app.config['tar-url'], tar_file)])
            call(['tar', 'xf', tar_file])
            os.remove(tar_file)
            call(['git', 'config', 'remote.origin.url', repo_url])
            call(['git', 'config', 'remote.origin.mirror', 'true'])
            if call(['git', 'config', 'remote.origin.fetch',
                     '+refs/*:refs/*']) != 0:
                raise BaseException('Did not get a valid git repo')
            call(['git', 'fetch', 'origin'])
    except:
        app.log(name, 'Try git clone from', repo_url)
        with open(os.devnull, "w") as fnull:
            if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
                app.exit(name, 'ERROR: failed to clone', repo)

    with app.chdir(tmpdir):
        if call(['git', 'rev-parse']):
            app.exit(name, 'ERROR: problem mirroring git repo at', tmpdir)

    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    try:
        os.rename(tmpdir, gitdir)
        app.log(name, 'Git repo is mirrored at', gitdir)
    except:
        pass
Beispiel #3
0
def mirror(name, repo):
    tempfile.tempdir = app.config['tmp']
    tmpdir = tempfile.mkdtemp()
    repo_url = get_repo_url(repo)
    try:
        tar_file = get_repo_name(repo_url) + '.tar'
        app.log(name, 'Try fetching tarball %s' % tar_file)
        # try tarball first
        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
            call(
                ['wget', os.path.join(app.config['tar-url'], tar_file)],
                stdout=fnull,
                stderr=fnull)
            call(['tar', 'xf', tar_file], stderr=fnull)
            os.remove(tar_file)
            update_mirror(name, repo, tmpdir)
    except:
        app.log(name, 'Try git clone from', repo_url)
        with open(os.devnull, "w") as fnull:
            if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
                app.log(name, 'Failed to clone', repo, exit=True)

    with app.chdir(tmpdir):
        if call(['git', 'rev-parse']):
            app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True)

    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    try:
        shutil.move(tmpdir, gitdir)
        app.log(name, 'Git repo is mirrored at', gitdir)
    except:
        pass
Beispiel #4
0
def mirror(name, repo):
    tempfile.tempdir = app.config['tmp']
    tmpdir = tempfile.mkdtemp()
    repo_url = get_repo_url(repo)
    try:
        tar_file = get_repo_name(repo_url) + '.tar'
        app.log(name, 'Try fetching tarball %s' % tar_file)
        # try tarball first
        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
            call(['wget', os.path.join(app.config['tar-url'], tar_file)])
            call(['tar', 'xf', tar_file], stderr=fnull)
            os.remove(tar_file)
            update_mirror(name, repo, tmpdir)
    except:
        app.log(name, 'Try git clone from', repo_url)
        with open(os.devnull, "w") as fnull:
            if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
                app.log(name, 'Failed to clone', repo, exit=True)

    with app.chdir(tmpdir):
        if call(['git', 'rev-parse']):
            app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True)

    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    try:
        shutil.move(tmpdir, gitdir)
        app.log(name, 'Git repo is mirrored at', gitdir)
    except:
        pass
Beispiel #5
0
def mirror(name, repo):
    tempfile.tempdir = app.config["tmp"]
    tmpdir = tempfile.mkdtemp()
    repo_url = get_repo_url(repo)
    try:
        tar_file = get_repo_name(repo_url) + ".tar"
        app.log(name, "Try fetching tarball %s" % tar_file)
        # try tarball first
        with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
            call(["wget", os.path.join(app.config["tar-url"], tar_file)])
            call(["tar", "xf", tar_file], stderr=fnull)
            os.remove(tar_file)
            call(["git", "config", "remote.origin.url", repo_url])
            call(["git", "config", "remote.origin.mirror", "true"])
            if call(["git", "config", "remote.origin.fetch", "+refs/*:refs/*"]) != 0:
                raise BaseException("Did not get a valid git repo")
            call(["git", "fetch", "origin"])
    except:
        app.log(name, "Try git clone from", repo_url)
        with open(os.devnull, "w") as fnull:
            if call(["git", "clone", "--mirror", "-n", repo_url, tmpdir]):
                app.exit(name, "ERROR: failed to clone", repo)

    with app.chdir(tmpdir):
        if call(["git", "rev-parse"]):
            app.exit(name, "ERROR: problem mirroring git repo at", tmpdir)

    gitdir = os.path.join(app.config["gits"], get_repo_name(repo))
    try:
        os.rename(tmpdir, gitdir)
        app.log(name, "Git repo is mirrored at", gitdir)
    except:
        pass
Beispiel #6
0
    def parse_files(self, directory):
        schemas = self.load_schemas()
        with chdir(directory):
            for dirname, dirnames, filenames in os.walk('.'):
                filenames.sort()
                dirnames.sort()
                if '.git' in dirnames:
                    dirnames.remove('.git')
                for filename in filenames:
                    if filename.endswith(('.def', '.morph')):
                        path = os.path.join(dirname, filename)
                        data = self._load(path)
                        if data is not None:
                            self.validate_schema(schemas, data)
                            data['path'] = path[2:]
                            self._fix_keys(data)
                            self._tidy_and_insert_recursively(data)

        if config.get('mode') == 'parse-only':
            with open(config['result-file'], 'w') as f:
                f.write(
                    yaml.dump(self._data,
                              default_flow_style=False,
                              Dumper=ExplicitDumper))
            log('RESULT', 'Parsed definitions data in yaml format is at',
                config['result-file'])
            os._exit(0)
Beispiel #7
0
    def __init__(self, directory="."):
        """Load all definitions from a directory tree."""
        self._definitions = {}
        self._trees = {}

        json_schema = self._load(app.config.get("json-schema"))
        definitions_schema = self._load(app.config.get("defs-schema"))
        if json_schema and definitions_schema:
            import jsonschema as js

            js.validate(json_schema, json_schema)
            js.validate(definitions_schema, json_schema)

        things_have_changed = not self._check_trees()
        with app.chdir(directory):
            for dirname, dirnames, filenames in os.walk("."):
                filenames.sort()
                dirnames.sort()
                if ".git" in dirnames:
                    dirnames.remove(".git")
                for filename in filenames:
                    if filename.endswith((".def", ".morph")):
                        contents = self._load(os.path.join(dirname, filename))
                        if contents is not None:
                            if things_have_changed and definitions_schema:
                                app.log(filename, "Validating schema")
                                js.validate(contents, definitions_schema)
                            self._fix_keys(contents)
                            self._tidy_and_insert_recursively(contents)

        self.defaults = defaults.Defaults()

        if self._check_trees():
            for path in self._definitions:
                self._definitions[path]["tree"] = self._trees.get(path)
Beispiel #8
0
def get_tree(this):
    ref = this['ref']
    gitdir = os.path.join(app.settings['gits'], get_repo_name(this['repo']))
    if not os.path.exists(gitdir):
        try:
            url = (app.settings['cache-server-url'] + 'repo='
                   + get_repo_url(this['repo']) + '&ref=' + ref)
            with urllib2.urlopen(url) as response:
                tree = json.loads(response.read().decode())['tree']
                return tree
        except:
            app.log(this, 'WARNING: no tree from cache-server', ref)
            mirror(this['name'], this['repo'])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
                stderr=fnull):
            # can't resolve this ref. is it upstream?
            call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)

        try:
            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist

            app.log(this, 'ERROR: could not find tree for ref', ref)
            raise SystemExit
Beispiel #9
0
def mirror(name, repo):
    # try tarball first
    gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
    repo_url = get_repo_url(repo)
    try:
        os.makedirs(gitdir)
        tar_file = get_repo_name(repo_url) + '.tar'
        app.log(name, 'Try fetching tarball %s' % tar_file)
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            call(['wget', app['tar-url']], stdout=fnull, stderr=fnull)
            call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull)
            os.remove(tar_file)
            call(['git', 'config', 'remote.origin.url', repo_url],
                 stdout=fnull, stderr=fnull)
            call(['git', 'config', 'remote.origin.mirror', 'true'],
                 stdout=fnull, stderr=fnull)
            if call(['git', 'config', 'remote.origin.fetch',
                     '+refs/*:refs/*'],
                    stdout=fnull, stderr=fnull) != 0:
                raise BaseException('Did not get a valid git repo')
            call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
    except:
        app.log(name, 'Using git clone from ', repo_url)
        try:
            with open(os.devnull, "w") as fnull:
                call(['git', 'clone', '--mirror', '-n', repo_url, gitdir],
                     stdout=fnull, stderr=fnull)
        except:
            app.log(name, 'ERROR: failed to clone', repo)
            raise SystemExit

    app.log(name, 'Git repo is mirrored at', gitdir)
Beispiel #10
0
def checkout(name, repo, ref, checkout):
    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    if not os.path.exists(gitdir):
        mirror(name, repo)
    elif not mirror_has_ref(gitdir, ref):
        update_mirror(name, repo, gitdir)
    # checkout the required version of this from git
    with open(os.devnull, "w") as fnull:
        # We need to pass '--no-hardlinks' because right now there's nothing to
        # stop the build from overwriting the files in the .git directory
        # inside the sandbox. If they were hardlinks, it'd be possible for a
        # build to corrupt the repo cache. I think it would be faster if we
        # removed --no-hardlinks, though.
        if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
                stdout=fnull, stderr=fnull):
            app.exit(name, 'ERROR: git clone failed for', ref)

        with app.chdir(checkout):
            if call(['git', 'checkout', '--force', ref], stdout=fnull,
                    stderr=fnull):
                app.exit(name, 'ERROR: git checkout failed for', ref)

            app.log(name, 'Git checkout %s in %s' % (repo, checkout))
            app.log(name, 'Upstream version %s' % get_version(checkout, ref))

            if os.path.exists('.gitmodules'):
                checkout_submodules(name, ref)

    utils.set_mtime_recursively(checkout)
Beispiel #11
0
def do_deployment_manifest(system, configuration):
    app.log(system, "Creating deployment manifest in", system['sandbox'])
    data = {'configuration': configuration}
    metafile = os.path.join(system['sandbox'], 'baserock', 'deployment.meta')
    with app.chdir(system['sandbox']), open(metafile, "w") as f:
        json.dump(data, f, indent=4, sort_keys=True, encoding='unicode-escape')
        f.flush()
Beispiel #12
0
def update_mirror(name, repo, gitdir):
    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        app.log(name, 'Refreshing mirror for %s' % repo)
        repo_url = get_repo_url(repo)
        if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
                stdout=fnull, stderr=fnull):
            app.log(name, 'Git update mirror failed', repo, exit=True)
Beispiel #13
0
def checkout(name, repo, ref, checkout):
    gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
    if not os.path.exists(gitdir):
        mirror(name, repo)
    elif not mirror_has_ref(gitdir, ref):
        update_mirror(name, repo, gitdir)
    # checkout the required version of this from git
    with open(os.devnull, "w") as fnull:
        # We need to pass '--no-hardlinks' because right now there's nothing to
        # stop the build from overwriting the files in the .git directory
        # inside the sandbox. If they were hardlinks, it'd be possible for a
        # build to corrupt the repo cache. I think it would be faster if we
        # removed --no-hardlinks, though.
        if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
                stdout=fnull, stderr=fnull):
            app.exit(name, 'ERROR: git clone failed for', ref)

        with app.chdir(checkout):
            if call(['git', 'checkout', '--force', ref], stdout=fnull,
                    stderr=fnull):
                app.exit(name, 'ERROR: git checkout failed for', ref)

            app.log(name, 'Git checkout %s in %s' % (repo, checkout))
            app.log(name, 'Upstream version %s' % get_version(checkout, ref))

            if os.path.exists('.gitmodules'):
                checkout_submodules(name, ref)

    utils.set_mtime_recursively(checkout)
Beispiel #14
0
def get_tree(this):
    ref = this['ref']
    gitdir = os.path.join(app.config['gits'], get_repo_name(this['repo']))

    if not os.path.exists(gitdir):
        try:
            url = (app.config['tree-server'] + 'repo=' +
                   get_repo_url(this['repo']) + '&ref=' + ref)
            response = requests.get(url=url)
            tree = response.json()['tree']
            return tree
        except:
            app.log(this, 'WARNING: no tree from tree-server', ref)
            mirror(this['name'], this['repo'])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
                stderr=fnull):
            # can't resolve this ref. is it upstream?
            app.log(this, 'Fetching from upstream to resolve %s' % ref)
            call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)

        try:
            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist
            app.exit(this, 'ERROR: could not find tree for ref', (ref, gitdir))
Beispiel #15
0
    def __init__(self, directory='.'):
        '''Load all definitions from a directory tree.'''
        self._definitions = {}
        self._trees = {}

        schemas = self.load_schemas()
        with app.chdir(directory):
            for dirname, dirnames, filenames in os.walk('.'):
                filenames.sort()
                dirnames.sort()
                if '.git' in dirnames:
                    dirnames.remove('.git')
                for filename in filenames:
                    if filename.endswith(('.def', '.morph')):
                        path = os.path.join(dirname, filename)
                        data = self._load(path)
                        if data is not None:
                            self.validate_schema(schemas, data)
                            data['path'] = path[2:]
                            self._fix_keys(data)
                            self._tidy_and_insert_recursively(data)

        self.defaults = defaults.Defaults()

        caches_are_valid = self._check_trees()
        for path in self._definitions:
            try:
                this = self._definitions[path]
                if this.get('ref') and self._trees.get(path):
                    if this['ref'] == self._trees.get(path)[0]:
                        this['tree'] = self._trees.get(path)[1]
            except:
                app.log('DEFINITIONS', 'WARNING: problem with .trees file')
                pass
Beispiel #16
0
def set_origin_url(gitdir, checkout):
    '''Sets the origin url of a checkout to that of its gitdir.

    git-lfs requires a remote server in order to fetch binaries, so we set the
    origin url to that of the mirror for lfs enabled checkouts.
    '''
    try:
        with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
            origin_url = check_output(
                ['git', 'config', '--get', 'remote.origin.url'], stderr=fnull)
        with open(os.devnull, 'w') as fnull, app.chdir(checkout):
            check_call(['git', 'config', 'remote.origin.url', origin_url],
                       stderr=fnull)
    except:
        app.log('UTILS', 'Setting origin url failed for', checkout, exit=True)
        raise
Beispiel #17
0
def get_tree(dn):
    ref = dn['ref']
    gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo']))
    if dn['repo'].startswith('file://') or dn['repo'].startswith('/'):
        gitdir = dn['repo'].replace('file://', '')
        if not os.path.isdir(gitdir):
            app.log(dn, 'Git repo not found:', dn['repo'], exit=True)

    if not os.path.exists(gitdir):
        try:
            params = {'repo': get_repo_url(dn['repo']), 'ref': ref}
            r = requests.get(url=app.config['tree-server'], params=params)
            return r.json()['tree']
        except:
            if app.config.get('tree-server'):
                app.log(dn, 'WARNING: no tree from tree-server for', ref)

        mirror(dn['name'], dn['repo'])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
                stderr=fnull):
            # can't resolve ref. is it upstream?
            app.log(dn, 'Fetching from upstream to resolve %s' % ref)
            update_mirror(dn['name'], dn['repo'], gitdir)

        try:
            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist
            app.log(dn, 'No tree for ref', (ref, gitdir), exit=True)
Beispiel #18
0
def get_tree(this):
    ref = this["ref"]
    gitdir = os.path.join(app.config["gits"], get_repo_name(this["repo"]))
    if this["repo"].startswith("file://") or this["repo"].startswith("/"):
        gitdir = this["repo"].replace("file://", "")
        if not os.path.isdir(gitdir):
            app.exit(this, "ERROR: git repo not found:", this["repo"])

    if not os.path.exists(gitdir):
        try:
            url = app.config["tree-server"] + "repo=" + get_repo_url(this["repo"]) + "&ref=" + ref
            response = requests.get(url=url)
            tree = response.json()["tree"]
            return tree
        except:
            if app.config.get("tree-server"):
                app.log(this, "WARNING: no tree from tree-server for", ref)

        mirror(this["name"], this["repo"])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(["git", "rev-parse", ref + "^{object}"], stdout=fnull, stderr=fnull):
            # can't resolve this ref. is it upstream?
            app.log(this, "Fetching from upstream to resolve %s" % ref)
            call(["git", "fetch", "origin"], stdout=fnull, stderr=fnull)

        try:
            tree = check_output(["git", "rev-parse", ref + "^{tree}"], universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist
            app.exit(this, "ERROR: could not find tree for ref", (ref, gitdir))
Beispiel #19
0
def do_manifest(this):
    metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
    with app.chdir(this['install']), open(metafile, "w") as f:
        f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
        f.write('elapsed_time: %s\n' % app.elapsed(this['start-time']))
        f.flush()
        call(['find'], stdout=f, stderr=f)
Beispiel #20
0
def update_mirror(name, repo, gitdir):
    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        app.log(name, 'Refreshing mirror for %s' % repo)
        repo_url = get_repo_url(repo)
        if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
                stdout=fnull,
                stderr=fnull):
            app.log(name, 'Git update mirror failed', repo, exit=True)
Beispiel #21
0
def checkout(this):
    _checkout(this['name'], this['repo'], this['ref'], this['build'])

    with app.chdir(this['build']):
        if os.path.exists('.gitmodules') or this.get('submodules'):
            checkout_submodules(this)

    utils.set_mtime_recursively(this['build'])
Beispiel #22
0
def do_manifest(this):
    metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
    with app.chdir(this['install']), open(metafile, "w") as f:
        f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
        f.flush()
        call(['find'], stdout=f, stderr=f)
    copyfile(metafile, os.path.join(app.settings['artifacts'],
                                    this['cache'] + '.meta'))
Beispiel #23
0
def get_last_tag(gitdir):
    try:
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            tag = check_output(['git', 'describe', '--abbrev=0',
                                '--tags', 'HEAD'], stderr=fnull)[0:-1]
        return tag
    except:
        return None
Beispiel #24
0
def checkout(dn):
    _checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])

    with app.chdir(dn['checkout']):
        if os.path.exists('.gitmodules') or dn.get('submodules'):
            checkout_submodules(dn)

    utils.set_mtime_recursively(dn['checkout'])
Beispiel #25
0
def checkout(this):
    _checkout(this['name'], this['repo'], this['ref'], this['build'])

    with app.chdir(this['build']):
        if os.path.exists('.gitmodules') or this.get('submodules'):
            checkout_submodules(this)

    utils.set_mtime_recursively(this['build'])
Beispiel #26
0
def checkout(dn):
    _checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])

    with app.chdir(dn['checkout']):
        if os.path.exists('.gitmodules') or dn.get('submodules'):
            checkout_submodules(dn)

    utils.set_mtime_recursively(dn['checkout'])
Beispiel #27
0
def do_manifest(this):
    metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
    with app.chdir(this['install']), open(metafile, "w") as f:
        f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
        f.flush()
        call(['find'], stdout=f, stderr=f)
    copyfile(metafile,
             os.path.join(app.config['artifacts'], this['cache'] + '.meta'))
Beispiel #28
0
def get_last_tag(gitdir):
    try:
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            tag = check_output(
                ['git', 'describe', '--abbrev=0', '--tags', 'HEAD'],
                stderr=fnull)[0:-1]
        return tag
    except:
        return None
Beispiel #29
0
def ref_expects_lfs(gitdir, ref):
    '''Parses .gitattributes at a ref to determine if git-lfs is required.'''
    with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
        blob = check_output(
            ['git', 'ls-tree', ref, '.gitattributes'], stderr=fnull)
        if blob:
            attributes = check_output(
                ['git', 'cat-file', 'blob', blob.split()[2]], stderr=fnull)
            return bool(re.search('filter=lfs.*-text', attributes))
    return False
Beispiel #30
0
    def save_trees(self):
        with app.chdir(app.settings['defdir']):
            checksum = check_output('ls -lRA */', shell=True)
        checksum = hashlib.md5(checksum).hexdigest()
        self.__trees = {'.checksum': checksum}
        for name in self.__definitions:
            if self.__definitions[name].get('tree') is not None:
                self.__trees[name] = self.__definitions[name]['tree']

        with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
            f.write(yaml.dump(self.__trees, default_flow_style=False))
Beispiel #31
0
def get_version(gitdir, ref="HEAD"):
    try:
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            described = check_output(["git", "describe", "--tags", "--dirty"], stderr=fnull)[0:-1]
            last_tag = check_output(["git", "describe", "--abbrev=0", "--tags", ref], stderr=fnull)[0:-1]
            commits = check_output(["git", "rev-list", last_tag + ".." + ref, "--count"])[0:-1]
        result = "%s %s (%s + %s commits)" % (ref[:8], described, last_tag, commits)
    except:
        result = ref[:8] + " (No tag found)"

    return result
Beispiel #32
0
def detect_format(source):
    with app.chdir(source):
        for dirname, dirnames, filenames in os.walk('.'):
            if '.git' in dirnames:
                dirnames.remove('.git')
            for filename in filenames:
                if filename.endswith('.morph'):
                    return 'baserock-morphologies'
            for filename in filenames:
                if filename.endswith('.cida'):
                    return 'cida-definitions'
    return None
Beispiel #33
0
def get_version(gitdir, ref='HEAD'):
    try:
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            version = check_output(['git', 'describe', '--tags', '--dirty'],
                                   stderr=fnull)[0:-1]
            tag = check_output(['git', 'describe', '--abbrev=0',
                                '--tags', ref], stderr=fnull)[0:-1]
            commits = check_output(['git', 'rev-list', tag + '..' + ref,
                                    '--count'])[0:-1]
        result = "%s %s (%s + %s commits)" % (ref[:8], version, tag, commits)
    except:
        result = ref[:8] + " (No tag found)"

    return result
Beispiel #34
0
def get_upstream_version(repo, ref):
    try:
        gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            last_tag = check_output(['git', 'describe', '--abbrev=0',
                                      '--tags', ref], stderr=fnull)[0:-1]
            commits = check_output(['git', 'rev-list', last_tag + '..' + ref,
                                    '--count'])

        result = "%s (%s + %s commits)" % (ref[:8], last_tag, commits[0:-1])
    except:
        result = ref[:8] + " " + "(No tag found)"

    return result
Beispiel #35
0
def get_version(gitdir, ref='HEAD'):
    try:
        with app.chdir(gitdir), open(os.devnull, "w") as fnull:
            described = check_output(['git', 'describe', '--tags', '--dirty'],
                                     stderr=fnull)[0:-1]
            last_tag = check_output(['git', 'describe', '--abbrev=0',
                                     '--tags', ref], stderr=fnull)[0:-1]
            commits = check_output(['git', 'rev-list', last_tag + '..' + ref,
                                    '--count'])[0:-1]
        result = "%s %s (%s + %s commits)" % (ref[:8], described, last_tag,
                                              commits)
    except:
        result = ref[:8] + " (No tag found)"

    return result
Beispiel #36
0
 def parse_files(self, directory):
     with chdir(directory):
         for dirname, dirnames, filenames in os.walk('.'):
             filenames.sort()
             dirnames.sort()
             if '.git' in dirnames:
                 dirnames.remove('.git')
             for filename in filenames:
                 if filename.endswith(('.def', '.morph')):
                     path = os.path.join(dirname, filename)
                     data = self._load(path)
                     if data is not None:
                         data['path'] = self._demorph(path[2:])
                         self._fix_keys(data)
                         self._tidy_and_insert_recursively(data)
Beispiel #37
0
 def _check_trees(self):
     try:
         with app.chdir(app.settings['defdir']):
             checksum = check_output('ls -lRA */', shell=True)
         checksum = hashlib.md5(checksum).hexdigest()
         with open('.trees') as f:
             text = f.read()
         self.__trees = yaml.safe_load(text)
         if self.__trees.get('.checksum') == checksum:
             return True
     except:
         if os.path.exists('.trees'):
             os.remove('.trees')
         self.__trees = {}
         return False
Beispiel #38
0
    def save_trees(self):
        '''Creates the .trees file for the current working directory

        .trees contains a list of git trees for all the definitions, and a
        checksum for the state of the working subdirectories
        '''
        with app.chdir(app.config['defdir']):
            checksum = check_output('ls -lRA */', shell=True)
        checksum = hashlib.md5(checksum).hexdigest()
        self._trees = {'.checksum': checksum}
        for name in self._definitions:
            if self._definitions[name].get('tree') is not None:
                self._trees[name] = self._definitions[name]['tree']

        with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
            f.write(yaml.dump(self._trees, default_flow_style=False))
Beispiel #39
0
    def save_trees(self):
        """Creates the .trees file for the current working directory

        .trees contains a list of git trees for all the definitions, and a
        checksum for the state of the working subdirectories
        """
        with app.chdir(app.config["defdir"]):
            checksum = check_output("ls -lRA */", shell=True)
        checksum = hashlib.md5(checksum).hexdigest()
        self._trees = {".checksum": checksum}
        for name in self._definitions:
            if self._definitions[name].get("tree") is not None:
                self._trees[name] = self._definitions[name]["tree"]

        with open(os.path.join(os.getcwd(), ".trees"), "w") as f:
            f.write(yaml.dump(self._trees, default_flow_style=False))
Beispiel #40
0
def checkout(name, repo, ref, checkoutdir):
    gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
    if not os.path.exists(gitdir):
        mirror(name, repo)
    app.log(name, 'Upstream version:', get_upstream_version(repo, ref))
    app.log(name, 'Git checkout %s in %s' % (repo, checkoutdir))
    # checkout the required version of this from git
    with app.chdir(checkoutdir), open(os.devnull, "w") as fnull:
        copy_repo(gitdir, checkoutdir)
        if call(['git', 'checkout', ref], stdout=fnull, stderr=fnull):
            app.log(name, 'ERROR: git checkout failed for', ref)
            raise SystemExit

        if os.path.exists('.gitmodules'):
            checkout_submodules(name, ref)

    utils.set_mtime_recursively(checkoutdir)
Beispiel #41
0
    def save_trees(self):
        '''Creates the .trees file for the current working directory

        .trees contains a list of git trees for all the definitions, and a
        checksum for the state of the working subdirectories
        '''
        with app.chdir(app.config['defdir']):
            checksum = check_output('ls -lRA */', shell=True)
        checksum = hashlib.md5(checksum).hexdigest()
        self._trees = {'.checksum': checksum}
        for name in self._definitions:
            if self._definitions[name].get('tree') is not None:
                self._trees[name] = [
                    self._definitions[name]['ref'],
                    self._definitions[name]['tree']
                ]

        with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
            f.write(yaml.safe_dump(self._trees, default_flow_style=False))
Beispiel #42
0
def log_changes(dn, tmpdir, old_defs, ref):
    do_git_log = False
    old_def = old_defs.get(dn['path'])
    log_file = os.path.join(tmpdir, dn['name'])
    with open(log_file, 'w') as f:
        keys = set(dn) - set(['tree', 'cache'])
        for key in keys:
            try:
                old_value = old_def.get(key)
            except:
                old_value = None
            if dn[key] != old_value:
                f.write('[%s] Value changed: %s\n' % (dn['path'], key))
                if type(dn[key]) is str:
                    f.write('%s | %s\n' % (old_value, dn[key]))
                if type(dn[key]) is not str and type(dn[key]) is not float:
                    if old_value:
                        for x in old_value:
                            f.write(repr(x))
                    f.write('\n                vvv\n')
                    if dn[key]:
                        for x in dn[key]:
                            f.write(repr(x))
                f.write('\n\n')

        if dn.get('kind', 'chunk') == 'chunk' and config.get('release-cmd'):
            log(dn, 'Logging git change history', tmpdir)
            try:
                gitdir = os.path.join(config['gits'],
                                      get_repo_name(dn['repo']))
                if not os.path.exists(gitdir):
                    mirror(dn['name'], dn['repo'])
                elif not mirror_has_ref(gitdir, ref):
                    update_mirror(dn['name'], dn['repo'], gitdir)
                with chdir(gitdir):
                    text = dn['ref'] + '..'
                    if old_def and old_def.get('ref'):
                        text += old_def['ref']
                    f.write(check_output(config['release-command'] + [text]))
            except:
                log(dn, 'WARNING: Failed to log git changes')
    if os.stat(log_file).st_size == 0:
        os.remove(log_file)
Beispiel #43
0
def log_changes(dn, tmpdir, old_defs, ref):
    do_git_log = False
    old_def = old_defs.get(dn['path'])
    log_file = os.path.join(tmpdir, dn['name'])
    with open(log_file, 'w') as f:
        keys = set(dn) - set(['tree', 'cache'])
        for key in keys:
            try:
                old_value = old_def.get(key)
            except:
                old_value = None
            if dn[key] != old_value:
                f.write('[%s] Value changed: %s\n' % (dn['path'], key))
                if type(dn[key]) is str:
                    f.write('%s | %s\n' % (old_value, dn[key]))
                if type(dn[key]) is not str and type(dn[key]) is not float:
                    if old_value:
                        for x in old_value:
                            f.write(repr(x))
                    f.write('\n                vvv\n')
                    if dn[key]:
                        for x in dn[key]:
                            f.write(repr(x))
                f.write('\n\n')

        if dn.get('kind', 'chunk') == 'chunk' and config['release-command']:
            log(dn, 'Logging git change history', tmpdir)
            try:
                gitdir = os.path.join(config['gits'],
                                      get_repo_name(dn['repo']))
                if not os.path.exists(gitdir):
                    mirror(dn['name'], dn['repo'])
                elif not mirror_has_ref(gitdir, ref):
                    update_mirror(dn['name'], dn['repo'], gitdir)
                with chdir(gitdir):
                    text = dn['ref'] + '..'
                    if old_def and old_def.get('ref'):
                        text += old_def['ref']
                    f.write(check_output(config['release-command'] + [text]))
            except:
                log(dn, 'WARNING: Failed to log git changes')
    if os.stat(log_file).st_size == 0:
        os.remove(log_file)
Beispiel #44
0
def run_extension(this, deployment, step, method):
    app.log(this, 'Running %s extension:' % step, method)
    extensions = utils.find_extensions()
    tempfile.tempdir = tmp = app.settings['tmp']
    cmd_tmp = tempfile.NamedTemporaryFile(delete=False)
    cmd_bin = extensions[step][method]

    if method == 'ssh-rsync':
        envlist = ['UPGRADE=yes']
    else:
        envlist = ['UPGRADE=no']

    if 'PYTHONPATH' in os.environ:
        envlist.append('PYTHONPATH=%s:%s' % (os.environ['PYTHONPATH'],
                                             app.settings['extsdir']))
    else:
        envlist.append('PYTHONPATH=%s' % app.settings['extsdir'])

    for key, value in deployment.iteritems():
        if key.isupper():
            envlist.append("%s=%s" % (key, value))

    command = ["env"] + envlist + [cmd_tmp.name]

    if step in ('write', 'configure'):
        command.append(this['sandbox'])

    if step in ('write', 'check'):
        command.append(deployment['location'])

    with app.chdir(app.settings['defdir']):
        try:
            with open(cmd_bin, "r") as infh:
                shutil.copyfileobj(infh, cmd_tmp)
            cmd_tmp.close()
            os.chmod(cmd_tmp.name, 0o700)

            if call(command):
                app.log(this, 'ERROR: %s extension failed:' % step, cmd_bin)
                raise SystemExit
        finally:
            os.remove(cmd_tmp.name)
    return
Beispiel #45
0
def write_chunk_metafile(chunk):
    '''Writes a chunk .meta file to the baserock dir of the chunk

    The split rules are used to divide up the installed files for the chunk
    into artifacts in the 'products' list

    '''
    log(chunk['name'], 'Splitting', chunk.get('kind'))
    rules, splits = compile_rules(chunk)

    with chdir(chunk['install']):
        for root, dirs, files in os.walk('.', topdown=False):
            for name in files + dirs:
                path = os.path.join(root, name)[2:]
                for artifact, rule in rules:
                    if rule.match(path) or rule.match(path + '/'):
                        splits[artifact].append(path)
                        break

    write_metafile(rules, splits, chunk)
Beispiel #46
0
def write_chunk_metafile(chunk):
    '''Writes a chunk .meta file to the baserock dir of the chunk

    The split rules are used to divide up the installed files for the chunk
    into artifacts in the 'products' list

    '''
    log(chunk['name'], 'Splitting', chunk.get('kind'))
    rules, splits = compile_rules(chunk)

    with chdir(chunk['install']):
        for root, dirs, files in os.walk('.', topdown=False):
            for name in files + dirs:
                path = os.path.join(root, name)[2:]
                for artifact, rule in rules:
                    if rule.match(path) or rule.match(path + '/'):
                        splits[artifact].append(path)
                        break

    write_metafile(rules, splits, chunk)
Beispiel #47
0
def do_manifest(defs, this):
    metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
    metadata = {}
    metadata['repo'] = this.get('repo')
    metadata['ref'] = this.get('ref')
    kind = this.get('kind', 'chunk')

    if kind == 'chunk':
        metadata['products'] = splitting.do_chunk_splits(defs, this, metafile)
    elif kind == 'stratum':
        metadata['products'] = splitting.do_stratum_splits(defs, this)

    if metadata.get('products'):
        defs.set_member(this['path'], '_artifacts', metadata['products'])

    with app.chdir(this['install']), open(metafile, "w") as f:
        yaml.safe_dump(metadata, f, default_flow_style=False)

    copyfile(metafile, os.path.join(app.config['artifacts'],
                                    this['cache'] + '.meta'))
Beispiel #48
0
    def __init__(self, directory='.'):
        '''Load all definitions from a directory tree.'''
        self._definitions = {}
        self._trees = {}

        json_schema = self._load(app.config.get('json-schema'))
        definitions_schema = self._load(app.config.get('defs-schema'))
        if json_schema and definitions_schema:
            import jsonschema as js
            js.validate(json_schema, json_schema)
            js.validate(definitions_schema, json_schema)

        things_have_changed = not self._check_trees()
        with app.chdir(directory):
            for dirname, dirnames, filenames in os.walk('.'):
                filenames.sort()
                dirnames.sort()
                if '.git' in dirnames:
                    dirnames.remove('.git')
                for filename in filenames:
                    if filename.endswith(('.def', '.morph')):
                        contents = self._load(os.path.join(dirname, filename))
                        if contents is not None:
                            if things_have_changed and definitions_schema:
                                app.log(filename, 'Validating schema')
                                js.validate(contents, definitions_schema)
                            self._fix_keys(contents)
                            self._tidy_and_insert_recursively(contents)

        self.defaults = defaults.Defaults()

        caches_are_valid = self._check_trees()
        for path in self._definitions:
            try:
                this = self._definitions[path]
                if this.get('ref') and self._trees.get(path):
                    if this['ref'] == self._trees.get(path)[0]:
                        this['tree'] = self._trees.get(path)[1]
            except:
                app.log('DEFINITIONS', 'WARNING: problem with .trees file')
                pass
Beispiel #49
0
def run_extension(dn, deployment, step, method):
    app.log(dn, 'Running %s extension:' % step, method)
    extensions = utils.find_extensions()
    tempfile.tempdir = app.config['tmp']
    cmd_tmp = tempfile.NamedTemporaryFile(delete=False)
    cmd_bin = extensions[step][method]

    envlist = ['UPGRADE=yes'] if method == 'ssh-rsync' else ['UPGRADE=no']

    if 'PYTHONPATH' in os.environ:
        envlist.append('PYTHONPATH=%s:%s' %
                       (os.environ['PYTHONPATH'], app.config['extsdir']))
    else:
        envlist.append('PYTHONPATH=%s' % app.config['extsdir'])

    for key, value in deployment.iteritems():
        if key.isupper():
            envlist.append("%s=%s" % (key, value))

    command = ["env"] + envlist + [cmd_tmp.name]

    if step in ('write', 'configure'):
        command.append(dn['sandbox'])

    if step in ('write', 'check'):
        command.append(
            deployment.get('location') or deployment.get('upgrade-location'))

    with app.chdir(app.config['defdir']):
        try:
            with open(cmd_bin, "r") as infh:
                shutil.copyfileobj(infh, cmd_tmp)
            cmd_tmp.close()
            os.chmod(cmd_tmp.name, 0o700)

            if call(command):
                app.log(dn, 'ERROR: %s extension failed:' % step, cmd_bin)
                raise SystemExit
        finally:
            os.remove(cmd_tmp.name)
    return
Beispiel #50
0
 def parse_files(self, directory):
     with chdir(directory):
         for dirname, dirnames, filenames in os.walk('.'):
             filenames.sort()
             dirnames.sort()
             if '.git' in dirnames:
                 dirnames.remove('.git')
             for filename in filenames:
                 if filename.endswith(('.def', '.morph')):
                     path = os.path.join(dirname, filename)
                     data = self._load(path)
                     if data is not None:
                         data['path'] = self._demorph(path[2:])
                         self._fix_keys(data)
                         self._tidy_and_insert_recursively(data)
     for x in self._data:
         dn = self._data[x]
         for field in dn:
             if field not in self.fields:
                 log(dn, 'Invalid field "%s" in' % field, dn['path'],
                     exit=True)
Beispiel #51
0
def make_deterministic_tar_archive(base_name, root):
    '''Make a tar archive of contents of 'root_dir'.

    This function takes extra steps to make the output more deterministic,
    compared to shutil.make_archive() - it sorts the results to ensure
    the ordering of the files in the archive is always the same.

    Also this puts the directory last, to workaround a bug in docker/overlayfs
    runners - see https://gitlab.com/baserock/ybd/issues/241

    FIXME: make this do timestamps

    '''

    with app.chdir(root), open(base_name + '.tar', 'wb') as f:
        with tarfile.TarFile(mode='w', fileobj=f) as f_tar:
            directories = [d[0] for d in os.walk('.')]
            for d in sorted(directories):
                files = [os.path.join(d, f) for f in os.listdir(d)]
                for path in sorted(files):
                    f_tar.add(name=path, recursive=False)
                f_tar.add(name=d, recursive=False)
Beispiel #52
0
    def _check_trees(self):
        '''True if the .trees file matches the current working subdirectories

        The .trees file lists all git trees for a set of definitions, and a
        checksum of the checked-out subdirectories when we calculated them.

        If the checksum for the current subdirectories matches, return True

        '''
        try:
            with app.chdir(app.config['defdir']):
                checksum = check_output('ls -lRA */', shell=True)
            checksum = hashlib.md5(checksum).hexdigest()
            with open('.trees') as f:
                text = f.read()
            self._trees = yaml.safe_load(text)
            if self._trees.get('.checksum') == checksum:
                return True
        except:
            self._trees = {}

        return False
Beispiel #53
0
def get_tree(this):
    ref = this['ref']
    gitdir = os.path.join(app.config['gits'], get_repo_name(this['repo']))
    if this['repo'].startswith('file://') or this['repo'].startswith('/'):
        gitdir = this['repo'].replace('file://', '')
        if not os.path.isdir(gitdir):
            app.exit(this, 'ERROR: git repo not found:', this['repo'])

    if not os.path.exists(gitdir):
        try:
            url = (app.config['tree-server'] + 'repo=' +
                   get_repo_url(this['repo']) + '&ref=' + ref)
            response = requests.get(url=url)
            tree = response.json()['tree']
            return tree
        except:
            if app.config.get('tree-server'):
                app.log(this, 'WARNING: no tree from tree-server for', ref)

        mirror(this['name'], this['repo'])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
                stderr=fnull):
            # can't resolve this ref. is it upstream?
            app.log(this, 'Fetching from upstream to resolve %s' % ref)
            call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)

        try:
            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist
            app.exit(this, 'ERROR: could not find tree for ref', (ref, gitdir))
Beispiel #54
0
def get_tree(dn):
    ref = str(dn['ref'])
    gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo']))
    if dn['repo'].startswith('file://') or dn['repo'].startswith('/'):
        gitdir = dn['repo'].replace('file://', '')
        if not os.path.isdir(gitdir):
            app.log(dn, 'Git repo not found:', dn['repo'], exit=True)

    if not os.path.exists(gitdir):
        try:
            params = {'repo': get_repo_url(dn['repo']), 'ref': ref}
            r = requests.get(url=app.config['tree-server'], params=params)
            return r.json()['tree']
        except:
            if app.config.get('tree-server'):
                app.log(dn, 'WARNING: no tree from tree-server for', ref)

        mirror(dn['name'], dn['repo'])

    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        if call(['git', 'rev-parse', ref + '^{object}'],
                stdout=fnull,
                stderr=fnull):
            # can't resolve ref. is it upstream?
            app.log(dn, 'Fetching from upstream to resolve %s' % ref)
            update_mirror(dn['name'], dn['repo'], gitdir)

        try:
            tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
                                universal_newlines=True)[0:-1]
            return tree

        except:
            # either we don't have a git dir, or ref is not unique
            # or ref does not exist
            app.log(dn, 'No tree for ref', (ref, gitdir), exit=True)
Beispiel #55
0
def source_date_epoch(checkout):
    with app.chdir(checkout):
        return check_output(['git', 'log', '-1', '--pretty=%ct'])[:-1]
Beispiel #56
0
def fetch(repo):
    with app.chdir(repo), open(os.devnull, "w") as fnull:
        call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
Beispiel #57
0
def update_mirror(name, repo, gitdir):
    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        app.log(name, 'Refreshing mirror for %s' % repo)
        if call(['git', 'remote', 'update', 'origin'], stdout=fnull,
                stderr=fnull):
            app.exit(name, 'ERROR: git update mirror failed', repo)
Beispiel #58
0
def mirror_has_ref(gitdir, ref):
    with app.chdir(gitdir), open(os.devnull, "w") as fnull:
        out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull)
        return out == 0