Пример #1
0
def update(filepath, github_account, discovery_documents):
    """Updates the google-api-php-client-services repository.

    Args:
        filepath (str): the directory to work in.
        github_account (GitHubAccount): the GitHub account to commit with.
        discovery_documents (dict(str, str)): a map of API IDs to Discovery
            document filenames to generate from.
    """
    repo = _git.clone_from_github(
        _REPO_PATH, join(filepath, _REPO_NAME), github_account=github_account)
    venv_filepath = join(repo.filepath, 'venv')
    check_output(['virtualenv', venv_filepath, '-p', 'python2.7'])
    # The PHP client library generator is published in the
    # "google-apis-client-generator" package.
    check_output([join(venv_filepath, 'bin/pip'),
                  'install',
                  'google-apis-client-generator==1.6.1'])
    added, updated = _generate_and_commit_all_clients(
        repo, venv_filepath, discovery_documents)
    commit_count = len(added) + len(updated)
    if commit_count == 0:
        return
    _run_tests(repo)
    repo.soft_reset('HEAD~{}'.format(commit_count))
    commitmsg = _commit_message.build(added, None, updated)
    repo.commit(commitmsg, github_account.name, github_account.email)
    repo.push()
def update(filepath, github_account):
    """Updates the google-api-go-client repository.

    Args:
        filepath (str): the directory to work in.
        github_account (GitHubAccount): the GitHub account to commit with.
    """
    env = os.environ.copy()
    env['GO111MODULE'] = 'on'
    repo = _git.clone('https://code.googlesource.com/google-api-go-client',
                      join(filepath, 'google-api-go-client'))
    generator_filepath = join(repo.filepath, 'google-api-go-generator')
    check_output(['make', 'all'], cwd=generator_filepath, env=env)
    repo.add(['.'])
    added, deleted, updated = set(), set(), set()
    status_to_ids = {
        _git.Status.ADDED: added,
        _git.Status.DELETED: deleted,
        _git.Status.UPDATED: updated
    }
    for filename, status in repo.diff_name_status():
        match = _NAME_VERSION_RE.match(filename)
        if not match:
            continue
        name_version = '{}/{}'.format(match.group(1), match.group(2))
        status_to_ids.get(status, set()).add(name_version)
    if not any([added, deleted, updated]):
        return
    check_output(['go', 'test', './...'], cwd=repo.filepath, env=env)
    subject = 'all: autogenerated update ({})'.format(date.today().isoformat())
    commitmsg = _commit_message.build(added, deleted, updated, subject=subject)
    repo.commit(commitmsg, github_account.name, github_account.email)
    repo.push(remote=_REMOTE_URL, nokeycheck=True)
Пример #3
0
def update(filepath, github_account):
    """Updates the discovery-artifact-manager repository.

    Args:
        filepath (str): the directory to work in.
        github_account (GitHubAccount): the GitHub account to commit and push
            with.
    """
    repo = _git.clone_from_github(_REPO_PATH,
                                  join(filepath, _REPO_NAME),
                                  github_account=github_account)
    with TemporaryDirectory() as gopath:
        os.makedirs(join(gopath, 'src'))
        check_output([
            'ln', '-s',
            join(repo.filepath, 'src'),
            join(gopath, 'src/discovery-artifact-manager')
        ])
        env = os.environ.copy()
        env['GOPATH'] = gopath
        check_output(['go', 'run', 'src/main/updatedisco/main.go'],
                     cwd=repo.filepath,
                     env=env)
    repo.add(['discoveries'])
    if not repo.diff_name_status():
        return
    repo.commit('Autogenerated Discovery document update', github_account.name,
                github_account.email)
    repo.push()
    def push(self,
             remote='origin',
             branch='master',
             tags=False,
             nokeycheck=False):
        """Updates remote refs.

        Args:
            remote (str): the remote name.
            branch (str): the branch name.
            tags (bool, optional): if true, only tags are pushed.
            nokeycheck (bool, optional): if true, the
                "--push-option nokeycheck" flag is passed.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        args = ['git', 'push', remote]
        if tags:
            args.append('--tags')
        else:
            args.append(branch)
        if nokeycheck:
            args.extend(['--push-option', 'nokeycheck'])
        check_output(args, cwd=self.filepath)
Пример #5
0
def test_check_output_error_stderr():
    program = ('import sys;'
               'print("stderr stuff", file=sys.stderr);'
               'quit(1)')
    with pytest.raises(CallError) as excinfo:
        check_output(['python3', '-c', program])
    assert str(excinfo.value) == '\nstderr:\nstderr stuff'
Пример #6
0
def test_check_output_env():
    env = os.environ.copy()
    env.pop('TESTVAR123', None)
    program = 'import os;print(os.environ["TESTVAR123"]);'
    with pytest.raises(CallError):
        check_output(['python3', '-c', program])
    env['TESTVAR123'] = 'TEST'
    stdoutdata = check_output(['python3', '-c', program], env=env)
    assert stdoutdata == 'TEST\n'
    def tag(self, name):
        """Creates a tag.

        Args:
            name (str): the name of the tag.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        check_output(['git', 'tag', name], cwd=self.filepath)
Пример #8
0
    def checkout_new(self, branch):
        """Create a new branch and switch to it.

        Args:
            branch (str): the name of the branch to checkout.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        check_output(['git', 'checkout', '-b', branch], cwd=self.filepath)
    def add(self, filepaths):
        """Add file contents to the index.

        Args:
            paths (list(str)): a list of filepaths to add content from.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        check_output(['git', 'add', *filepaths], cwd=self.filepath)
    def checkout(self, branch):
        """Switches branches.

        Args:
            branch (str): the name of the branch to checkout.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        check_output(['git', 'checkout', branch], cwd=self.filepath)
Пример #11
0
def _package_and_push_gem(repo, rubygems_account, new_version):
    check_output(['./script/package'], cwd=repo.filepath)
    credentials_filename = os.path.expanduser('~/.gem/credentials')
    with open(credentials_filename, 'w') as file_:
        file_.write('---\n:rubygems_api_key: {}\n'.format(
            rubygems_account.api_key))
    # The credentials file must have permissions of `0600`.
    os.chmod(credentials_filename, 0o600)
    check_output(
        ['gem', 'push', 'pkg/google-api-client-{}.gem'.format(new_version)],
        cwd=repo.filepath)
    def soft_reset(self, rev):
        """Soft resets current HEAD to `rev`.

        Args:
            rev (str): a revision parameter. For example: "d1f3ffe7",
                or "0.13.2".
            mode (str, optional): the type of reset to perform.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        args = ['git', 'reset', '--soft', rev]
        check_output(args, cwd=self.filepath)
def _update_and_publish_gh_pages(repo, new_version, github_account):
    check_output(['npm', 'run', 'doc'], cwd=repo.filepath)
    repo.checkout('gh-pages')
    check_output(['rm', '-rf', 'latest'], cwd=repo.filepath)
    doc_filepath = 'doc/googleapis/{}'.format(new_version)
    check_output(['cp', '-r', doc_filepath, 'latest'], cwd=repo.filepath)
    check_output(['cp', '-r', doc_filepath, new_version], cwd=repo.filepath)
    index_md_filename = join(repo.filepath, 'index.md')
    lines = []
    with open(index_md_filename) as file_:
        lines = file_.readlines()
    # index.md should be at least 5 lines long and have the first bullet
    # (latest) on line 4.
    if len(lines) < 5 or lines[3] != '\n' or not lines[4].startswith('*'):
        raise Exception('index.md has an unexpected format')
    lines[4] = lines[4].replace(' (latest)', '', 1)
    bullet = ('* [v{nv} (latest)]'
              '(http://google.github.io/google-api-nodejs-client'
              '/{nv}/index.html)\n').format(nv=new_version)
    lines.insert(4, bullet)
    with open(index_md_filename, 'w') as file_:
        file_.write(''.join(lines))
    repo.add(['latest', new_version])
    repo.commit(new_version, github_account.name, github_account.email)
    repo.push(branch='gh-pages')
    repo.checkout('master')
def _generate_all_clients(repo):
    check_output(['rm', '-rf', 'generated'], cwd=repo.filepath)
    # The `discovery_v1` service is used by the generator.
    check_output([
        'git', 'checkout', 'generated/google/apis/discovery_v1.rb',
        'generated/google/apis/discovery_v1'
    ],
                 cwd=repo.filepath)
    check_output(['./script/generate'], cwd=repo.filepath)
    # Temporarily disable generation of health care services.
    check_output([
        'rm', '-rf', 'generated/google/apis/healthcare_v1alpha2',
        'generated/google/apis/healthcare_v1alpha2.rb',
        'generated/google/apis/healthcare_v1alpha',
        'generated/google/apis/healthcare_v1alpha.rb'
    ],
                 cwd=repo.filepath)
    added, deleted, updated = set(), set(), set()
    status_to_ids = {
        _git.Status.ADDED: added,
        _git.Status.DELETED: deleted,
        _git.Status.UPDATED: updated
    }
    for filename, status in repo.diff_name_status(staged=False):
        match = _SERVICE_FILENAME_RE.match(filename)
        if not match:
            continue
        status_to_ids.get(status, set()).add(match.group(1))
    return added, deleted, updated
def _generate_all_clients(repo):
    check_output(['make', 'generate'], cwd=repo.filepath)
    added, deleted, updated = set(), set(), set()
    status_to_ids = {
        _git.Status.ADDED: added,
        _git.Status.DELETED: deleted,
        _git.Status.UPDATED: updated
    }
    for filename, status in repo.diff_name_status(staged=False):
        match = _SERVICE_FILENAME_RE.match(filename)
        if not match:
            continue
        name_version = '{}:{}'.format(match.group(1), match.group(2))
        status_to_ids.get(status, set()).add(name_version)
    return added, deleted, updated
def clone(url, dest):
    """Clones a git Repository.

    Args:
        url (str): the URL of the repository.
        dest (str): the destination filepath.

    Raises:
        CallError: if the call returns a non-zero return code.

    Returns:
        Repository: the repository.
    """
    check_output(['git', 'clone', url, dest])
    return Repository(dest)
    def diff_name_status(self, rev=None, staged=True):
        """Returns a list of status, filename pairs of changes from `rev`.

        Args:
            rev (str, optional): a revision parameter. If set, `staged` is
                ignored. For example: "d1f3ffe7", or "0.13.2".
            staged (bool, optional): if true, staged changes are returned.

        Raises:
            CallError: if the call returns a non-zero return code.

        Returns:
            list((str, Status)): a list of filename, Status pairs of changes
                from HEAD.
        """
        args = ['git', 'diff', '--name-status']
        if rev:
            args.append('{}..HEAD'.format(rev))
        elif staged:
            args.append('--staged')
        output = check_output(args, cwd=self.filepath).strip()
        pairs = []
        if not output:
            return pairs
        for line in output.split('\n'):
            match = _DIFF_NAME_STATUS_RE.match(line)
            if not match:
                continue
            status = {
                'A': Status.ADDED,
                'D': Status.DELETED,
                'M': Status.UPDATED
            }.get(match.group(1), Status.UNKNOWN)
            pairs.append((match.group(2), status))
        return pairs
Пример #18
0
def _check_latest_version(latest_tag):
    output = check_output(['gem', 'search', '-r', '^google-api-client$'])
    latest_version = _GEM_SEARCH_RE.match(output).group(1)
    if latest_tag != latest_version:
        raise Exception(
            ('latest tag does not match the latest package version on'
             ' RubyGems: {} != {}').format(latest_tag, latest_version))
def _check_latest_version(latest_tag):
    latest_version = check_output(['npm', 'view', 'googleapis', 'version'])
    latest_version = latest_version.strip()
    if latest_tag != latest_version:
        raise Exception(
            ('latest tag does not match the latest package version on npm:'
             ' {} != {}').format(latest_tag, latest_version))
    def commit(self, message, name, email):
        """Records changes to the repository.

        Args:
            message (str): the commit message.
            name (str): the user's name.
            email (str): the user's email.

        Raises:
            CallError: if the call returns a non-zero return code.
        """
        check_output([
            'git', '-c', 'user.name={}'.format(name), '-c',
            'user.email={}'.format(email), 'commit', '-a',
            '--allow-empty-message', '-m', message
        ],
                     cwd=self.filepath)
Пример #21
0
def _update_disco(repo: _git.Repository,
                  github_account: accounts.GitHubAccount) -> int:
    """Invokes updatedisco on the repo.  Returns the number of commits."""
    with TemporaryDirectory() as gopath:
        os.makedirs(join(gopath, 'src'))
        check_output([
            'ln', '-s',
            join(repo.filepath, 'src'),
            join(gopath, 'src/discovery-artifact-manager')
        ])
        env = os.environ.copy()
        env['GOPATH'] = gopath
        check_output(['go', 'run', 'src/main/updatedisco/main.go'],
                     cwd=repo.filepath,
                     env=env)
    repo.add(['discoveries'])
    if not repo.diff_name_status():
        return 0
    repo.commit('Autogenerated Discovery document update', github_account.name,
                github_account.email)
    return 1
    def latest_tag(self):
        """Returns the latest tag.

        Raises:
            CallError: if the call returns a non-zero return code.

        Returns:
            str: the latest tag.
        """
        data = check_output(['git', 'describe', '--tags', '--abbrev=0'],
                            cwd=self.filepath)
        return data.strip()
Пример #23
0
def _generate_client(repo, venv_filepath, ddoc_filename):
    client_filepath = join(repo.filepath, 'src/Google/Service')
    with TemporaryDirectory() as dest_filepath:
        check_output([join(venv_filepath, 'bin/generate_library'),
                      '--input={}'.format(ddoc_filename),
                      '--language=php',
                      '--language_variant=1.2.1',
                      '--output_dir={}'.format(dest_filepath)])
        dirs = os.listdir(dest_filepath)
        client_name = os.path.splitext(dirs[0])[0]  # ex: "BigQuery"
        old_client_filepath = join(client_filepath, client_name)
        old_client_filename = '{}.php'.format(old_client_filepath)
        check_output(['rm', '-rf', old_client_filename, old_client_filepath])
        new_client_filepath = join(dest_filepath, client_name)
        new_client_filename = '{}.php'.format(new_client_filepath)
        check_output(['cp', new_client_filename, old_client_filename])
        check_output(['cp', '-r', new_client_filepath, old_client_filepath])
    def authors_since(self, rev):
        """Returns a list of emails of the authors of all commits since `rev`.

        Args:
            rev (str): a revision parameter. For example: "d1f3ffe7", or
                "0.13.2".

        Raises:
            CallError: if the call returns a non-zero return code.

        Returns:
            list(str): a list of emails of the authors of all commits since
                `rev`.
        """
        data = check_output(
            ['git', 'log', '{}..HEAD'.format(rev), '--pretty=format:%ae'],
            cwd=self.filepath)
        # Strip whitespace and filter so '' is never returned as an email.
        data = data.strip().split('\n')
        return list(filter(None, data))
def _publish_package(repo, npm_account):
    with open(os.path.expanduser('~/.npmrc'), 'w') as file_:
        file_.write('//registry.npmjs.org/:_authToken={}\n'.format(
            npm_account.auth_token))
    check_output(['npm', 'publish'], cwd=repo.filepath)
Пример #26
0
def _install_dependencies(repo):
    check_output(['bundle', 'install', '--path', 'vendor/bundle'],
                 cwd=repo.filepath)
Пример #27
0
def test_check_output():
    stdoutdata = check_output(['echo', 'hello world'])
    assert stdoutdata == 'hello world\n'
Пример #28
0
def _run_tests(repo):
    check_output(['bundle', 'exec', 'rake', 'spec'], cwd=repo.filepath)
Пример #29
0
def test_check_output_error_stdout():
    program = 'print("stdout stuff");quit(1)'
    with pytest.raises(CallError) as excinfo:
        check_output(['python3', '-c', program])
    assert str(excinfo.value) == '\nstdout:\nstdout stuff'
Пример #30
0
def test_check_output_cwd():
    with TemporaryDirectory() as filepath:
        with open(os.path.join(filepath, 'test'), 'w') as file_:
            file_.write('hello world')
        stdoutdata = check_output(['cat', 'test'], cwd=filepath)
        assert stdoutdata == 'hello world'