def update(filepath, github_account, discovery_documents):
"""Updates the google-api-php-client-services repository.
Args:
filepath (str): the directory to work in.
github_account (GitHubAccount): the GitHub account to commit with.
discovery_documents (dict(str, str)): a map of API IDs to Discovery
document filenames to generate from.
"""
repo = _git.clone_from_github(
_REPO_PATH, join(filepath, _REPO_NAME), github_account=github_account)
venv_filepath = join(repo.filepath, 'venv')
check_output(['virtualenv', venv_filepath, '-p', 'python2.7'])
# The PHP client library generator is published in the
# "google-apis-client-generator" package.
check_output([join(venv_filepath, 'bin/pip'),
'install',
'google-apis-client-generator==1.6.1'])
added, updated = _generate_and_commit_all_clients(
repo, venv_filepath, discovery_documents)
commit_count = len(added) + len(updated)
if commit_count == 0:
return
_run_tests(repo)
repo.soft_reset('HEAD~{}'.format(commit_count))
commitmsg = _commit_message.build(added, None, updated)
repo.commit(commitmsg, github_account.name, github_account.email)
repo.push()
def update(filepath, github_account):
"""Updates the google-api-go-client repository.
Args:
filepath (str): the directory to work in.
github_account (GitHubAccount): the GitHub account to commit with.
"""
env = os.environ.copy()
env['GO111MODULE'] = 'on'
repo = _git.clone('https://code.googlesource.com/google-api-go-client',
join(filepath, 'google-api-go-client'))
generator_filepath = join(repo.filepath, 'google-api-go-generator')
check_output(['make', 'all'], cwd=generator_filepath, env=env)
repo.add(['.'])
added, deleted, updated = set(), set(), set()
status_to_ids = {
_git.Status.ADDED: added,
_git.Status.DELETED: deleted,
_git.Status.UPDATED: updated
}
for filename, status in repo.diff_name_status():
match = _NAME_VERSION_RE.match(filename)
if not match:
continue
name_version = '{}/{}'.format(match.group(1), match.group(2))
status_to_ids.get(status, set()).add(name_version)
if not any([added, deleted, updated]):
return
check_output(['go', 'test', './...'], cwd=repo.filepath, env=env)
subject = 'all: autogenerated update ({})'.format(date.today().isoformat())
commitmsg = _commit_message.build(added, deleted, updated, subject=subject)
repo.commit(commitmsg, github_account.name, github_account.email)
repo.push(remote=_REMOTE_URL, nokeycheck=True)
def update(filepath, github_account):
"""Updates the discovery-artifact-manager repository.
Args:
filepath (str): the directory to work in.
github_account (GitHubAccount): the GitHub account to commit and push
with.
"""
repo = _git.clone_from_github(_REPO_PATH,
join(filepath, _REPO_NAME),
github_account=github_account)
with TemporaryDirectory() as gopath:
os.makedirs(join(gopath, 'src'))
check_output([
'ln', '-s',
join(repo.filepath, 'src'),
join(gopath, 'src/discovery-artifact-manager')
])
env = os.environ.copy()
env['GOPATH'] = gopath
check_output(['go', 'run', 'src/main/updatedisco/main.go'],
cwd=repo.filepath,
env=env)
repo.add(['discoveries'])
if not repo.diff_name_status():
return
repo.commit('Autogenerated Discovery document update', github_account.name,
github_account.email)
repo.push()
def push(self,
remote='origin',
branch='master',
tags=False,
nokeycheck=False):
"""Updates remote refs.
Args:
remote (str): the remote name.
branch (str): the branch name.
tags (bool, optional): if true, only tags are pushed.
nokeycheck (bool, optional): if true, the
"--push-option nokeycheck" flag is passed.
Raises:
CallError: if the call returns a non-zero return code.
"""
args = ['git', 'push', remote]
if tags:
args.append('--tags')
else:
args.append(branch)
if nokeycheck:
args.extend(['--push-option', 'nokeycheck'])
check_output(args, cwd=self.filepath)
def test_check_output_error_stderr():
program = ('import sys;'
'print("stderr stuff", file=sys.stderr);'
'quit(1)')
with pytest.raises(CallError) as excinfo:
check_output(['python3', '-c', program])
assert str(excinfo.value) == '\nstderr:\nstderr stuff'
def test_check_output_env():
env = os.environ.copy()
env.pop('TESTVAR123', None)
program = 'import os;print(os.environ["TESTVAR123"]);'
with pytest.raises(CallError):
check_output(['python3', '-c', program])
env['TESTVAR123'] = 'TEST'
stdoutdata = check_output(['python3', '-c', program], env=env)
assert stdoutdata == 'TEST\n'
def tag(self, name):
"""Creates a tag.
Args:
name (str): the name of the tag.
Raises:
CallError: if the call returns a non-zero return code.
"""
check_output(['git', 'tag', name], cwd=self.filepath)
def checkout_new(self, branch):
"""Create a new branch and switch to it.
Args:
branch (str): the name of the branch to checkout.
Raises:
CallError: if the call returns a non-zero return code.
"""
check_output(['git', 'checkout', '-b', branch], cwd=self.filepath)
def add(self, filepaths):
"""Add file contents to the index.
Args:
paths (list(str)): a list of filepaths to add content from.
Raises:
CallError: if the call returns a non-zero return code.
"""
check_output(['git', 'add', *filepaths], cwd=self.filepath)
def checkout(self, branch):
"""Switches branches.
Args:
branch (str): the name of the branch to checkout.
Raises:
CallError: if the call returns a non-zero return code.
"""
check_output(['git', 'checkout', branch], cwd=self.filepath)
def _package_and_push_gem(repo, rubygems_account, new_version):
check_output(['./script/package'], cwd=repo.filepath)
credentials_filename = os.path.expanduser('~/.gem/credentials')
with open(credentials_filename, 'w') as file_:
file_.write('---\n:rubygems_api_key: {}\n'.format(
rubygems_account.api_key))
# The credentials file must have permissions of `0600`.
os.chmod(credentials_filename, 0o600)
check_output(
['gem', 'push', 'pkg/google-api-client-{}.gem'.format(new_version)],
cwd=repo.filepath)
def soft_reset(self, rev):
"""Soft resets current HEAD to `rev`.
Args:
rev (str): a revision parameter. For example: "d1f3ffe7",
or "0.13.2".
mode (str, optional): the type of reset to perform.
Raises:
CallError: if the call returns a non-zero return code.
"""
args = ['git', 'reset', '--soft', rev]
check_output(args, cwd=self.filepath)
def _update_and_publish_gh_pages(repo, new_version, github_account):
check_output(['npm', 'run', 'doc'], cwd=repo.filepath)
repo.checkout('gh-pages')
check_output(['rm', '-rf', 'latest'], cwd=repo.filepath)
doc_filepath = 'doc/googleapis/{}'.format(new_version)
check_output(['cp', '-r', doc_filepath, 'latest'], cwd=repo.filepath)
check_output(['cp', '-r', doc_filepath, new_version], cwd=repo.filepath)
index_md_filename = join(repo.filepath, 'index.md')
lines = []
with open(index_md_filename) as file_:
lines = file_.readlines()
# index.md should be at least 5 lines long and have the first bullet
# (latest) on line 4.
if len(lines) < 5 or lines[3] != '\n' or not lines[4].startswith('*'):
raise Exception('index.md has an unexpected format')
lines[4] = lines[4].replace(' (latest)', '', 1)
bullet = ('* [v{nv} (latest)]'
'(http://google.github.io/google-api-nodejs-client'
'/{nv}/index.html)\n').format(nv=new_version)
lines.insert(4, bullet)
with open(index_md_filename, 'w') as file_:
file_.write(''.join(lines))
repo.add(['latest', new_version])
repo.commit(new_version, github_account.name, github_account.email)
repo.push(branch='gh-pages')
repo.checkout('master')
def _generate_all_clients(repo):
check_output(['rm', '-rf', 'generated'], cwd=repo.filepath)
# The `discovery_v1` service is used by the generator.
check_output([
'git', 'checkout', 'generated/google/apis/discovery_v1.rb',
'generated/google/apis/discovery_v1'
],
cwd=repo.filepath)
check_output(['./script/generate'], cwd=repo.filepath)
# Temporarily disable generation of health care services.
check_output([
'rm', '-rf', 'generated/google/apis/healthcare_v1alpha2',
'generated/google/apis/healthcare_v1alpha2.rb',
'generated/google/apis/healthcare_v1alpha',
'generated/google/apis/healthcare_v1alpha.rb'
],
cwd=repo.filepath)
added, deleted, updated = set(), set(), set()
status_to_ids = {
_git.Status.ADDED: added,
_git.Status.DELETED: deleted,
_git.Status.UPDATED: updated
}
for filename, status in repo.diff_name_status(staged=False):
match = _SERVICE_FILENAME_RE.match(filename)
if not match:
continue
status_to_ids.get(status, set()).add(match.group(1))
return added, deleted, updated
def _generate_all_clients(repo):
check_output(['make', 'generate'], cwd=repo.filepath)
added, deleted, updated = set(), set(), set()
status_to_ids = {
_git.Status.ADDED: added,
_git.Status.DELETED: deleted,
_git.Status.UPDATED: updated
}
for filename, status in repo.diff_name_status(staged=False):
match = _SERVICE_FILENAME_RE.match(filename)
if not match:
continue
name_version = '{}:{}'.format(match.group(1), match.group(2))
status_to_ids.get(status, set()).add(name_version)
return added, deleted, updated
def clone(url, dest):
"""Clones a git Repository.
Args:
url (str): the URL of the repository.
dest (str): the destination filepath.
Raises:
CallError: if the call returns a non-zero return code.
Returns:
Repository: the repository.
"""
check_output(['git', 'clone', url, dest])
return Repository(dest)
def diff_name_status(self, rev=None, staged=True):
"""Returns a list of status, filename pairs of changes from `rev`.
Args:
rev (str, optional): a revision parameter. If set, `staged` is
ignored. For example: "d1f3ffe7", or "0.13.2".
staged (bool, optional): if true, staged changes are returned.
Raises:
CallError: if the call returns a non-zero return code.
Returns:
list((str, Status)): a list of filename, Status pairs of changes
from HEAD.
"""
args = ['git', 'diff', '--name-status']
if rev:
args.append('{}..HEAD'.format(rev))
elif staged:
args.append('--staged')
output = check_output(args, cwd=self.filepath).strip()
pairs = []
if not output:
return pairs
for line in output.split('\n'):
match = _DIFF_NAME_STATUS_RE.match(line)
if not match:
continue
status = {
'A': Status.ADDED,
'D': Status.DELETED,
'M': Status.UPDATED
}.get(match.group(1), Status.UNKNOWN)
pairs.append((match.group(2), status))
return pairs
def _check_latest_version(latest_tag):
output = check_output(['gem', 'search', '-r', '^google-api-client$'])
latest_version = _GEM_SEARCH_RE.match(output).group(1)
if latest_tag != latest_version:
raise Exception(
('latest tag does not match the latest package version on'
' RubyGems: {} != {}').format(latest_tag, latest_version))
def _check_latest_version(latest_tag):
latest_version = check_output(['npm', 'view', 'googleapis', 'version'])
latest_version = latest_version.strip()
if latest_tag != latest_version:
raise Exception(
('latest tag does not match the latest package version on npm:'
' {} != {}').format(latest_tag, latest_version))
def commit(self, message, name, email):
"""Records changes to the repository.
Args:
message (str): the commit message.
name (str): the user's name.
email (str): the user's email.
Raises:
CallError: if the call returns a non-zero return code.
"""
check_output([
'git', '-c', 'user.name={}'.format(name), '-c',
'user.email={}'.format(email), 'commit', '-a',
'--allow-empty-message', '-m', message
],
cwd=self.filepath)
def _update_disco(repo: _git.Repository,
github_account: accounts.GitHubAccount) -> int:
"""Invokes updatedisco on the repo. Returns the number of commits."""
with TemporaryDirectory() as gopath:
os.makedirs(join(gopath, 'src'))
check_output([
'ln', '-s',
join(repo.filepath, 'src'),
join(gopath, 'src/discovery-artifact-manager')
])
env = os.environ.copy()
env['GOPATH'] = gopath
check_output(['go', 'run', 'src/main/updatedisco/main.go'],
cwd=repo.filepath,
env=env)
repo.add(['discoveries'])
if not repo.diff_name_status():
return 0
repo.commit('Autogenerated Discovery document update', github_account.name,
github_account.email)
return 1
def latest_tag(self):
"""Returns the latest tag.
Raises:
CallError: if the call returns a non-zero return code.
Returns:
str: the latest tag.
"""
data = check_output(['git', 'describe', '--tags', '--abbrev=0'],
cwd=self.filepath)
return data.strip()
def _generate_client(repo, venv_filepath, ddoc_filename):
client_filepath = join(repo.filepath, 'src/Google/Service')
with TemporaryDirectory() as dest_filepath:
check_output([join(venv_filepath, 'bin/generate_library'),
'--input={}'.format(ddoc_filename),
'--language=php',
'--language_variant=1.2.1',
'--output_dir={}'.format(dest_filepath)])
dirs = os.listdir(dest_filepath)
client_name = os.path.splitext(dirs[0])[0] # ex: "BigQuery"
old_client_filepath = join(client_filepath, client_name)
old_client_filename = '{}.php'.format(old_client_filepath)
check_output(['rm', '-rf', old_client_filename, old_client_filepath])
new_client_filepath = join(dest_filepath, client_name)
new_client_filename = '{}.php'.format(new_client_filepath)
check_output(['cp', new_client_filename, old_client_filename])
check_output(['cp', '-r', new_client_filepath, old_client_filepath])
def authors_since(self, rev):
"""Returns a list of emails of the authors of all commits since `rev`.
Args:
rev (str): a revision parameter. For example: "d1f3ffe7", or
"0.13.2".
Raises:
CallError: if the call returns a non-zero return code.
Returns:
list(str): a list of emails of the authors of all commits since
`rev`.
"""
data = check_output(
['git', 'log', '{}..HEAD'.format(rev), '--pretty=format:%ae'],
cwd=self.filepath)
# Strip whitespace and filter so '' is never returned as an email.
data = data.strip().split('\n')
return list(filter(None, data))
def _publish_package(repo, npm_account):
with open(os.path.expanduser('~/.npmrc'), 'w') as file_:
file_.write('//registry.npmjs.org/:_authToken={}\n'.format(
npm_account.auth_token))
check_output(['npm', 'publish'], cwd=repo.filepath)
def _install_dependencies(repo):
check_output(['bundle', 'install', '--path', 'vendor/bundle'],
cwd=repo.filepath)
def test_check_output():
stdoutdata = check_output(['echo', 'hello world'])
assert stdoutdata == 'hello world\n'
def _run_tests(repo):
check_output(['bundle', 'exec', 'rake', 'spec'], cwd=repo.filepath)
def test_check_output_error_stdout():
program = 'print("stdout stuff");quit(1)'
with pytest.raises(CallError) as excinfo:
check_output(['python3', '-c', program])
assert str(excinfo.value) == '\nstdout:\nstdout stuff'
def test_check_output_cwd():
with TemporaryDirectory() as filepath:
with open(os.path.join(filepath, 'test'), 'w') as file_:
file_.write('hello world')
stdoutdata = check_output(['cat', 'test'], cwd=filepath)
assert stdoutdata == 'hello world'
