def createhook(user, repo, skip_email, yes_i_know):
"""Create the hook in repository for given user.
USER can be either an email or a user ID.
REPO can be either the repository name
(e.g. `some-organization/some-repository`) or its GitHub ID.
Examples:
github createhook [email protected] foobar-org/foobar-repo
github createhook 12345 55555
"""
user = resolve_user(user)
repo = resolve_repo(repo)
if repo.user:
click.secho('Hook is already installed for {user}'.format(
user=repo.user), fg='red')
return
msg = "Creating a hook for {user} and {repo}. Continue?".format(
user=user, repo=repo)
if not (yes_i_know or click.confirm(msg)):
click.echo('Aborted.')
gha = GitHubAPI(user_id=user.id)
if not skip_email:
verify_email(user)
gha.create_hook(repo.github_id, repo.name)
db.session.commit()
def assign(user, repos, skip_email, yes_i_know):
"""Assign an already owned repository to another user.
First argument, USER, is the user to whom the repository will be assigned.
Value of USER can be either the email or user's ID.
Arguments REPOS specify one or more repositories, each can be either a
repository names or a repository GitHub IDs.
Examples:
Assign repository 'foobar-org/repo-name' to user '*****@*****.**':
github assign [email protected] foobar-org/repo-name
Assign three GitHub repositories to user with ID 999:
github assign 999 15001500 baz-org/somerepo 12001200
"""
user = resolve_user(user)
repos = resolve_repos(repos)
prompt_msg = 'This will change the ownership for {0} repositories' \
'. Continue?'.format(len(repos))
if not (yes_i_know or click.confirm(prompt_msg)):
click.echo('Aborted.')
return
gha_new = GitHubAPI(user_id=user.id)
if not skip_email:
verify_email(user)
for repo in repos:
gha_prev = GitHubAPI(user_id=repo.user_id)
move_repository(repo, gha_prev, gha_new)
def repo_list(user, sync, with_all, skip_email):
"""List user's repositories hooks.
Lists repositories currently `enabled` by the user.
If `--all` flag is specified, lists full list of repositories from remote
account extra data. For best result `--all` should be used with `--sync`,
so that the GitHub information is fresh. Positional argument USER can be
either an email or user ID.
Examples:
github list [email protected]
github list 12345 --sync --all
"""
user = resolve_user(user)
gha = GitHubAPI(user_id=user.id)
if not skip_email:
verify_email(user)
if sync:
gha.sync(hooks=True, async_hooks=False) # Sync hooks asynchronously
db.session.commit()
if with_all:
repos = gha.account.extra_data['repos']
click.echo("User has {0} repositories in total.".format(len(repos)))
for gid, repo in repos.items():
click.echo(' {name}:{gid}'.format(name=repo['full_name'], gid=gid))
repos = Repository.query.filter(Repository.user_id == user.id,
Repository.hook.isnot(None))
click.echo("User has {0} enabled repositories.".format(repos.count()))
for r in repos:
click.echo(" {0}".format(r))
def github_api(app, db, tester_id, remote_token):
"""Github API mock."""
import github3
from . import fixtures
mock_api = MagicMock()
mock_api.me.return_value = github3.users.User(
fixtures.USER(login='******', email='*****@*****.**'))
repo_1 = github3.repos.Repository(fixtures.REPO('auser', 'repo-1', 1))
repo_1.hooks = MagicMock(return_value=[])
repo_1.file_contents = MagicMock(return_value=None)
repo_2 = github3.repos.Repository(fixtures.REPO('auser', 'repo-2', 2))
repo_2.hooks = MagicMock(return_value=[])
def mock_metadata_contents(path, ref):
data = json.dumps(
dict(upload_type='dataset',
license='mit-license',
creators=[
dict(name='Smith, Joe', affiliation='CERN'),
dict(name='Smith, Sam', affiliation='NASA'),
]))
return MagicMock(decoded=b(data))
repo_2.file_contents = MagicMock(side_effect=mock_metadata_contents)
repo_3 = github3.repos.Repository(fixtures.REPO('auser', 'arepo', 3))
repo_3.hooks = MagicMock(return_value=[])
repo_3.file_contents = MagicMock(return_value=None)
repos = {1: repo_1, 2: repo_2, 3: repo_3}
repos_by_name = {r.full_name: r for r in repos.values()}
mock_api.repositories.return_value = repos.values()
def mock_repo_with_id(id):
return repos.get(id)
def mock_repo_by_name(owner, name):
return repos_by_name.get('/'.join((owner, name)))
mock_api.repository_with_id.side_effect = mock_repo_with_id
mock_api.repository.side_effect = mock_repo_by_name
mock_api.markdown.side_effect = lambda x: x
mock_api.session.head.return_value = MagicMock(status_code=200)
mock_api.session.get.return_value = MagicMock(raw=fixtures.ZIPBALL())
with patch('invenio_github.api.GitHubAPI.api', new=mock_api):
with patch('invenio_github.api.GitHubAPI._sync_hooks'):
gh = GitHubAPI(user_id=tester_id)
with db.session.begin_nested():
gh.init_account()
db.session.expire(remote_token.remote_account)
yield mock_api
def github_api(app, db, tester_id, remote_token):
"""Github API mock."""
import github3
from . import fixtures
mock_api = MagicMock()
mock_api.me.return_value = github3.users.User(
fixtures.USER(login='******', email='*****@*****.**'))
repo_1 = github3.repos.Repository(fixtures.REPO('auser', 'repo-1', 1))
repo_1.hooks = MagicMock(return_value=[])
repo_1.file_contents = MagicMock(return_value=None)
repo_2 = github3.repos.Repository(fixtures.REPO('auser', 'repo-2', 2))
repo_2.hooks = MagicMock(return_value=[])
def mock_metadata_contents(path, ref):
data = json.dumps(dict(
upload_type='dataset',
license='mit-license',
creators=[
dict(name='Smith, Joe', affiliation='CERN'),
dict(name='Smith, Sam', affiliation='NASA'),
]
))
return MagicMock(decoded=b(data))
repo_2.file_contents = MagicMock(side_effect=mock_metadata_contents)
repo_3 = github3.repos.Repository(fixtures.REPO('auser', 'arepo', 3))
repo_3.hooks = MagicMock(return_value=[])
repo_3.file_contents = MagicMock(return_value=None)
repos = {1: repo_1, 2: repo_2, 3: repo_3}
repos_by_name = {r.full_name: r for r in repos.values()}
mock_api.repositories.return_value = repos.values()
def mock_repo_with_id(id):
return repos.get(id)
def mock_repo_by_name(owner, name):
return repos_by_name.get('/'.join((owner, name)))
mock_api.repository_with_id.side_effect = mock_repo_with_id
mock_api.repository.side_effect = mock_repo_by_name
mock_api.markdown.side_effect = lambda x: x
mock_api.session.head.return_value = MagicMock(status_code=302)
mock_api.session.get.return_value = MagicMock(raw=fixtures.ZIPBALL())
with patch('invenio_github.api.GitHubAPI.api', new=mock_api):
with patch('invenio_github.api.GitHubAPI._sync_hooks'):
gh = GitHubAPI(user_id=tester_id)
with db.session.begin_nested():
gh.init_account()
db.session.expire(remote_token.remote_account)
yield mock_api
def sync(user, hooks, async_hooks, skip_email):
"""Sync user's repositories.
USER can be either an email or user ID.
Examples:
github sync [email protected]
github sync 999
"""
user = resolve_user(user)
gh_api = GitHubAPI(user_id=user.id)
gh_api.sync(hooks=hooks, async_hooks=async_hooks)
if not skip_email:
verify_email(user)
db.session.commit()
def verify_email(user):
"""Check if user's GitHub and account emails match."""
gha = GitHubAPI(user_id=user.id)
gh_email = gha.api.me().email
if gh_email != user.email:
click.confirm("Warning: User's GitHub email ({0}) does not match"
" the account's ({1}). Continue?".format(gh_email,
user.email),
abort=True)
def update_local_gh_db(gh_db, remote_account_id, logger=None):
"""Fetch the missing GitHub repositories (from RemoteAccount information).
:param gh_db: mapping from remote accounts information to github IDs.
:type gh_db: dict
:param dst_path: Path to destination file.
:type dst_path: str
:param remote_account_id: Specify a single remote account ID to update.
:type remote_account_id: int
Updates the local GitHub repository name mapping (``gh_db``) with the
missing entries from RemoteAccount query.
The exact structure of the ``gh_db`` dictionary is as follows:
gh_db[remote_account_id:str][repository_name:str] = (id:int, name:str)
E.g.:
gh_db = {
"1234": {
"johndoe/repo1": (123456, "johndoe/repo1"),
"johndoe/repo2": (132457, "johndoe/repo2")
},
"2345": {
"janedoe/janesrepo1": (123458, "janedoe/repo1"),
"DoeOrganization/code1234": (123459, "DoeOrganization/code1234")
}
"3456": {} # No active repositories for this remote account.
}
"""
gh_db = deepcopy(gh_db)
if remote_account_id:
gh_ras = [RemoteAccount.query.filter_by(id=remote_account_id).one(), ]
else:
gh_ras = [ra for ra in RemoteAccount.query.all()
if 'repos' in ra.extra_data]
for ra in gh_ras:
gh_db.setdefault(str(ra.id), dict())
repos = ra.extra_data['repos'].items()
gh_api = GitHubAPI(ra.user.id)
for full_repo_name, repo_vals in repos:
if '/' not in full_repo_name:
if logger is not None:
logger.warning("Repository migrated: {name} ({id})".format(
name=full_repo_name, id=ra.id))
continue
if not repo_vals['hook']:
continue
if full_repo_name not in gh_db[str(ra.id)]:
try:
repo_info = fetch_gh_info(full_repo_name, gh_api.api)
gh_db[str(ra.id)][full_repo_name] = repo_info
except Exception as e:
if logger is not None:
logger.exception("GH fail: {name} ({id}): {e}".format(
name=full_repo_name, id=ra.id, e=e))
return gh_db
def removehook(repo, user, skip_email, yes_i_know):
"""Remove the hook from GitHub repository.
Positional argment REPO can be either the repository name, e.g.
`some-organization/some-repository` or its GitHub ID.
Option '--user' can be either an email or a user ID.
Examples:
github removehook foobar-org/foobar-repo
github removehook 55555 -u [email protected]
"""
repo = resolve_repo(repo)
if not repo.user and not user:
click.secho("Repository doesn't have an owner, please specify a user.")
return
if user:
user = resolve_user(user)
if not repo.user:
click.secho('Warning: Repository is not owned by any user.',
fg='yellow')
elif repo.user != user:
click.secho('Warning: Specified user is not the owner of this'
' repository.', fg='yellow')
else:
user = repo.user
if not skip_email:
verify_email(user)
msg = "Removing the hook for {user} and {repo}. Continue?".format(
user=user, repo=repo)
if not (yes_i_know or click.confirm(msg)):
click.echo('Aborted.')
return
gha = GitHubAPI(user_id=user.id)
gha.remove_hook(repo.github_id, repo.name)
db.session.commit()
def fetch_gh_info(full_repo_name, gh_api):
"""Fetch the GitHub repository from repository name."""
owner, repo_name = full_repo_name.split('/')
try:
gh_repo = gh_api.repository(owner, repo_name)
return (int(gh_repo.id), str(gh_repo.full_name))
except AuthenticationFailed as e:
pass # re-try with dev API
try:
dev_api = GitHubAPI._dev_api()
gh_repo = dev_api.repository(owner, repo_name)
return (int(gh_repo.id), str(gh_repo.full_name))
except AuthenticationFailed:
raise
def is_github_owner(user, pid, sync=False):
"""Return true if the user can create a new version for a GitHub record.
:param user: User to check for ownership.
:param pid: pid of a record.
:param sync: Condition to sync the user's repository info from GitHub.
"""
depid = fetch_depid(pid)
if sync:
try:
commit = False
with db.session.begin_nested():
gh = GitHubAPI(user_id=user.id)
if gh.account and gh.check_sync():
gh.sync(hooks=False)
commit = True
if commit:
db.session.commit()
except Exception:
# TODO: Log a warning?
# TODO: (In case GitHub is down, we still want to render the page)
pass
repo = get_github_repository(depid)
return repo.user == user if repo else False
def github_sync_old_remoteaccounts():
"""Synchronize the GitHub's remote account extra_data."""
def not_fetched(ra):
repos = ra.extra_data['repos']
return any(('/' in repo_name) for repo_name, _ in repos.items())
ras = RemoteAccount.query.all()
ras = [ra for ra in ras if 'repos' in ra.extra_data and not_fetched(ra)]
with click.progressbar(ras) as gh_ra_bar:
for ra in gh_ra_bar:
try:
GitHubAPI(ra.user_id).sync(hooks=False)
db.session.commit()
except Exception as e:
click.echo("Failed for user {0}. Error: {1}".format(
ra.user_id, e))
def migrate_github_remote_account(gh_db_ra, remote_account_id, logger=None):
"""Migrate the GitHub remote accounts."""
ra = RemoteAccount.query.filter_by(id=remote_account_id).first()
for full_repo_name, repo_vals in ra.extra_data['repos'].items():
if '/' not in full_repo_name:
if logger is not None:
logger.warning("Repository migrated: {name} ({id})".format(
name=full_repo_name, id=ra.id))
continue
if repo_vals['hook']:
owner, repo_name = full_repo_name.split('/')
# If repository name is cached, get from database, otherwise fetch
if full_repo_name in gh_db_ra:
gh_id, gh_full_name = gh_db_ra[full_repo_name]
else:
gh_api = GitHubAPI(ra.user.id)
gh_id, gh_full_name = fetch_gh_info(full_repo_name, gh_api.api)
try:
repo = Repository.get(user_id=ra.user_id, github_id=gh_id,
name=gh_full_name)
except NoResultFound:
repo = Repository.create(user_id=ra.user_id, github_id=gh_id,
name=gh_full_name)
except RepositoryAccessError as e:
if logger is not None:
repo = Repository.query.filter_by(github_id=gh_id).one()
logger.warning(
"User (uid: {user_id}) repository "
"'{repo_name}' from remote account ID:{ra_id} has "
"already been claimed by another user ({user2_id})."
"Repository ID: {repo_id}.".format(
user_id=ra.user.id, repo_name=full_repo_name,
ra_id=ra.id, user2_id=repo.user_id,
repo_id=repo.id))
continue
# TODO: Hook for this user will not be added.
repo.hook = repo_vals['hook']
if repo_vals['depositions']:
for dep in repo_vals['depositions']:
try:
pid = PersistentIdentifier.get(
pid_type='recid', pid_value=str(dep['record_id']))
release = Release.query.filter_by(
tag=dep['github_ref'], repository_id=repo.id,
record_id=pid.get_assigned_object()).first()
if not release:
release = Release(
tag=dep['github_ref'], errors=dep['errors'],
record_id=pid.get_assigned_object(),
repository_id=repo.id,
status=ReleaseStatus.PUBLISHED)
# TODO: DO SOMETHING WITH dep['doi']
# TODO: Update the date dep['submitted']
db.session.add(release)
except PIDDoesNotExistError as e:
if logger is not None:
logger.exception(
'Could not create release {tag} for repository'
' {repo_id}, because corresponding PID: {pid} '
'does not exist')
raise e
db.session.commit()