def g_repositories(app, db, g_repositories_data):
"""Fixture for GitHub Repository objects."""
for rd in g_repositories_data:
repository = Repository(**rd)
# We don't call GitHub hence the hook is not important, yet it should
# be not null
repository.hook = 12345
db.session.add(repository)
db.session.commit()
rd['id'] = repository.id
return g_repositories_data
def g_repositories(app, db, g_repositories_data):
"""Fixture for GitHub Repository objects."""
for rd in g_repositories_data:
repository = Repository(**rd)
# We don't call GitHub hence the hook is not important, yet it should
# be not null
repository.hook = 12345
db.session.add(repository)
db.session.commit()
rd['id'] = repository.id
return g_repositories_data
def test_extract_metadata(app, db, tester_id, remote_token, github_api):
Repository.enable(tester_id, github_id=2, name='repo-2', hook=1234)
event = Event(
receiver_id='github',
user_id=tester_id,
payload=fixtures.PAYLOAD('auser', 'repo-2', 2, tag='v1.0'),
)
release = Release.create(event)
gh = GitHubRelease(release)
metadata = gh.metadata
assert metadata['upload_type'] == 'dataset'
assert metadata['license'] == 'mit-license'
assert len(metadata['creators']) == 2
def test_extract_metadata(app, db, tester_id, remote_token, github_api):
Repository.enable(tester_id, github_id=2, name='repo-2', hook=1234)
event = Event(
receiver_id='github',
user_id=tester_id,
payload=fixtures.PAYLOAD('auser', 'repo-2', 2, tag='v1.0'),
)
release = Release.create(event)
gh = GitHubRelease(release)
metadata = gh.metadata
assert metadata['upload_type'] == 'dataset'
assert metadata['license'] == 'mit-license'
assert len(metadata['creators']) == 2
def load_github_releases(releases_file):
"""Load GitHub releases information.
Updates the missing releases and the submission dates.
"""
from invenio_github.models import Release, Repository, ReleaseStatus
from sqlalchemy.orm.exc import NoResultFound
import arrow
import json
releases_db = json.load(releases_file)
with click.progressbar(releases_db) as releases:
for release in releases:
repo_name, new_repo_name, gh_repo_id, ra_id, user_id, dep = release
try:
repo = Repository.query.filter_by(github_id=gh_repo_id).one()
except NoResultFound:
repo = Repository.create(user_id=user_id, github_id=gh_repo_id,
name=new_repo_name)
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(dep['record_id']))
rel = Release.query.filter_by(
repository_id=repo.id,
record_id=pid.get_assigned_object()).first()
created = arrow.get(dep['submitted']).datetime.replace(tzinfo=None)
if rel:
rel.created = created
else:
rel = Release(tag=dep['github_ref'],
errors=dep['errors'],
record_id=pid.get_assigned_object(),
repository_id=repo.id,
status=ReleaseStatus.PUBLISHED,
created=created)
db.session.add(rel)
db.session.commit()
def repository_model(app, db, tester_id):
"""Github repository fixture."""
repository = Repository(
github_id=1, name='testuser/testrepo', user_id=tester_id)
db.session.add(repository)
db.session.commit()
return repository
def test_webhook_post(app, db, tester_id, location, remote_token, github_api):
"""Test payload parsing on webhook."""
from . import fixtures
with patch('invenio_deposit.api.Deposit.indexer'):
# Enable repository webhook.
Repository.enable(tester_id, github_id=3, name='arepo', hook=1234)
db.session.commit()
# JSON payload parsing.
payload = json.dumps(fixtures.PAYLOAD('auser', 'arepo', 3, 'v1.0'))
headers = [('Content-Type', 'application/json')]
with app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id='github', user_id=tester_id)
db.session.commit()
event.process()
db.session.commit()
from invenio_records.models import RecordMetadata
assert RecordMetadata.query.count() == 2
def test_webhook_post(app, db, tester_id, location, remote_token, github_api):
"""Test payload parsing on webhook."""
from . import fixtures
with patch('invenio_deposit.api.Deposit.indexer'):
# Enable repository webhook.
Repository.enable(tester_id, github_id=3, name='arepo', hook=1234)
db.session.commit()
# JSON payload parsing.
payload = json.dumps(fixtures.PAYLOAD('auser', 'arepo', 3, 'v1.0'))
headers = [('Content-Type', 'application/json')]
with app.test_request_context(headers=headers, data=payload):
event = Event.create(receiver_id='github', user_id=tester_id)
db.session.commit()
event.process()
db.session.commit()
from invenio_records.models import RecordMetadata
assert RecordMetadata.query.count() == 2
def test_handle_payload(app, db, location, tester_id, remote_token,
github_api):
from invenio_webhooks.models import Event
extra_data = remote_token.remote_account.extra_data
assert '1' in extra_data['repos']
assert 'repo-1' in extra_data['repos']['1']['full_name']
assert '2' in extra_data['repos']
assert 'repo-2' in extra_data['repos']['2']['full_name']
# Create the repository that will make the release
with db.session.begin_nested():
Repository.enable(tester_id, github_id=1, name='repo-1', hook=1234)
event = Event(
receiver_id='github',
user_id=tester_id,
payload=fixtures.PAYLOAD('auser', 'repo-1', 1)
)
db.session.add(event)
with patch('invenio_deposit.api.Deposit.indexer'):
event.process()
repo_1 = Repository.query.filter_by(name='repo-1', github_id=1).first()
assert repo_1.releases.count() == 1
release = repo_1.releases.first()
assert release.status == ReleaseStatus.PUBLISHED
assert release.errors is None
assert release.tag == 'v1.0'
assert release.record is not None
assert release.record.get('control_number') == '1'
record_files = release.record.get('_files')
assert len(record_files) == 1
assert record_files[0]['size'] > 0
bucket = Bucket.get(record_files[0]['bucket'])
assert bucket is not None
assert len(bucket.objects) == 1
assert bucket.objects[0].key == 'auser/repo-1-v1.0.zip'
def test_handle_payload(app, db, location, tester_id, remote_token,
github_api):
from invenio_webhooks.models import Event
extra_data = remote_token.remote_account.extra_data
assert '1' in extra_data['repos']
assert 'repo-1' in extra_data['repos']['1']['full_name']
assert '2' in extra_data['repos']
assert 'repo-2' in extra_data['repos']['2']['full_name']
# Create the repository that will make the release
with db.session.begin_nested():
Repository.enable(tester_id, github_id=1, name='repo-1', hook=1234)
event = Event(receiver_id='github',
user_id=tester_id,
payload=fixtures.PAYLOAD('auser', 'repo-1', 1))
db.session.add(event)
with patch('invenio_deposit.api.Deposit.indexer'):
event.process()
repo_1 = Repository.query.filter_by(name='repo-1', github_id=1).first()
assert repo_1.releases.count() == 1
release = repo_1.releases.first()
assert release.status == ReleaseStatus.PUBLISHED
assert release.errors is None
assert release.tag == 'v1.0'
assert release.record is not None
assert release.record.get('control_number') == '1'
record_files = release.record.get('_files')
assert len(record_files) == 1
assert record_files[0]['size'] > 0
bucket = Bucket.get(record_files[0]['bucket'])
assert bucket is not None
assert len(bucket.objects) == 1
assert bucket.objects[0].key == 'auser/repo-1-v1.0.zip'
def load_github_releases(releases_file):
"""Load GitHub releases information.
Updates the missing releases and the submission dates.
"""
from invenio_github.models import Release, Repository, ReleaseStatus
from sqlalchemy.orm.exc import NoResultFound
import arrow
import json
releases_db = json.load(releases_file)
with click.progressbar(releases_db) as releases:
for release in releases:
repo_name, new_repo_name, gh_repo_id, ra_id, user_id, dep = release
try:
repo = Repository.query.filter_by(github_id=gh_repo_id).one()
except NoResultFound:
repo = Repository.create(user_id=user_id,
github_id=gh_repo_id,
name=new_repo_name)
pid = PersistentIdentifier.get(pid_type='recid',
pid_value=str(dep['record_id']))
rel = Release.query.filter_by(
repository_id=repo.id,
record_id=pid.get_assigned_object()).first()
created = arrow.get(dep['submitted']).datetime.replace(tzinfo=None)
if rel:
rel.created = created
else:
rel = Release(tag=dep['github_ref'],
errors=dep['errors'],
record_id=pid.get_assigned_object(),
repository_id=repo.id,
status=ReleaseStatus.PUBLISHED,
created=created)
db.session.add(rel)
db.session.commit()
def test_repository_unbound(app):
"""Test create_badge method."""
assert Repository(name='org/repo', github_id=1).latest_release() is None
def test_github_newversion_permissions(app, db, minimal_record, users, g_users,
g_remoteaccounts):
"""Test new version creation permissions for GitHub records."""
old_owner, new_owner = [User.query.get(u['id']) for u in g_users]
# Create repository, and set owner to `old_owner`
repo = Repository.create(
name='foo/bar', github_id=8000, user_id=old_owner.id, hook=1234)
# Create concpetrecid for the GitHub records
conceptrecid = PersistentIdentifier.create(
'recid', '100', status=PIDStatus.RESERVED)
def create_deposit_and_record(pid_value, owner):
"""Utility function for creating records and deposits."""
recid = PersistentIdentifier.create(
'recid', pid_value, status=PIDStatus.RESERVED)
pv = PIDVersioning(parent=conceptrecid)
pv.insert_draft_child(recid)
depid = PersistentIdentifier.create(
'depid', pid_value, status=PIDStatus.REGISTERED)
deposit = ZenodoRecord.create({'_deposit': {'id': depid.pid_value},
'conceptrecid': conceptrecid.pid_value,
'recid': recid.pid_value})
deposit.commit()
depid.assign('rec', deposit.id)
record_metadata = deepcopy(minimal_record)
record_metadata['_deposit'] = {'id': depid.pid_value}
record_metadata['conceptrecid'] = conceptrecid.pid_value
record_metadata['recid'] = int(recid.pid_value)
record_metadata['owners'] = [owner.id]
record = ZenodoRecord.create(record_metadata)
zenodo_record_minter(record.id, record)
record.commit()
return (depid, deposit, recid, record)
# Create first GitHub record (by `old_owner`)
depid1, d1, recid1, r1 = create_deposit_and_record('101', old_owner)
rel1 = Release(release_id=111, repository_id=repo.id, record_id=d1.id,
status=ReleaseStatus.PUBLISHED)
db.session.add(rel1)
db.session.commit()
assert is_github_versioned(recid1)
@contextmanager
def set_identity(user):
from flask_principal import AnonymousIdentity, Identity
principal = current_app.extensions['security'].principal
principal.set_identity(Identity(user))
yield
principal.set_identity(AnonymousIdentity())
with app.test_request_context():
with set_identity(old_owner):
assert is_github_owner(old_owner, recid1)
assert has_update_permission(old_owner, r1)
assert has_newversion_permission(old_owner, r1)
with set_identity(new_owner):
assert not is_github_owner(new_owner, recid1)
assert not has_update_permission(new_owner, r1)
assert not has_newversion_permission(new_owner, r1)
# Change the repository owner
repo.user_id = new_owner.id
db.session.add(repo)
db.session.commit()
with app.test_request_context():
with set_identity(old_owner):
assert not is_github_owner(old_owner, recid1)
# `old_owner` can edit his record of course
assert has_update_permission(old_owner, r1)
assert has_newversion_permission(old_owner, r1)
with set_identity(new_owner):
assert is_github_owner(new_owner, recid1)
# `new_owner` can't edit the `old_owner`'s record
assert not has_update_permission(new_owner, r1)
assert not has_newversion_permission(new_owner, r1)
# Create second GitHub record (by `new_owner`)
depid2, d2, recid2, r2 = create_deposit_and_record('102', new_owner)
rel2 = Release(release_id=222, repository_id=repo.id, record_id=d2.id,
status=ReleaseStatus.PUBLISHED)
db.session.add(rel2)
db.session.commit()
with app.test_request_context():
with set_identity(old_owner):
assert not is_github_owner(old_owner, recid1)
assert not is_github_owner(old_owner, recid2)
assert has_update_permission(old_owner, r1)
# `old_owner` can't edit the `new_owner`'s record
assert not has_update_permission(old_owner, r2)
assert not has_newversion_permission(old_owner, r1)
assert not has_newversion_permission(old_owner, r2)
with set_identity(new_owner):
assert is_github_owner(new_owner, recid1)
assert is_github_owner(new_owner, recid2)
assert not has_update_permission(new_owner, r1)
# `new_owner` can edit his newly released record
assert has_update_permission(new_owner, r2)
assert has_newversion_permission(new_owner, r1)
assert has_newversion_permission(new_owner, r2)
# Create a manual record (by `new_owner`)
depid3, d3, recid3, r3 = create_deposit_and_record('103', new_owner)
db.session.commit()
with app.test_request_context():
with set_identity(old_owner):
assert not is_github_owner(old_owner, recid3)
assert not has_update_permission(old_owner, r3)
assert not has_newversion_permission(old_owner, r3)
with set_identity(new_owner):
assert is_github_owner(new_owner, recid3)
assert has_update_permission(new_owner, r3)
assert has_newversion_permission(new_owner, r3)
def migrate_github_remote_account(gh_db_ra, remote_account_id, logger=None):
"""Migrate the GitHub remote accounts."""
ra = RemoteAccount.query.filter_by(id=remote_account_id).first()
for full_repo_name, repo_vals in ra.extra_data['repos'].items():
if '/' not in full_repo_name:
if logger is not None:
logger.warning("Repository migrated: {name} ({id})".format(
name=full_repo_name, id=ra.id))
continue
if repo_vals['hook']:
owner, repo_name = full_repo_name.split('/')
# If repository name is cached, get from database, otherwise fetch
if full_repo_name in gh_db_ra:
gh_id, gh_full_name = gh_db_ra[full_repo_name]
else:
gh_api = GitHubAPI(ra.user.id)
gh_id, gh_full_name = fetch_gh_info(full_repo_name, gh_api.api)
try:
repo = Repository.get(user_id=ra.user_id, github_id=gh_id,
name=gh_full_name)
except NoResultFound:
repo = Repository.create(user_id=ra.user_id, github_id=gh_id,
name=gh_full_name)
except RepositoryAccessError as e:
if logger is not None:
repo = Repository.query.filter_by(github_id=gh_id).one()
logger.warning(
"User (uid: {user_id}) repository "
"'{repo_name}' from remote account ID:{ra_id} has "
"already been claimed by another user ({user2_id})."
"Repository ID: {repo_id}.".format(
user_id=ra.user.id, repo_name=full_repo_name,
ra_id=ra.id, user2_id=repo.user_id,
repo_id=repo.id))
continue
# TODO: Hook for this user will not be added.
repo.hook = repo_vals['hook']
if repo_vals['depositions']:
for dep in repo_vals['depositions']:
try:
pid = PersistentIdentifier.get(
pid_type='recid', pid_value=str(dep['record_id']))
release = Release.query.filter_by(
tag=dep['github_ref'], repository_id=repo.id,
record_id=pid.get_assigned_object()).first()
if not release:
release = Release(
tag=dep['github_ref'], errors=dep['errors'],
record_id=pid.get_assigned_object(),
repository_id=repo.id,
status=ReleaseStatus.PUBLISHED)
# TODO: DO SOMETHING WITH dep['doi']
# TODO: Update the date dep['submitted']
db.session.add(release)
except PIDDoesNotExistError as e:
if logger is not None:
logger.exception(
'Could not create release {tag} for repository'
' {repo_id}, because corresponding PID: {pid} '
'does not exist')
raise e
db.session.commit()
