def g_repositories(app, db, g_repositories_data): """Fixture for GitHub Repository objects.""" for rd in g_repositories_data: repository = Repository(**rd) # We don't call GitHub hence the hook is not important, yet it should # be not null repository.hook = 12345 db.session.add(repository) db.session.commit() rd['id'] = repository.id return g_repositories_data
def test_extract_metadata(app, db, tester_id, remote_token, github_api): Repository.enable(tester_id, github_id=2, name='repo-2', hook=1234) event = Event( receiver_id='github', user_id=tester_id, payload=fixtures.PAYLOAD('auser', 'repo-2', 2, tag='v1.0'), ) release = Release.create(event) gh = GitHubRelease(release) metadata = gh.metadata assert metadata['upload_type'] == 'dataset' assert metadata['license'] == 'mit-license' assert len(metadata['creators']) == 2
def load_github_releases(releases_file): """Load GitHub releases information. Updates the missing releases and the submission dates. """ from invenio_github.models import Release, Repository, ReleaseStatus from sqlalchemy.orm.exc import NoResultFound import arrow import json releases_db = json.load(releases_file) with click.progressbar(releases_db) as releases: for release in releases: repo_name, new_repo_name, gh_repo_id, ra_id, user_id, dep = release try: repo = Repository.query.filter_by(github_id=gh_repo_id).one() except NoResultFound: repo = Repository.create(user_id=user_id, github_id=gh_repo_id, name=new_repo_name) pid = PersistentIdentifier.get(pid_type='recid', pid_value=str(dep['record_id'])) rel = Release.query.filter_by( repository_id=repo.id, record_id=pid.get_assigned_object()).first() created = arrow.get(dep['submitted']).datetime.replace(tzinfo=None) if rel: rel.created = created else: rel = Release(tag=dep['github_ref'], errors=dep['errors'], record_id=pid.get_assigned_object(), repository_id=repo.id, status=ReleaseStatus.PUBLISHED, created=created) db.session.add(rel) db.session.commit()
def repository_model(app, db, tester_id): """Github repository fixture.""" repository = Repository( github_id=1, name='testuser/testrepo', user_id=tester_id) db.session.add(repository) db.session.commit() return repository
def test_webhook_post(app, db, tester_id, location, remote_token, github_api): """Test payload parsing on webhook.""" from . import fixtures with patch('invenio_deposit.api.Deposit.indexer'): # Enable repository webhook. Repository.enable(tester_id, github_id=3, name='arepo', hook=1234) db.session.commit() # JSON payload parsing. payload = json.dumps(fixtures.PAYLOAD('auser', 'arepo', 3, 'v1.0')) headers = [('Content-Type', 'application/json')] with app.test_request_context(headers=headers, data=payload): event = Event.create(receiver_id='github', user_id=tester_id) db.session.commit() event.process() db.session.commit() from invenio_records.models import RecordMetadata assert RecordMetadata.query.count() == 2
def test_handle_payload(app, db, location, tester_id, remote_token, github_api): from invenio_webhooks.models import Event extra_data = remote_token.remote_account.extra_data assert '1' in extra_data['repos'] assert 'repo-1' in extra_data['repos']['1']['full_name'] assert '2' in extra_data['repos'] assert 'repo-2' in extra_data['repos']['2']['full_name'] # Create the repository that will make the release with db.session.begin_nested(): Repository.enable(tester_id, github_id=1, name='repo-1', hook=1234) event = Event( receiver_id='github', user_id=tester_id, payload=fixtures.PAYLOAD('auser', 'repo-1', 1) ) db.session.add(event) with patch('invenio_deposit.api.Deposit.indexer'): event.process() repo_1 = Repository.query.filter_by(name='repo-1', github_id=1).first() assert repo_1.releases.count() == 1 release = repo_1.releases.first() assert release.status == ReleaseStatus.PUBLISHED assert release.errors is None assert release.tag == 'v1.0' assert release.record is not None assert release.record.get('control_number') == '1' record_files = release.record.get('_files') assert len(record_files) == 1 assert record_files[0]['size'] > 0 bucket = Bucket.get(record_files[0]['bucket']) assert bucket is not None assert len(bucket.objects) == 1 assert bucket.objects[0].key == 'auser/repo-1-v1.0.zip'
def test_handle_payload(app, db, location, tester_id, remote_token, github_api): from invenio_webhooks.models import Event extra_data = remote_token.remote_account.extra_data assert '1' in extra_data['repos'] assert 'repo-1' in extra_data['repos']['1']['full_name'] assert '2' in extra_data['repos'] assert 'repo-2' in extra_data['repos']['2']['full_name'] # Create the repository that will make the release with db.session.begin_nested(): Repository.enable(tester_id, github_id=1, name='repo-1', hook=1234) event = Event(receiver_id='github', user_id=tester_id, payload=fixtures.PAYLOAD('auser', 'repo-1', 1)) db.session.add(event) with patch('invenio_deposit.api.Deposit.indexer'): event.process() repo_1 = Repository.query.filter_by(name='repo-1', github_id=1).first() assert repo_1.releases.count() == 1 release = repo_1.releases.first() assert release.status == ReleaseStatus.PUBLISHED assert release.errors is None assert release.tag == 'v1.0' assert release.record is not None assert release.record.get('control_number') == '1' record_files = release.record.get('_files') assert len(record_files) == 1 assert record_files[0]['size'] > 0 bucket = Bucket.get(record_files[0]['bucket']) assert bucket is not None assert len(bucket.objects) == 1 assert bucket.objects[0].key == 'auser/repo-1-v1.0.zip'
def test_repository_unbound(app): """Test create_badge method.""" assert Repository(name='org/repo', github_id=1).latest_release() is None
def test_github_newversion_permissions(app, db, minimal_record, users, g_users, g_remoteaccounts): """Test new version creation permissions for GitHub records.""" old_owner, new_owner = [User.query.get(u['id']) for u in g_users] # Create repository, and set owner to `old_owner` repo = Repository.create( name='foo/bar', github_id=8000, user_id=old_owner.id, hook=1234) # Create concpetrecid for the GitHub records conceptrecid = PersistentIdentifier.create( 'recid', '100', status=PIDStatus.RESERVED) def create_deposit_and_record(pid_value, owner): """Utility function for creating records and deposits.""" recid = PersistentIdentifier.create( 'recid', pid_value, status=PIDStatus.RESERVED) pv = PIDVersioning(parent=conceptrecid) pv.insert_draft_child(recid) depid = PersistentIdentifier.create( 'depid', pid_value, status=PIDStatus.REGISTERED) deposit = ZenodoRecord.create({'_deposit': {'id': depid.pid_value}, 'conceptrecid': conceptrecid.pid_value, 'recid': recid.pid_value}) deposit.commit() depid.assign('rec', deposit.id) record_metadata = deepcopy(minimal_record) record_metadata['_deposit'] = {'id': depid.pid_value} record_metadata['conceptrecid'] = conceptrecid.pid_value record_metadata['recid'] = int(recid.pid_value) record_metadata['owners'] = [owner.id] record = ZenodoRecord.create(record_metadata) zenodo_record_minter(record.id, record) record.commit() return (depid, deposit, recid, record) # Create first GitHub record (by `old_owner`) depid1, d1, recid1, r1 = create_deposit_and_record('101', old_owner) rel1 = Release(release_id=111, repository_id=repo.id, record_id=d1.id, status=ReleaseStatus.PUBLISHED) db.session.add(rel1) db.session.commit() assert is_github_versioned(recid1) @contextmanager def set_identity(user): from flask_principal import AnonymousIdentity, Identity principal = current_app.extensions['security'].principal principal.set_identity(Identity(user)) yield principal.set_identity(AnonymousIdentity()) with app.test_request_context(): with set_identity(old_owner): assert is_github_owner(old_owner, recid1) assert has_update_permission(old_owner, r1) assert has_newversion_permission(old_owner, r1) with set_identity(new_owner): assert not is_github_owner(new_owner, recid1) assert not has_update_permission(new_owner, r1) assert not has_newversion_permission(new_owner, r1) # Change the repository owner repo.user_id = new_owner.id db.session.add(repo) db.session.commit() with app.test_request_context(): with set_identity(old_owner): assert not is_github_owner(old_owner, recid1) # `old_owner` can edit his record of course assert has_update_permission(old_owner, r1) assert has_newversion_permission(old_owner, r1) with set_identity(new_owner): assert is_github_owner(new_owner, recid1) # `new_owner` can't edit the `old_owner`'s record assert not has_update_permission(new_owner, r1) assert not has_newversion_permission(new_owner, r1) # Create second GitHub record (by `new_owner`) depid2, d2, recid2, r2 = create_deposit_and_record('102', new_owner) rel2 = Release(release_id=222, repository_id=repo.id, record_id=d2.id, status=ReleaseStatus.PUBLISHED) db.session.add(rel2) db.session.commit() with app.test_request_context(): with set_identity(old_owner): assert not is_github_owner(old_owner, recid1) assert not is_github_owner(old_owner, recid2) assert has_update_permission(old_owner, r1) # `old_owner` can't edit the `new_owner`'s record assert not has_update_permission(old_owner, r2) assert not has_newversion_permission(old_owner, r1) assert not has_newversion_permission(old_owner, r2) with set_identity(new_owner): assert is_github_owner(new_owner, recid1) assert is_github_owner(new_owner, recid2) assert not has_update_permission(new_owner, r1) # `new_owner` can edit his newly released record assert has_update_permission(new_owner, r2) assert has_newversion_permission(new_owner, r1) assert has_newversion_permission(new_owner, r2) # Create a manual record (by `new_owner`) depid3, d3, recid3, r3 = create_deposit_and_record('103', new_owner) db.session.commit() with app.test_request_context(): with set_identity(old_owner): assert not is_github_owner(old_owner, recid3) assert not has_update_permission(old_owner, r3) assert not has_newversion_permission(old_owner, r3) with set_identity(new_owner): assert is_github_owner(new_owner, recid3) assert has_update_permission(new_owner, r3) assert has_newversion_permission(new_owner, r3)
def migrate_github_remote_account(gh_db_ra, remote_account_id, logger=None): """Migrate the GitHub remote accounts.""" ra = RemoteAccount.query.filter_by(id=remote_account_id).first() for full_repo_name, repo_vals in ra.extra_data['repos'].items(): if '/' not in full_repo_name: if logger is not None: logger.warning("Repository migrated: {name} ({id})".format( name=full_repo_name, id=ra.id)) continue if repo_vals['hook']: owner, repo_name = full_repo_name.split('/') # If repository name is cached, get from database, otherwise fetch if full_repo_name in gh_db_ra: gh_id, gh_full_name = gh_db_ra[full_repo_name] else: gh_api = GitHubAPI(ra.user.id) gh_id, gh_full_name = fetch_gh_info(full_repo_name, gh_api.api) try: repo = Repository.get(user_id=ra.user_id, github_id=gh_id, name=gh_full_name) except NoResultFound: repo = Repository.create(user_id=ra.user_id, github_id=gh_id, name=gh_full_name) except RepositoryAccessError as e: if logger is not None: repo = Repository.query.filter_by(github_id=gh_id).one() logger.warning( "User (uid: {user_id}) repository " "'{repo_name}' from remote account ID:{ra_id} has " "already been claimed by another user ({user2_id})." "Repository ID: {repo_id}.".format( user_id=ra.user.id, repo_name=full_repo_name, ra_id=ra.id, user2_id=repo.user_id, repo_id=repo.id)) continue # TODO: Hook for this user will not be added. repo.hook = repo_vals['hook'] if repo_vals['depositions']: for dep in repo_vals['depositions']: try: pid = PersistentIdentifier.get( pid_type='recid', pid_value=str(dep['record_id'])) release = Release.query.filter_by( tag=dep['github_ref'], repository_id=repo.id, record_id=pid.get_assigned_object()).first() if not release: release = Release( tag=dep['github_ref'], errors=dep['errors'], record_id=pid.get_assigned_object(), repository_id=repo.id, status=ReleaseStatus.PUBLISHED) # TODO: DO SOMETHING WITH dep['doi'] # TODO: Update the date dep['submitted'] db.session.add(release) except PIDDoesNotExistError as e: if logger is not None: logger.exception( 'Could not create release {tag} for repository' ' {repo_id}, because corresponding PID: {pid} ' 'does not exist') raise e db.session.commit()