def test_grant_funder_not_found(app):
"""Test the grant loading with non-existent funder."""
loader = LocalOAIRELoader(
source='tests/testdata/openaire_test.sqlite',
funder_resolver=FundRefDOIResolver(data={'foo': 'bar'}))
with pytest.raises(FunderNotFoundError):
list(loader.iter_grants())
def test_grant_funder_not_found(app):
"""Test the grant loading with non-existent funder."""
loader = LocalOAIRELoader(
source='tests/testdata/openaire_test.sqlite',
funder_resolver=FundRefDOIResolver(data={'foo': 'bar'}))
with pytest.raises(FunderNotFoundError):
list(loader.iter_grants())
def loadgrants(source=None, setspec=None, all_grants=False):
"""Harvest grants from OpenAIRE.
:param source: Load the grants from a local sqlite db (offline).
The value of the parameter should be a path to the local file.
:type source: str
:param setspec: Harvest specific set through OAI-PMH
Creates a remote connection to OpenAIRE.
:type setspec: str
:param all_grants: Harvest all sets through OAI-PMH,
as specified in the configuration OPENAIRE_GRANTS_SPEC. Sets are
harvested sequentially in the order specified in the configuration.
Creates a remote connection to OpenAIRE.
:type all_grants: bool
"""
assert all_grants or setspec or source, \
"Either '--all', '--setspec' or '--source' is required parameter."
if all_grants:
harvest_all_openaire_projects.delay()
elif setspec:
click.echo("Remote grants loading sent to queue.")
harvest_openaire_projects.delay(setspec=setspec)
else: # if source
loader = LocalOAIRELoader(source=source)
loader._connect()
cnt = loader._count()
click.echo("Sending grants to queue.")
with click.progressbar(loader.iter_grants(), length=cnt) as grants_bar:
for grant_json in grants_bar:
register_grant.delay(grant_json)
def test_local_openaire_loader_db_connection(app):
"""Test the SQLite local loader."""
loader = LocalOAIRELoader(source='tests/testdata/openaire_test.sqlite')
loader._connect()
# connecting twice should raise an exception:
with pytest.raises(Exception, message='DB already connected.'):
loader._connect(throw=True)
loader._disconnect()
# disconnecting twice should raise an exception:
with pytest.raises(Exception, message='DB not connected.'):
loader._disconnect(throw=True)
def test_oaire_dumper(db, sqlite_tmpdb):
recuuid = uuid.uuid4()
PersistentIdentifier.create(
'frdoi', '10.13039/501100000925',
object_type='rec', object_uuid=recuuid, status='R')
Record.create({'acronyms': ['EC']}, id_=recuuid)
dumper = OAIREDumper(destination=sqlite_tmpdb)
# We expect to harvest 5 record from the MockSickle.
# with 'commit_batch_size=2', we will make 3 commits to sqlite db
dumper.dump(commit_batch_size=2)
loader = LocalOAIRELoader(source=sqlite_tmpdb)
records = list(loader.iter_grants())
assert len(records) == 5
def test_oaire_dumper(db, sqlite_tmpdb):
"""Test the grants dumper to local destination."""
recuuid = uuid.uuid4()
PersistentIdentifier.create(
'frdoi', '10.13039/501100000925',
object_type='rec', object_uuid=recuuid, status='R')
Record.create({'acronyms': ['EC']}, id_=recuuid)
dumper = OAIREDumper(destination=sqlite_tmpdb)
# We expect to harvest 5 record from the MockSickle.
# with 'commit_batch_size=2', we will make 3 commits to sqlite db
dumper.dump(commit_batch_size=2)
loader = LocalOAIRELoader(source=sqlite_tmpdb)
records = list(loader.iter_grants())
assert len(records) == 5
def loadgrants(source=None, setspec=None, all_grants=False):
"""Harvest grants from OpenAIRE.
:param source: Load the grants from a local sqlite db (offline).
The value of the parameter should be a path to the local file.
:type source: str
:param setspec: Harvest specific set through OAI-PMH
Creates a remote connection to OpenAIRE.
:type setspec: str
:param all_grants: Harvest all sets through OAI-PMH,
as specified in the configuration OPENAIRE_GRANTS_SPEC. Sets are
harvested sequentially in the order specified in the configuration.
Creates a remote connection to OpenAIRE.
:type all_grants: bool
"""
assert all_grants or setspec or source, \
"Either '--all', '--setspec' or '--source' is required parameter."
if all_grants:
harvest_all_openaire_projects.delay()
elif setspec:
click.echo("Remote grants loading sent to queue.")
harvest_openaire_projects.delay(setspec=setspec)
else: # if source
loader = LocalOAIRELoader(source=source)
loader._connect()
cnt = loader._count()
click.echo("Sending grants to queue.")
with click.progressbar(loader.iter_grants(), length=cnt) as grants_bar:
for grant_json in grants_bar:
register_grant.delay(grant_json)
def test_local_openaire_loader(app):
"""Test the SQLite local loader."""
loader = LocalOAIRELoader(source='tests/testdata/openaire_test.sqlite')
records = list(loader.iter_grants())
assert len(records) == 10
def test_local_openaire_loader(app):
"""Test the SQLite local loader."""
loader = LocalOAIRELoader(source='tests/testdata/openaire_test.sqlite')
records = list(loader.iter_grants())
assert len(records) == 10