def test_details(self):
reader = DataSourceReader(
lambda: access_token,
'https://data-access.staging-bip-app.ssb.no/',
'https://dapla-catalog.staging-bip-app.ssb.no')
paths = reader.details('/felles/bjornandre/python/test1')
print(paths)
def test_read_from_path(self):
reader = DataSourceReader(
lambda: access_token,
'https://data-access.staging-bip-app.ssb.no/',
'https://dapla-catalog.staging-bip-app.ssb.no')
table = reader.read('/felles/bjornandre/python/test1')
print(table.head(5))
def details(path):
reader = DataSourceReader(AuthClient.get_access_token,
os.environ['DATA_ACCESS_URL'],
os.environ['CATALOG_URL'])
try:
return reader.details(path)
except AuthError as err:
err.print_warning()
def test_read_file(self):
json_response = {
'accessAllowed': True,
'parentUri': 'file://{}'.format(os.path.dirname(__file__)),
'version': '1591300975435',
'accessToken': 'mock-access-token',
'expirationTime': '1592492757749'
}
responses.add(responses.POST,
'http://mock.no/rpc/DataAccessService/readLocation',
json=json_response,
status=200)
reader = DataSourceReader(lambda: 'mock-user-token', 'http://mock.no/',
'http://mock.no/')
table = reader.read('/testdata')
print(table.head(5))
def test_read_write_gcs(self):
access_token = 'TODO'
reader = DataSourceReader(
lambda: access_token,
'https://data-access.staging-bip-app.ssb.no/',
'https://dapla-catalog.staging-bip-app.ssb.no')
table = reader.read('/felles/bjornandre/bilmerker_og_innskudd')
print(table.head(5))
writer = DataSourceWriter(
lambda: access_token,
'https://data-access.staging-bip-app.ssb.no/',
'https://metadata-distributor.staging-bip-app.ssb.no',
'staging-bip',
'metadata-distributor-dataset-updates',
'https://dapla-catalog.staging-bip-app.ssb.no',
)
writer.write(table,
'/felles/bjornandre/python/test1',
valuation="INTERNAL",
state="INPUT")
def read_pandas(path, columns=None):
"""
Read Parquet data from a path in Dapla.
Parameters
----------
path : str
A path that will be mapped to a GCS path in Dapla
columns : List[str], optional
Subset of columns to read.
Returns
-------
pyarrow.Table
Content as a table (of columns).
"""
reader = DataSourceReader(AuthClient.get_access_token,
os.environ['DATA_ACCESS_URL'],
os.environ['CATALOG_URL'])
try:
return reader.read(path, columns)
except AuthError as err:
err.print_warning()