def parentinit(self, testcfg):
api = getapi(testcfg.get('local', False))
settings = copy.deepcopy(cfg_settings_parent)
if testcfg.get('encrypt', True):
settings = api.encode(settings)
# check clean
r, d = api.cnxn.pipes.get()
assert d == []
assert cfg_parent_name not in api.list_pipes()
# create
if testcfg.get('local', False):
settings['protocol'] = 's3'
settings['options'] = {
'remotepath':
's3://{}/'.format(cfg_settings_parent['location'])
}
response, data = d6tpipe.upsert_pipe(api, settings)
assert cfg_parent_name in api.list_pipes()
assert response.status_code == 201
yield True
r, d = api.cnxn.pipes._(cfg_parent_name).delete()
assert r.status_code == 204
def pipeinit(self, testcfg):
api = getapi(testcfg.get('local',False))
settings = copy.deepcopy(cfg_settings_pipe)
assert cfg_pipe_name not in api.list_pipes()
if testcfg.get('local',False):
settings['protocol']='s3'
settings['credentials']=cfg_settings_parent['credentials']
settings['options']={**settings['options'],**{'remotepath': 's3://{}/{}'.format(cfg_settings_parent['location'],cfg_settings_pipe['options']['dir'])}}
response, data = d6tpipe.upsert_pipe(api, settings)
assert cfg_pipe_name in api.list_pipes()
yield True
r, d = api.cnxn.pipes._(cfg_pipe_name).delete()
assert r.status_code == 204
def test_d6tfree_share(self, cleanup, signup, testcfg):
if not testcfg.get('local',False):
cfg_name = 'utest-d6tfree'
cfg_name_child = cfg_name+'-child'
api = getapi(testcfg.get('local', False))
api2 = getapi2(testcfg.get('local', False))
# test quick create
d6tpipe.upsert_pipe(api, {'name': cfg_name})
d6tpipe.upsert_pipe(api, {'name': cfg_name_child, 'parent':cfg_name, 'options':{'dir':'child'}})
lp = api.list_pipes()
assert cfg_name in lp and cfg_name_child in lp
# share parent pipe
with pytest.raises(APIError, match='403'):
api2.cnxn.pipes._(cfg_name).get()
d6tpipe.upsert_permissions(api, cfg_name, {'username':cfg_usr2, 'role':'read'})
assert api2.cnxn.pipes._(cfg_name).get()[1]['name']==cfg_name
assert api2.cnxn.pipes._(cfg_name_child).get()[1]['name']==cfg_name_child
d6tpipe.upsert_permissions(api, cfg_name, {'username':cfg_usr2, 'role':'revoke'})
with pytest.raises(APIError, match='403'):
api2.cnxn.pipes._(cfg_name).get()
# share child pipe
with pytest.raises(APIError, match='403'):
api2.cnxn.pipes._(cfg_name).get()
d6tpipe.upsert_permissions(api, cfg_name_child, {'username':cfg_usr2, 'role':'read'})
with pytest.raises(APIError, match='403'):
api2.cnxn.pipes._(cfg_name).get()
assert api2.cnxn.pipes._(cfg_name_child).get()[1]['name']==cfg_name_child
d6tpipe.upsert_permissions(api, cfg_name_child, {'username':cfg_usr2, 'role':'revoke'})
with pytest.raises(APIError, match='403'):
api2.cnxn.pipes._(cfg_name_child).get()
def test_pipes_pull(self, cleanup, signup, parentinit, pipeinit, testcfg):
api = getapi(testcfg.get('local',False))
pipe = getpipe(api)
assert pipe.name in api.list_pipes()
cfg_chk_crc = ['8a9782e9efa8befa9752045ca506a62e',
'5fe579d6b71031dad399e8d4ea82820b',
'4c7da169df85253d7ff737dde1e7400b',
'ca62a122993494e763fd1676cce95e76']
# assert False
assert pipe.files() == []
assert pipe.scan_remote() == cfg_filenames_chk
r, d = pipe.scan_remote(attributes=True)
assert _filenames(d) == cfg_filenames_chk
assert [o['crc'] for o in d]==cfg_chk_crc
assert api.list_local_pipes()==[]
assert pipe.pull_preview() == cfg_filenames_chk
assert pipe.pull() == cfg_filenames_chk
assert pipe.pull_preview() == []
assert api.list_local_pipes()==[pipe.name]
assert pipe.files() == cfg_filenames_chk
assert pipe.filepaths() == [Path(pipe.dirpath)/f for f in pipe.files()]
assert pipe.filepaths(aspathlib=False) == [str(Path(pipe.dirpath)/f) for f in pipe.files()]
pipe = getpipe(api, chk_empty=False, mode='all')
assert pipe.pull_preview() == cfg_filenames_chk
# PipeLocal
pipelocal = d6tpipe.PipeLocal(pipe.name,profile=cfg_profile, filecfg=cfg_cfgfname)
assert pipelocal.files() == cfg_filenames_chk
assert pipelocal.scan_local() == cfg_filenames_chk
assert pipelocal.schema == cfg_settings_pipe['schema']
df = pd.read_csv(pipe.dirpath/cfg_filenames_chk[0], **pipe.schema['pandas'])
# permissions
if not testcfg.get('local',False):
api2 = getapi2(testcfg.get('local', False))
with pytest.raises(APIError, match='403'):
pipe2 = getpipe(api2, name=cfg_pipe_name, mode='all')
pipe2.pull()
settings = {"username": cfg_usr2, "role": "read"}
r,d = d6tpipe.upsert_permissions(api, cfg_parent_name, settings)
pipe2 = getpipe(api2, name=cfg_pipe_name, mode='all')
assert pipe2.pull()==cfg_filenames_chk
# cleanup
pipe.delete_files_local(confirm=False,delete_all=True)