def test_includeexclude(self, cleanup, init, pull):
api = getapi(local=True)
pipe = d6tpipe.pipe.Pipe(api, cfg_pipe_name)
assert pipe.scan_local(include='machinedata-2018-01*.csv', names_only=True)[1] == cfg_filenames_chk[:1]
assert pipe.scan_local(exclude='machinedata-2018-01*.csv', names_only=True)[1] == cfg_filenames_chk[1:]
assert pipe.files(include='machinedata-2018-01*.csv') == [pipe.dirpath/s for s in cfg_filenames_chk[:1]]
assert pipe.files(exclude='machinedata-2018-01*.csv') == [pipe.dirpath/s for s in cfg_filenames_chk[1:]]
def test_pipes_pull(self, cleanup, signup, parentinit, pipeinit, testcfg):
api = getapi(testcfg.get('local',False))
pipe = getpipe(api)
assert pipe.name in api.list_pipes()
cfg_chk_crc = ['8a9782e9efa8befa9752045ca506a62e',
'5fe579d6b71031dad399e8d4ea82820b',
'4c7da169df85253d7ff737dde1e7400b',
'ca62a122993494e763fd1676cce95e76']
# assert False
assert pipe.files() == []
assert pipe.scan_remote() == cfg_filenames_chk
r, d = pipe.scan_remote(attributes=True)
assert _filenames(d) == cfg_filenames_chk
assert [o['crc'] for o in d]==cfg_chk_crc
assert api.list_local_pipes()==[]
assert pipe.pull_preview() == cfg_filenames_chk
assert pipe.pull() == cfg_filenames_chk
assert pipe.pull_preview() == []
assert api.list_local_pipes()==[pipe.name]
assert pipe.files() == cfg_filenames_chk
assert pipe.filepaths() == [Path(pipe.dirpath)/f for f in pipe.files()]
assert pipe.filepaths(aspathlib=False) == [str(Path(pipe.dirpath)/f) for f in pipe.files()]
pipe = getpipe(api, chk_empty=False, mode='all')
assert pipe.pull_preview() == cfg_filenames_chk
# PipeLocal
pipelocal = d6tpipe.PipeLocal(pipe.name,profile=cfg_profile, filecfg=cfg_cfgfname)
assert pipelocal.files() == cfg_filenames_chk
assert pipelocal.scan_local() == cfg_filenames_chk
assert pipelocal.schema == cfg_settings_pipe['schema']
df = pd.read_csv(pipe.dirpath/cfg_filenames_chk[0], **pipe.schema['pandas'])
# permissions
if not testcfg.get('local',False):
api2 = getapi2(testcfg.get('local', False))
with pytest.raises(APIError, match='403'):
pipe2 = getpipe(api2, name=cfg_pipe_name, mode='all')
pipe2.pull()
settings = {"username": cfg_usr2, "role": "read"}
r,d = d6tpipe.upsert_permissions(api, cfg_parent_name, settings)
pipe2 = getpipe(api2, name=cfg_pipe_name, mode='all')
assert pipe2.pull()==cfg_filenames_chk
# cleanup
pipe.delete_files_local(confirm=False,delete_all=True)
def test_pipes_push(self, cleanup, signup, parentinit, pipeinit, testcfg):
api = getapi(testcfg.get('local', False))
pipe = getpipe(api, chk_empty=False)
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
pipe = getpipe(api)
with pytest.raises(PushError):
pipe.push_preview()
pipe.pull()
# push works
cfg_copyfile = 'test.csv'
df = pd.DataFrame({'a': range(10)})
df.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert set(pipe.scan_local()) == set(cfg_filenames_chk +
[cfg_copyfile])
assert pipe.files() == cfg_filenames_chk
assert pipe.push_preview() == [cfg_copyfile]
assert pipe.push() == [cfg_copyfile]
assert pipe.push_preview() == []
pipe._cache_scan.clear()
assert pipe.pull_preview() == []
# doesn't take files not meet pattern
cfg_copyfile2 = 'test.xlsx'
df.to_csv(pipe.dirpath / cfg_copyfile2)
assert pipe.push_preview() == []
(pipe.dirpath / cfg_copyfile2).unlink()
# todo: push exclude
# files() works
assert pipe.files() == cfg_filenames_chk + [cfg_copyfile]
assert pipe.files(include='Machine*.csv') == cfg_filenames_chk
assert pipe.files(exclude='Machine*.csv') == [cfg_copyfile]
assert pipe.files(sortby='mod')[-1] == cfg_copyfile
# crc works
df2 = pd.read_csv(pipe.dirpath / cfg_copyfile, **pipe.schema['pandas'])
df2.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert pipe.push_preview() == []
df.to_csv(pipe.dirpath / cfg_copyfile, index=True)
assert pipe.push_preview() == [cfg_copyfile]
# files param works
assert pipe.pull(files=[cfg_copyfile]) == [cfg_copyfile]
pipe.delete_files_remote(files=[cfg_copyfile], confirm=False)
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [False]
assert pipe.push(files=[cfg_copyfile]) == [cfg_copyfile]
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [True]
# remove_orphans works
(pipe.dirpath / cfg_copyfile).unlink()
pipe._cache_scan.clear()
assert pipe.remove_orphans(direction='both',
dryrun=True)['remote'] == [cfg_copyfile]
assert pipe.remove_orphans(direction='both',
dryrun=False)['remote'] == [cfg_copyfile]
assert pipe._pullpush_luigi(['test.csv'], 'exists') == [False]
# cleanup
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
# def test_pipes_includeexclude(self, cleanup, parentinit, pipeinit, testcfg):
# pass
'''