def test_includeexclude(self, cleanup, init, pull):
api = getapi(local=True)
pipe = d6tpipe.pipe.Pipe(api, cfg_pipe_name)
assert pipe.scan_local(include='machinedata-2018-01*.csv', names_only=True)[1] == cfg_filenames_chk[:1]
assert pipe.scan_local(exclude='machinedata-2018-01*.csv', names_only=True)[1] == cfg_filenames_chk[1:]
assert pipe.files(include='machinedata-2018-01*.csv') == [pipe.dirpath/s for s in cfg_filenames_chk[:1]]
assert pipe.files(exclude='machinedata-2018-01*.csv') == [pipe.dirpath/s for s in cfg_filenames_chk[1:]]
def test_ftp_base(self, cleanup, signup, setup_ftp_base, testcfg):
# assert False
pipe = setup_ftp_base
files = pipe.scan_remote()
assert files[0][0] != '/' # check no root dir
files2 = pipe.pull()
assert len(files2) == len(files)
assert len(pipe.scan_local()) == len(files)
def test_pipes_push(self, cleanup, signup, parentinit, pipeinit, testcfg):
api = getapi(testcfg.get('local', False))
pipe = getpipe(api, chk_empty=False)
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
pipe = getpipe(api)
with pytest.raises(PushError):
pipe.push_preview()
pipe.pull()
# push works
cfg_copyfile = 'test.csv'
df = pd.DataFrame({'a': range(10)})
df.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert set(pipe.scan_local()) == set(cfg_filenames_chk +
[cfg_copyfile])
assert pipe.files() == cfg_filenames_chk
assert pipe.push_preview() == [cfg_copyfile]
assert pipe.push() == [cfg_copyfile]
assert pipe.push_preview() == []
pipe._cache_scan.clear()
assert pipe.pull_preview() == []
# doesn't take files not meet pattern
cfg_copyfile2 = 'test.xlsx'
df.to_csv(pipe.dirpath / cfg_copyfile2)
assert pipe.push_preview() == []
(pipe.dirpath / cfg_copyfile2).unlink()
# todo: push exclude
# files() works
assert pipe.files() == cfg_filenames_chk + [cfg_copyfile]
assert pipe.files(include='Machine*.csv') == cfg_filenames_chk
assert pipe.files(exclude='Machine*.csv') == [cfg_copyfile]
assert pipe.files(sortby='mod')[-1] == cfg_copyfile
# crc works
df2 = pd.read_csv(pipe.dirpath / cfg_copyfile, **pipe.schema['pandas'])
df2.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert pipe.push_preview() == []
df.to_csv(pipe.dirpath / cfg_copyfile, index=True)
assert pipe.push_preview() == [cfg_copyfile]
# files param works
assert pipe.pull(files=[cfg_copyfile]) == [cfg_copyfile]
pipe.delete_files_remote(files=[cfg_copyfile], confirm=False)
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [False]
assert pipe.push(files=[cfg_copyfile]) == [cfg_copyfile]
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [True]
# remove_orphans works
(pipe.dirpath / cfg_copyfile).unlink()
pipe._cache_scan.clear()
assert pipe.remove_orphans(direction='both',
dryrun=True)['remote'] == [cfg_copyfile]
assert pipe.remove_orphans(direction='both',
dryrun=False)['remote'] == [cfg_copyfile]
assert pipe._pullpush_luigi(['test.csv'], 'exists') == [False]
# cleanup
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
# def test_pipes_includeexclude(self, cleanup, parentinit, pipeinit, testcfg):
# pass
'''