def test_sftp(self, cleanup, signup, testcfg):
cfg_name = cfg_settings_pipe_sftp['name']
api = getapi(testcfg.get('local', False))
# test quick create
d6tpipe.upsert_pipe(api, cfg_settings_parent_sftp)
d6tpipe.upsert_pipe(api, cfg_settings_pipe_sftp)
# test paths
r, d = api.cnxn.pipes._(cfg_settings_parent_sftp['name']).get()
assert d['options']['remotepath'] == '/'
r, d = api.cnxn.pipes._(cfg_settings_pipe_sftp['name']).get()
assert d['options'][
'remotepath'] == cfg_settings_pipe_sftp['options']['dir'] + '/'
# test push/pull
pipe = getpipe(api, name=cfg_name, mode='all')
pipe.delete_files_remote(confirm=False)
cfg_copyfile = 'test.csv'
df = pd.DataFrame({'a': range(10)})
df.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert pipe.scan_remote(cached=False) == []
assert pipe.pull() == []
assert pipe.push_preview() == [cfg_copyfile]
assert pipe.push() == [cfg_copyfile]
pipe._cache_scan.clear()
assert pipe.pull() == [cfg_copyfile]
# cleanup
pipe.delete_files_remote(confirm=False)
assert pipe.scan_remote(cached=False) == []
pipe.delete_files_local(confirm=False, delete_all=True)
def test_ftp(self, cleanup, signup, testcfg):
api = getapi(testcfg.get('local', False))
# test quick create
d6tpipe.upsert_pipe_json(api, 'tests/.creds-test.json',
'pipe-test-ftp')
cfg_name = 'test-ftp'
# test paths
r, d = api.cnxn.pipes._(cfg_name).get()
assert d['options']['remotepath'] == '/utest/'
# test push/pull
pipe = getpipe(api, name=cfg_name, mode='all')
cfg_copyfile = 'test.csv'
df = pd.DataFrame({'a': range(10)})
df.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert pipe.scan_remote(cached=False) == []
assert pipe.pull() == []
assert pipe.push_preview() == [cfg_copyfile]
assert pipe.push() == [cfg_copyfile]
pipe._cache_scan.clear()
assert pipe.pull() == [cfg_copyfile]
pipe.delete_files(confirm=False, all_local=True)
assert pipe.scan_remote(cached=False) == []
def test_pipes_push(self, cleanup, signup, parentinit, pipeinit, testcfg):
api = getapi(testcfg.get('local', False))
pipe = getpipe(api, chk_empty=False)
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
pipe = getpipe(api)
with pytest.raises(PushError):
pipe.push_preview()
pipe.pull()
# push works
cfg_copyfile = 'test.csv'
df = pd.DataFrame({'a': range(10)})
df.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert set(pipe.scan_local()) == set(cfg_filenames_chk +
[cfg_copyfile])
assert pipe.files() == cfg_filenames_chk
assert pipe.push_preview() == [cfg_copyfile]
assert pipe.push() == [cfg_copyfile]
assert pipe.push_preview() == []
pipe._cache_scan.clear()
assert pipe.pull_preview() == []
# doesn't take files not meet pattern
cfg_copyfile2 = 'test.xlsx'
df.to_csv(pipe.dirpath / cfg_copyfile2)
assert pipe.push_preview() == []
(pipe.dirpath / cfg_copyfile2).unlink()
# todo: push exclude
# files() works
assert pipe.files() == cfg_filenames_chk + [cfg_copyfile]
assert pipe.files(include='Machine*.csv') == cfg_filenames_chk
assert pipe.files(exclude='Machine*.csv') == [cfg_copyfile]
assert pipe.files(sortby='mod')[-1] == cfg_copyfile
# crc works
df2 = pd.read_csv(pipe.dirpath / cfg_copyfile, **pipe.schema['pandas'])
df2.to_csv(pipe.dirpath / cfg_copyfile, index=False)
assert pipe.push_preview() == []
df.to_csv(pipe.dirpath / cfg_copyfile, index=True)
assert pipe.push_preview() == [cfg_copyfile]
# files param works
assert pipe.pull(files=[cfg_copyfile]) == [cfg_copyfile]
pipe.delete_files_remote(files=[cfg_copyfile], confirm=False)
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [False]
assert pipe.push(files=[cfg_copyfile]) == [cfg_copyfile]
assert pipe._pullpush_luigi([cfg_copyfile], 'exists') == [True]
# remove_orphans works
(pipe.dirpath / cfg_copyfile).unlink()
pipe._cache_scan.clear()
assert pipe.remove_orphans(direction='both',
dryrun=True)['remote'] == [cfg_copyfile]
assert pipe.remove_orphans(direction='both',
dryrun=False)['remote'] == [cfg_copyfile]
assert pipe._pullpush_luigi(['test.csv'], 'exists') == [False]
# cleanup
pipe.delete_files_local(confirm=False, delete_all=True)
assert pipe.scan_local() == []
# def test_pipes_includeexclude(self, cleanup, parentinit, pipeinit, testcfg):
# pass
'''