def test_convert_through_temporary_local_storage():
with filetext('name,quantity\nAlice,100\nBob,200', extension='csv') as fn:
csv = CSV(fn)
df = into(pd.DataFrame, csv)
scsv = into(Temp(SSH(CSV)), csv, hostname='localhost')
assert into(list, csv) == into(list, scsv)
scsv2 = into(Temp(SSH(CSV)), df, hostname='localhost')
assert into(list, scsv2) == into(list, df)
sjson = into(Temp(SSH(JSONLines)), df, hostname='localhost')
assert (into(np.ndarray, sjson) == into(np.ndarray, df)).all()
def test_temp_ssh_files():
with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
csv = CSV(fn)
scsv = into(Temp(SSH(CSV)), csv, hostname='localhost')
assert discover(csv) == discover(scsv)
assert isinstance(scsv, _Temp)
def test_s3_to_ssh():
pytest.importorskip('boto')
tips_uri = 's3://nyqpug/tips.csv'
with tmpfile('.csv') as fn:
result = into(Temp(SSH(CSV))(fn, hostname='localhost'), tips_uri)
assert into(list, result) == into(list, tips_uri)
assert discover(result) == discover(resource(tips_uri))
def test_ssh_csv_to_s3_csv():
# for some reason this can only be run in the same file as other ssh tests
# and must be a Temp(SSH(CSV)) otherwise tests above this one fail
s3_bucket = pytest.importorskip('odo.backends.tests.test_aws').s3_bucket
with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
remote = into(Temp(SSH(CSV)), CSV(fn), hostname='localhost')
with s3_bucket('.csv') as b:
result = into(b, remote)
assert discover(result) == discover(resource(b))
def test_copy_remote_csv():
with tmpfile('csv') as target:
with filetext('name,balance\nAlice,100\nBob,200',
extension='csv') as fn:
csv = resource(fn)
uri = 'ssh://localhost:%s.csv' % target
scsv = into(uri, csv)
assert isinstance(scsv, SSH(CSV))
assert discover(scsv) == discover(csv)
# Round trip
csv2 = into(target, scsv)
assert into(list, csv) == into(list, csv2)
def test_drop():
with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
with tmpfile('csv') as target:
scsv = SSH(CSV)(target, hostname='localhost')
assert not os.path.exists(target)
conn = sftp(**scsv.auth)
conn.put(fn, target)
assert os.path.exists(target)
drop(scsv)
drop(scsv)
assert not os.path.exists(target)
def test_discover():
with filetext('name,balance\nAlice,100\nBob,200') as fn:
local = CSV(fn)
remote = SSH(CSV)(fn, hostname='localhost')
assert discover(local) == discover(remote)
def test_resource():
r = resource('ssh://joe@localhost:/path/to/myfile.csv')
assert isinstance(r, SSH(CSV))
assert r.path == '/path/to/myfile.csv'
assert r.auth['hostname'] == 'localhost'
assert r.auth['username'] == 'joe'
def test_convert_local_file_to_temp_ssh_file():
with filetext('name,balance\nAlice,100\nBob,200', extension='csv') as fn:
csv = CSV(fn)
scsv = convert(Temp(SSH(CSV)), csv, hostname='localhost')
assert into(list, csv) == into(list, scsv)
def test_drop_of_csv_json_lines_use_ssh_version():
from odo.backends.ssh import drop_ssh
for typ in [CSV, JSON, JSONLines]:
assert drop.dispatch(SSH(typ)) == drop_ssh