def test_autoload_missing_path(self):
with temppath() as module_path:
with temppath() as config_path:
config = Config(config_path)
config.add_section(config.global_section)
config.set(config.global_section, 'autoload.paths', module_path)
config._autoload()
def test_upload_overwrite(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello')
self.client.upload('up', tpath)
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('there')
self.client.upload('up', tpath, overwrite=True)
self._check_content('up', 'there')
def test_upload_overwrite(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello')
self.client.upload('up', tpath)
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('there')
self.client.upload('up', tpath, overwrite=True)
self._check_content('up', 'there')
def test_autoload_client_from_path(self):
with temppath() as module_path:
self._write_client_module(module_path, 'PathClient')
with temppath() as config_path:
config = Config(config_path)
config.add_section(config.global_section)
config.set(config.global_section, 'autoload.paths', module_path)
config._autoload()
client = Client.from_options({'url': ''}, 'PathClient')
eq_(client.one, 1)
def test_upload_overwrite(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello')
self.client.upload('up', tpath)
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('there')
self.client.upload('up', tpath, overwrite=True)
eq_(self._read('up'), b'there')
def test_upload_overwrite(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello')
self.client.upload('up', tpath)
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('there')
self.client.upload('up', tpath, overwrite=True)
eq_(self._read('up'), b'there')
def test_upload_overwrite(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello')
self.client.upload('up', tpath)
first_mtime = self._get_mtime('up')
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('there')
self.client.upload('up', tpath, overwrite=True)
second_mtime = self._get_mtime('up')
self._check_content('up', 'there')
ok_(second_mtime > first_mtime)
def test_from_local_path(self):
with temppath() as dpath:
os.mkdir(dpath)
fpath1 = osp.join(dpath, 'foo')
with open(fpath1, 'w') as writer:
writer.write('hey')
os.mkdir(osp.join(dpath, 'bar'))
fpath2 = osp.join(dpath, 'bar', 'baz')
with open(fpath2, 'w') as writer:
writer.write('hello')
with temppath() as tpath:
with open(tpath, 'w') as writer:
progress = _Progress.from_local_path(dpath, writer=writer)
eq_(progress._total_bytes, 8)
eq_(progress._pending_files, 2)
def test_from_local_path(self):
with temppath() as dpath:
os.mkdir(dpath)
fpath1 = osp.join(dpath, 'foo')
with open(fpath1, 'w') as writer:
writer.write('hey')
os.mkdir(osp.join(dpath, 'bar'))
fpath2 = osp.join(dpath, 'bar', 'baz')
with open(fpath2, 'w') as writer:
writer.write('hello')
with temppath() as tpath:
with open(tpath, 'w') as writer:
progress = _Progress.from_local_path(dpath, writer=writer)
eq_(progress._total_bytes, 8)
eq_(progress._pending_files, 2)
def test_autoload_client_from_module(self):
with temppath() as module_dpath:
os.mkdir(module_dpath)
sys.path.append(module_dpath)
module_fpath = osp.join(module_dpath, 'mclient.py')
self._write_client_module(module_fpath, 'ModuleClient')
try:
with temppath() as config_path:
config = Config(config_path)
config.add_section(config.global_section)
config.set(config.global_section, 'autoload.modules', 'mclient')
config._autoload()
client = Client.from_options({'url': ''}, 'ModuleClient')
eq_(client.one, 1)
finally:
sys.path.remove(module_dpath)
def test_read_part_file(self):
data = {
'part-m-00000.avro': [{
'name': 'jane'
}, {
'name': 'bob'
}],
'part-m-00001.avro': [{
'name': 'john'
}, {
'name': 'liz'
}],
}
for fname, records in data.items():
with AvroWriter(self.client, 'data.avro/%s' % (fname, )) as writer:
for record in records:
writer.write(record)
with temppath() as tpath:
with open(tpath, 'w') as writer:
main(['read', 'data.avro', '--parts', '1,'],
client=self.client,
stdout=writer)
with open(tpath) as reader:
records = [loads(line) for line in reader]
eq_(records, data['part-m-00001.avro'])
def test_download_file_to_existing_folder(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
os.mkdir(tpath)
self.client.download('dl', tpath)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
def test_nonpartitioned_file(self):
partname = 'part-r-00000'
self.client.write('dl/' + partname, 'world')
with temppath() as tpath:
fname = self.client.download('dl/' + partname, tpath)
with open(fname) as reader:
eq_(reader.read(), 'world')
def test_download_file_to_existing_folder_with_matching_file(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
os.mkdir(tpath)
with open(osp.join(tpath, 'dl'), 'w') as writer:
writer.write('hey')
self.client.download('dl', tpath)
def test_download_file_to_existing_folder_with_matching_file(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
os.mkdir(tpath)
with open(osp.join(tpath, 'dl'), 'w') as writer:
writer.write('hey')
self.client.download('dl', tpath)
def test_read_file_from_offset_with_limit(self):
self.client.write('foo', 'hello, world!')
with temppath() as tpath:
with open(tpath, 'w') as writer:
self._read(writer, 'foo', offset=7, length=5)
with open(tpath) as reader:
eq_(reader.read(), 'world')
def test_create_from_file_object(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello, world!')
with open(tpath) as reader:
self.client.write('up', reader)
eq_(self._read('up'), b'hello, world!')
def test_download_file_to_existing_folder(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
os.mkdir(tpath)
self.client.download('dl', tpath)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
def test_nonpartitioned_file(self):
partname = 'part-r-00000'
self.client.write('dl/' + partname, 'world')
with temppath() as tpath:
fname = self.client.download('dl/' + partname, tpath)
with open(fname) as reader:
eq_(reader.read(), 'world')
def test_create_from_file_object(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello, world!')
with open(tpath) as reader:
self.client.write('up', reader)
eq_(self._read('up'), b'hello, world!')
def test_read_file(self):
self.client.write('foo', 'hello, world!')
with temppath() as tpath:
with open(tpath, 'w') as writer:
self._read(writer, 'foo')
with open(tpath) as reader:
eq_(reader.read(), 'hello, world!')
def test_create_from_file_object(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hello, world!')
with open(tpath) as reader:
self.client.write('up', reader)
self._check_content('up', 'hello, world!')
def test_schema(self):
self.client.upload('weather.avro', osp.join(self.dpath, 'weather.avro'))
with temppath() as tpath:
with open(tpath, 'w') as writer:
main(['schema', 'weather.avro'], client=self.client, stdout=writer)
with open(tpath) as reader:
schema = load(reader)
eq_(self.schema, schema)
def test_download_overwrite(self):
self.client.upload('foo', self.dpath)
with temppath() as tpath:
with open(tpath, 'w'):
pass
main(['download', 'foo', tpath, '--silent', '--threads', '1'],
self.client)
self._dircmp(tpath)
def test_create_client_with_alias(self):
with temppath() as tpath:
config = Config(path=tpath)
section = 'dev.alias'
config.add_section(section)
config.set(section, 'url', 'http://host:port')
save_config(config)
Config(path=tpath).get_client('dev')
def test_download_file_to_existing_file_with_overwrite(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hi')
self.client.download('dl', tpath, overwrite=True)
with open(tpath) as reader:
eq_(reader.read(), 'hello')
def test_upload(self):
main(
['upload', self.dpath, 'bar', '--silent', '--threads', '1'],
self.client
)
with temppath() as tpath:
self.client.download('bar', tpath)
self._dircmp(tpath)
def test_singly_partitioned_file(self):
partname = 'part-r-00000'
self.client.write('dl/' + partname, 'world')
with temppath() as tpath:
os.mkdir(tpath)
fname = self.client.download('dl', tpath)
with open(osp.join(fname, partname)) as reader:
eq_(reader.read(), 'world')
def test_download(self):
self.client.upload('foo', self.dpath)
with temppath() as tpath:
main(
['download', 'foo', tpath, '--silent', '--threads', '1'],
self.client
)
self._dircmp(tpath)
def test_upload(self):
main(
['upload', self.dpath, 'bar', '--silent', '--threads', '1'],
self.client
)
with temppath() as tpath:
self.client.download('bar', tpath)
self._dircmp(tpath)
def test_schema(self):
self.client.upload('weather.avro', osp.join(self.dpath, 'weather.avro'))
with temppath() as tpath:
with open(tpath, 'w') as writer:
main(['schema', 'weather.avro'], client=self.client, stdout=writer)
with open(tpath) as reader:
schema = load(reader)
eq_(self.schema, schema)
def test_download_file_to_existing_file_with_overwrite(self):
self.client.write('dl', 'hello')
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hi')
self.client.download('dl', tpath, overwrite=True)
with open(tpath) as reader:
eq_(reader.read(), 'hello')
def test_download(self):
self.client.upload('foo', self.dpath)
with temppath() as tpath:
main(
['download', 'foo', tpath, '--silent', '--threads', '1'],
self.client
)
self._dircmp(tpath)
def test_overwrite_file(self):
with temppath() as tpath:
self.client.write('dl', 'hello')
self.client.download('dl', tpath)
self.client.write('dl', 'there', overwrite=True)
fname = self.client.download('dl', tpath, overwrite=True)
with open(fname) as reader:
eq_(reader.read(), 'there')
def test_overwrite_file(self):
with temppath() as tpath:
self.client.write('dl', 'hello')
self.client.download('dl', tpath)
self.client.write('dl', 'there', overwrite=True)
fname = self.client.download('dl', tpath, overwrite=True)
with open(fname) as reader:
eq_(reader.read(), 'there')
def test_singly_partitioned_file(self):
partname = 'part-r-00000'
self.client.write('dl/' + partname, 'world')
with temppath() as tpath:
os.mkdir(tpath)
fname = self.client.download('dl', tpath)
with open(osp.join(fname, partname)) as reader:
eq_(reader.read(), 'world')
def test_download_folder_to_missing_folder(self):
self.client.write('foo/dl', 'hello')
self.client.write('foo/bar/dl', 'there')
with temppath() as tpath:
self.client.download('foo', tpath)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
with open(osp.join(tpath, 'bar', 'dl')) as reader:
eq_(reader.read(), 'there')
def test_upload_force(self):
self.client.write('bar', 'hey')
main([
'upload', self.dpath, 'bar', '--silent', '--threads', '1',
'--force'
], self.client)
with temppath() as tpath:
self.client.download('bar', tpath)
self._dircmp(tpath)
def test_upload_force(self):
self.client.write('bar', 'hey')
main(
['upload', self.dpath, 'bar', '--silent', '--threads', '1', '--force'],
self.client
)
with temppath() as tpath:
self.client.download('bar', tpath)
self._dircmp(tpath)
def test_download_file_to_existing_folder_overwrite_matching_file(self):
self._write('dl', b'hello')
with temppath() as tpath:
os.mkdir(tpath)
with open(osp.join(tpath, 'dl'), 'w') as writer:
writer.write('hey')
self.client.download('dl', tpath, overwrite=True)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
def test_download_file_to_existing_folder_overwrite_matching_file(self):
self._write('dl', b'hello')
with temppath() as tpath:
os.mkdir(tpath)
with open(osp.join(tpath, 'dl'), 'w') as writer:
writer.write('hey')
self.client.download('dl', tpath, overwrite=True)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
def test_normal_read(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('abcd')
with open(tpath) as reader:
sreader = _SeekableReader(reader)
eq_(sreader.read(3), 'abc')
eq_(sreader.read(2), 'd')
ok_(not sreader.read(1))
def test_download_folder_to_missing_folder(self):
self.client.write('foo/dl', 'hello')
self.client.write('foo/bar/dl', 'there')
with temppath() as tpath:
self.client.download('foo', tpath)
with open(osp.join(tpath, 'dl')) as reader:
eq_(reader.read(), 'hello')
with open(osp.join(tpath, 'bar', 'dl')) as reader:
eq_(reader.read(), 'there')
def test_normal_read(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('abcd')
with open(tpath) as reader:
sreader = _SeekableReader(reader)
eq_(sreader.read(3), 'abc')
eq_(sreader.read(2), 'd')
ok_(not sreader.read(1))
def test_disable_file_logging(self):
with temppath() as tpath:
config = Config(tpath)
config.add_section('cmd.command')
config.set('cmd.command', 'log.disable', 'true')
save_config(config)
config = Config(tpath)
handler = config.get_log_handler('cmd')
ok_(not isinstance(handler, TimedRotatingFileHandler))
def test_download_folder_to_existing_folder_parallel(self):
self.client.write('foo/dl', 'hello')
self.client.write('foo/bar/dl', 'there')
with temppath() as tpath:
os.mkdir(tpath)
self.client.download('foo', tpath, n_threads=0)
with open(osp.join(tpath, 'foo', 'dl')) as reader:
eq_(reader.read(), 'hello')
with open(osp.join(tpath, 'foo', 'bar', 'dl')) as reader:
eq_(reader.read(), 'there')
def test_with_alias(self):
url = 'http://host:port'
with temppath() as tpath:
config = Config(path=tpath)
section = 'dev.alias'
config.add_section(section)
config.set(section, 'url', url)
args = {'--alias': 'dev', '--log': False, '--verbose': 0}
client = configure_client('test', args, config=config)
eq_(client.url, url)
def test_download_overwrite(self):
self.client.upload('foo', self.dpath)
with temppath() as tpath:
with open(tpath, 'w'):
pass
main(
['download', 'foo', tpath, '--silent', '--threads', '1'],
self.client
)
self._dircmp(tpath)
def test_with_alias(self):
url = 'http://host:port'
with temppath() as tpath:
config = Config(path=tpath)
section = 'dev.alias'
config.add_section(section)
config.set(section, 'url', url)
args = {'--alias': 'dev', '--log': False, '--verbose': 0}
client = configure_client('test', args, config=config)
eq_(client.url, url)
def test_download_folder_to_existing_folder_parallel(self):
self.client.write('foo/dl', 'hello')
self.client.write('foo/bar/dl', 'there')
with temppath() as tpath:
os.mkdir(tpath)
self.client.download('foo', tpath, n_threads=0)
with open(osp.join(tpath, 'foo', 'dl')) as reader:
eq_(reader.read(), 'hello')
with open(osp.join(tpath, 'foo', 'bar', 'dl')) as reader:
eq_(reader.read(), 'there')
def test_upload_append(self):
with temppath() as tpath:
with open(tpath, 'w') as writer:
writer.write('hey')
main(['upload', tpath, 'bar', '--silent', '--threads', '1'], self.client)
main(
['upload', tpath, 'bar', '--silent', '--threads', '1', '--append'],
self.client
)
with self.client.read('bar') as reader:
eq_(reader.read(), b'heyhey')
