Exemple #1
0
 def test_autoload_missing_path(self):
   with temppath() as module_path:
     with temppath() as config_path:
       config = Config(config_path)
       config.add_section(config.global_section)
       config.set(config.global_section, 'autoload.paths', module_path)
       config._autoload()
Exemple #2
0
 def test_upload_overwrite(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello')
     self.client.upload('up', tpath)
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('there')
     self.client.upload('up', tpath, overwrite=True)
   self._check_content('up', 'there')
Exemple #3
0
 def test_upload_overwrite(self):
     with temppath() as tpath:
         with open(tpath, 'w') as writer:
             writer.write('hello')
         self.client.upload('up', tpath)
     with temppath() as tpath:
         with open(tpath, 'w') as writer:
             writer.write('there')
         self.client.upload('up', tpath, overwrite=True)
     self._check_content('up', 'there')
Exemple #4
0
 def test_autoload_client_from_path(self):
   with temppath() as module_path:
     self._write_client_module(module_path, 'PathClient')
     with temppath() as config_path:
       config = Config(config_path)
       config.add_section(config.global_section)
       config.set(config.global_section, 'autoload.paths', module_path)
       config._autoload()
       client = Client.from_options({'url': ''}, 'PathClient')
       eq_(client.one, 1)
Exemple #5
0
 def test_upload_overwrite(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello')
     self.client.upload('up', tpath)
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('there')
     self.client.upload('up', tpath, overwrite=True)
   eq_(self._read('up'), b'there')
Exemple #6
0
 def test_upload_overwrite(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello')
     self.client.upload('up', tpath)
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('there')
     self.client.upload('up', tpath, overwrite=True)
   eq_(self._read('up'), b'there')
Exemple #7
0
 def test_upload_overwrite(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello')
     self.client.upload('up', tpath)
     first_mtime = self._get_mtime('up')
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('there')
     self.client.upload('up', tpath, overwrite=True)
     second_mtime = self._get_mtime('up')
   self._check_content('up', 'there')
   ok_(second_mtime > first_mtime)
Exemple #8
0
 def test_from_local_path(self):
     with temppath() as dpath:
         os.mkdir(dpath)
         fpath1 = osp.join(dpath, 'foo')
         with open(fpath1, 'w') as writer:
             writer.write('hey')
         os.mkdir(osp.join(dpath, 'bar'))
         fpath2 = osp.join(dpath, 'bar', 'baz')
         with open(fpath2, 'w') as writer:
             writer.write('hello')
         with temppath() as tpath:
             with open(tpath, 'w') as writer:
                 progress = _Progress.from_local_path(dpath, writer=writer)
                 eq_(progress._total_bytes, 8)
                 eq_(progress._pending_files, 2)
Exemple #9
0
 def test_from_local_path(self):
   with temppath() as dpath:
     os.mkdir(dpath)
     fpath1 = osp.join(dpath, 'foo')
     with open(fpath1, 'w') as writer:
       writer.write('hey')
     os.mkdir(osp.join(dpath, 'bar'))
     fpath2 = osp.join(dpath, 'bar', 'baz')
     with open(fpath2, 'w') as writer:
       writer.write('hello')
     with temppath() as tpath:
       with open(tpath, 'w') as writer:
         progress = _Progress.from_local_path(dpath, writer=writer)
         eq_(progress._total_bytes, 8)
         eq_(progress._pending_files, 2)
Exemple #10
0
 def test_autoload_client_from_module(self):
   with temppath() as module_dpath:
     os.mkdir(module_dpath)
     sys.path.append(module_dpath)
     module_fpath = osp.join(module_dpath, 'mclient.py')
     self._write_client_module(module_fpath, 'ModuleClient')
     try:
       with temppath() as config_path:
         config = Config(config_path)
         config.add_section(config.global_section)
         config.set(config.global_section, 'autoload.modules', 'mclient')
         config._autoload()
         client = Client.from_options({'url': ''}, 'ModuleClient')
         eq_(client.one, 1)
     finally:
       sys.path.remove(module_dpath)
Exemple #11
0
 def test_read_part_file(self):
     data = {
         'part-m-00000.avro': [{
             'name': 'jane'
         }, {
             'name': 'bob'
         }],
         'part-m-00001.avro': [{
             'name': 'john'
         }, {
             'name': 'liz'
         }],
     }
     for fname, records in data.items():
         with AvroWriter(self.client, 'data.avro/%s' % (fname, )) as writer:
             for record in records:
                 writer.write(record)
     with temppath() as tpath:
         with open(tpath, 'w') as writer:
             main(['read', 'data.avro', '--parts', '1,'],
                  client=self.client,
                  stdout=writer)
         with open(tpath) as reader:
             records = [loads(line) for line in reader]
         eq_(records, data['part-m-00001.avro'])
Exemple #12
0
 def test_download_file_to_existing_folder(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     os.mkdir(tpath)
     self.client.download('dl', tpath)
     with open(osp.join(tpath, 'dl')) as reader:
       eq_(reader.read(), 'hello')
Exemple #13
0
 def test_nonpartitioned_file(self):
   partname = 'part-r-00000'
   self.client.write('dl/' + partname, 'world')
   with temppath() as tpath:
     fname = self.client.download('dl/' + partname, tpath)
     with open(fname) as reader:
       eq_(reader.read(), 'world')
Exemple #14
0
 def test_download_file_to_existing_folder_with_matching_file(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     os.mkdir(tpath)
     with open(osp.join(tpath, 'dl'), 'w') as writer:
       writer.write('hey')
     self.client.download('dl', tpath)
Exemple #15
0
 def test_download_file_to_existing_folder_with_matching_file(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     os.mkdir(tpath)
     with open(osp.join(tpath, 'dl'), 'w') as writer:
       writer.write('hey')
     self.client.download('dl', tpath)
Exemple #16
0
 def test_read_file_from_offset_with_limit(self):
   self.client.write('foo', 'hello, world!')
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       self._read(writer, 'foo', offset=7, length=5)
     with open(tpath) as reader:
       eq_(reader.read(), 'world')
Exemple #17
0
 def test_create_from_file_object(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello, world!')
     with open(tpath) as reader:
       self.client.write('up', reader)
   eq_(self._read('up'), b'hello, world!')
Exemple #18
0
 def test_download_file_to_existing_folder(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     os.mkdir(tpath)
     self.client.download('dl', tpath)
     with open(osp.join(tpath, 'dl')) as reader:
       eq_(reader.read(), 'hello')
Exemple #19
0
 def test_nonpartitioned_file(self):
   partname = 'part-r-00000'
   self.client.write('dl/' + partname, 'world')
   with temppath() as tpath:
     fname = self.client.download('dl/' + partname, tpath)
     with open(fname) as reader:
       eq_(reader.read(), 'world')
Exemple #20
0
 def test_create_from_file_object(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello, world!')
     with open(tpath) as reader:
       self.client.write('up', reader)
   eq_(self._read('up'), b'hello, world!')
Exemple #21
0
 def test_read_file(self):
   self.client.write('foo', 'hello, world!')
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       self._read(writer, 'foo')
     with open(tpath) as reader:
       eq_(reader.read(), 'hello, world!')
Exemple #22
0
 def test_create_from_file_object(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hello, world!')
     with open(tpath) as reader:
       self.client.write('up', reader)
   self._check_content('up', 'hello, world!')
Exemple #23
0
 def test_schema(self):
   self.client.upload('weather.avro', osp.join(self.dpath, 'weather.avro'))
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       main(['schema', 'weather.avro'], client=self.client, stdout=writer)
     with open(tpath) as reader:
       schema = load(reader)
     eq_(self.schema, schema)
Exemple #24
0
 def test_download_overwrite(self):
     self.client.upload('foo', self.dpath)
     with temppath() as tpath:
         with open(tpath, 'w'):
             pass
         main(['download', 'foo', tpath, '--silent', '--threads', '1'],
              self.client)
         self._dircmp(tpath)
Exemple #25
0
 def test_create_client_with_alias(self):
   with temppath() as tpath:
     config = Config(path=tpath)
     section = 'dev.alias'
     config.add_section(section)
     config.set(section, 'url', 'http://host:port')
     save_config(config)
     Config(path=tpath).get_client('dev')
Exemple #26
0
 def test_download_file_to_existing_file_with_overwrite(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hi')
     self.client.download('dl', tpath, overwrite=True)
     with open(tpath) as reader:
       eq_(reader.read(), 'hello')
Exemple #27
0
 def test_upload(self):
   main(
     ['upload', self.dpath, 'bar', '--silent', '--threads', '1'],
     self.client
   )
   with temppath() as tpath:
     self.client.download('bar', tpath)
     self._dircmp(tpath)
Exemple #28
0
 def test_singly_partitioned_file(self):
   partname = 'part-r-00000'
   self.client.write('dl/' + partname, 'world')
   with temppath() as tpath:
     os.mkdir(tpath)
     fname = self.client.download('dl', tpath)
     with open(osp.join(fname, partname)) as reader:
       eq_(reader.read(), 'world')
Exemple #29
0
 def test_download(self):
   self.client.upload('foo', self.dpath)
   with temppath() as tpath:
     main(
       ['download', 'foo', tpath, '--silent', '--threads', '1'],
       self.client
     )
     self._dircmp(tpath)
Exemple #30
0
 def test_upload(self):
   main(
     ['upload', self.dpath, 'bar', '--silent', '--threads', '1'],
     self.client
   )
   with temppath() as tpath:
     self.client.download('bar', tpath)
     self._dircmp(tpath)
Exemple #31
0
 def test_schema(self):
   self.client.upload('weather.avro', osp.join(self.dpath, 'weather.avro'))
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       main(['schema', 'weather.avro'], client=self.client, stdout=writer)
     with open(tpath) as reader:
       schema = load(reader)
     eq_(self.schema, schema)
Exemple #32
0
 def test_download_file_to_existing_file_with_overwrite(self):
   self.client.write('dl', 'hello')
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hi')
     self.client.download('dl', tpath, overwrite=True)
     with open(tpath) as reader:
       eq_(reader.read(), 'hello')
Exemple #33
0
 def test_download(self):
   self.client.upload('foo', self.dpath)
   with temppath() as tpath:
     main(
       ['download', 'foo', tpath, '--silent', '--threads', '1'],
       self.client
     )
     self._dircmp(tpath)
Exemple #34
0
 def test_overwrite_file(self):
   with temppath() as tpath:
     self.client.write('dl', 'hello')
     self.client.download('dl', tpath)
     self.client.write('dl', 'there', overwrite=True)
     fname = self.client.download('dl', tpath, overwrite=True)
     with open(fname) as reader:
       eq_(reader.read(), 'there')
Exemple #35
0
 def test_overwrite_file(self):
   with temppath() as tpath:
     self.client.write('dl', 'hello')
     self.client.download('dl', tpath)
     self.client.write('dl', 'there', overwrite=True)
     fname = self.client.download('dl', tpath, overwrite=True)
     with open(fname) as reader:
       eq_(reader.read(), 'there')
Exemple #36
0
 def test_singly_partitioned_file(self):
   partname = 'part-r-00000'
   self.client.write('dl/' + partname, 'world')
   with temppath() as tpath:
     os.mkdir(tpath)
     fname = self.client.download('dl', tpath)
     with open(osp.join(fname, partname)) as reader:
       eq_(reader.read(), 'world')
Exemple #37
0
 def test_download_folder_to_missing_folder(self):
   self.client.write('foo/dl', 'hello')
   self.client.write('foo/bar/dl', 'there')
   with temppath() as tpath:
     self.client.download('foo', tpath)
     with open(osp.join(tpath, 'dl')) as reader:
       eq_(reader.read(), 'hello')
     with open(osp.join(tpath, 'bar', 'dl')) as reader:
       eq_(reader.read(), 'there')
Exemple #38
0
 def test_upload_force(self):
     self.client.write('bar', 'hey')
     main([
         'upload', self.dpath, 'bar', '--silent', '--threads', '1',
         '--force'
     ], self.client)
     with temppath() as tpath:
         self.client.download('bar', tpath)
         self._dircmp(tpath)
Exemple #39
0
 def test_upload_force(self):
   self.client.write('bar', 'hey')
   main(
     ['upload', self.dpath, 'bar', '--silent', '--threads', '1', '--force'],
     self.client
   )
   with temppath() as tpath:
     self.client.download('bar', tpath)
     self._dircmp(tpath)
Exemple #40
0
 def test_download_file_to_existing_folder_overwrite_matching_file(self):
     self._write('dl', b'hello')
     with temppath() as tpath:
         os.mkdir(tpath)
         with open(osp.join(tpath, 'dl'), 'w') as writer:
             writer.write('hey')
         self.client.download('dl', tpath, overwrite=True)
         with open(osp.join(tpath, 'dl')) as reader:
             eq_(reader.read(), 'hello')
Exemple #41
0
 def test_download_file_to_existing_folder_overwrite_matching_file(self):
   self._write('dl', b'hello')
   with temppath() as tpath:
     os.mkdir(tpath)
     with open(osp.join(tpath, 'dl'), 'w') as writer:
       writer.write('hey')
     self.client.download('dl', tpath, overwrite=True)
     with open(osp.join(tpath, 'dl')) as reader:
       eq_(reader.read(), 'hello')
Exemple #42
0
 def test_normal_read(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('abcd')
     with open(tpath) as reader:
       sreader = _SeekableReader(reader)
       eq_(sreader.read(3), 'abc')
       eq_(sreader.read(2), 'd')
       ok_(not sreader.read(1))
Exemple #43
0
 def test_download_folder_to_missing_folder(self):
   self.client.write('foo/dl', 'hello')
   self.client.write('foo/bar/dl', 'there')
   with temppath() as tpath:
     self.client.download('foo', tpath)
     with open(osp.join(tpath, 'dl')) as reader:
       eq_(reader.read(), 'hello')
     with open(osp.join(tpath, 'bar', 'dl')) as reader:
       eq_(reader.read(), 'there')
Exemple #44
0
 def test_normal_read(self):
     with temppath() as tpath:
         with open(tpath, 'w') as writer:
             writer.write('abcd')
         with open(tpath) as reader:
             sreader = _SeekableReader(reader)
             eq_(sreader.read(3), 'abc')
             eq_(sreader.read(2), 'd')
             ok_(not sreader.read(1))
Exemple #45
0
 def test_disable_file_logging(self):
   with temppath() as tpath:
     config = Config(tpath)
     config.add_section('cmd.command')
     config.set('cmd.command', 'log.disable', 'true')
     save_config(config)
     config = Config(tpath)
     handler = config.get_log_handler('cmd')
     ok_(not isinstance(handler, TimedRotatingFileHandler))
Exemple #46
0
 def test_download_folder_to_existing_folder_parallel(self):
   self.client.write('foo/dl', 'hello')
   self.client.write('foo/bar/dl', 'there')
   with temppath() as tpath:
     os.mkdir(tpath)
     self.client.download('foo', tpath, n_threads=0)
     with open(osp.join(tpath, 'foo', 'dl')) as reader:
       eq_(reader.read(), 'hello')
     with open(osp.join(tpath, 'foo', 'bar', 'dl')) as reader:
       eq_(reader.read(), 'there')
Exemple #47
0
 def test_with_alias(self):
     url = 'http://host:port'
     with temppath() as tpath:
         config = Config(path=tpath)
         section = 'dev.alias'
         config.add_section(section)
         config.set(section, 'url', url)
         args = {'--alias': 'dev', '--log': False, '--verbose': 0}
         client = configure_client('test', args, config=config)
         eq_(client.url, url)
Exemple #48
0
 def test_download_overwrite(self):
   self.client.upload('foo', self.dpath)
   with temppath() as tpath:
     with open(tpath, 'w'):
       pass
     main(
       ['download', 'foo', tpath, '--silent', '--threads', '1'],
       self.client
     )
     self._dircmp(tpath)
Exemple #49
0
 def test_with_alias(self):
   url = 'http://host:port'
   with temppath() as tpath:
     config = Config(path=tpath)
     section = 'dev.alias'
     config.add_section(section)
     config.set(section, 'url', url)
     args = {'--alias': 'dev', '--log': False, '--verbose': 0}
     client = configure_client('test', args, config=config)
     eq_(client.url, url)
Exemple #50
0
 def test_download_folder_to_existing_folder_parallel(self):
   self.client.write('foo/dl', 'hello')
   self.client.write('foo/bar/dl', 'there')
   with temppath() as tpath:
     os.mkdir(tpath)
     self.client.download('foo', tpath, n_threads=0)
     with open(osp.join(tpath, 'foo', 'dl')) as reader:
       eq_(reader.read(), 'hello')
     with open(osp.join(tpath, 'foo', 'bar', 'dl')) as reader:
       eq_(reader.read(), 'there')
Exemple #51
0
 def test_upload_append(self):
   with temppath() as tpath:
     with open(tpath, 'w') as writer:
       writer.write('hey')
     main(['upload', tpath, 'bar', '--silent', '--threads', '1'], self.client)
     main(
       ['upload', tpath, 'bar', '--silent', '--threads', '1', '--append'],
       self.client
     )
   with self.client.read('bar') as reader:
     eq_(reader.read(), b'heyhey')