def setUpClass(cls): cls.cache_dir1 = tempfile.mkdtemp() cls.cache_dir2 = tempfile.mkdtemp() cls.server_dir = tempfile.mkdtemp() cls.fallback_server_dir = tempfile.mkdtemp() cls.temp_dir = tempfile.mkdtemp() cls.fallback_server_process = Process(target=_start_fallback_server, args=(cls.fallback_server_dir, )) cls.fallback_server_process.start() fallback_url = 'http://127.0.0.1:{}'.format(_TEST_FALLBACK_PORT_NUMBER) cls.server_process = Process(target=_start_migration_server, args=(cls.server_dir, fallback_url)) cls.server_process.start() time.sleep(2) # give servers some time to start cls.client = Client( cache_dir=cls.cache_dir1, remote_url='http://127.0.0.1:{}'.format(_TEST_PRIMARY_PORT_NUMBER), ) cls.fallback_client = Client(cache_dir=cls.cache_dir2, remote_url=fallback_url)
def setUp(self): self.temp_dir = tempfile.mkdtemp() os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar')) os.makedirs(os.path.join(self.temp_dir, 'new_root')) self.server_process = Process( target=_start_server, args=(os.path.join(self.temp_dir, 'new_root'),) ) self.server_process.start() time.sleep(2) self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER) self.client = Client(local_store=None, remote_url=self.server_url)
def main(): usage = "usage: %prog [options] command [command-specific options]" commands = [s for s in globals() if s.startswith('cmd_')] commands = sorted([s[4:] for s in commands]) epilog = """ Options specified above are filled from environment (FILETRACKER_DIR, FILETRACKER_URL, FILETRACKER_PUBLIC_URL) if not specified on the command line. Each command has its own --help text. Supported commands: %s.""" % ', '.join(commands) parser = OptionParser(usage=usage, epilog=epilog) parser.disable_interspersed_args() parser.add_option( '-r', '--remote-url', dest='remote_url', default=None, help="URL of remote (central) Filetracker server", ) parser.add_option( '-c', '--cache-dir', dest='cache_dir', default=None, help="Path to the local cache directory", ) parser.add_option('-v', '--verbose', dest='verbose', default=0, action='count', help="Be verbose") options, args = parser.parse_args() if not args: parser.error("Missing command. Try --help for list of available " "commands.") cmd = globals().get('cmd_' + args[0], lambda *a: parser.error("Unknown command: " + args[0])) level = logging.WARNING if options.verbose: level = logging.DEBUG logging.basicConfig( format="%(asctime)-15s %(name)s %(levelname)s: %(message)s", level=level) client = Client(remote_url=options.remote_url, cache_dir=options.cache_dir) cmd(client, *args[1:])
def setUpClass(cls): cls.cache_dir = tempfile.mkdtemp() cls.server_dir = tempfile.mkdtemp() cls.temp_dir = tempfile.mkdtemp() cls.server_process = Process(target=_start_server, args=(cls.server_dir, )) cls.server_process.start() time.sleep(2) # give server some time to start cls.client = Client( cache_dir=cls.cache_dir, remote_url='http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER))
def _scan_disk(self): logger.info("Scanning disk...") dirs = [] for glob_cache_dir in self.glob_cache_dirs: dirs.extend(glob.glob(glob_cache_dir)) logger.debug("Glob expressions have expanded to %s", dirs) self.clients = [ Client(cache_dir=dir, remote_store=None) for dir in dirs ] self.file_index = [] for client in self.clients: for file_info in client.list_local_files(): self.file_index.append( CacheCleaner.FileIndexEntry(file_info=file_info, client=client))
def setUpClass(cls): cls.server_dir = tempfile.mkdtemp() cls.temp_dir = tempfile.mkdtemp() cls.server_process = Process(target=_start_server, args=(cls.server_dir, )) cls.server_process.start() time.sleep(1) # give server some time to start cls.clients = [] for _ in range(_PARALLEL_CLIENTS): client = Client( local_store=None, remote_url='http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER)) # A hack to avoid negotiating version (making a request), because # it slows down client spawn time significantly. client.remote_store._protocol_ver = 2 cls.clients.append(client)
class MigrateScriptTest(unittest.TestCase): def setUp(self): self.temp_dir = tempfile.mkdtemp() os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar')) os.makedirs(os.path.join(self.temp_dir, 'new_root')) self.server_process = Process( target=_start_server, args=(os.path.join(self.temp_dir, 'new_root'),) ) self.server_process.start() time.sleep(2) self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER) self.client = Client(local_store=None, remote_url=self.server_url) def tearDown(self): self.server_process.terminate() shutil.rmtree(self.temp_dir) def test_should_upload_files_with_correct_relative_root(self): _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'a.txt')) _touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar', 'b.txt')) _touch(os.path.join(self.temp_dir, 'old_root', 'c.txt')) _touch(os.path.join(self.temp_dir, 'old_root', 'd.txt')) migrate.main( [ os.path.join(self.temp_dir, 'old_root', 'foo'), self.server_url, '--root', os.path.join(self.temp_dir, 'old_root'), '-s', ] ) self.assertEqual(self.client.get_stream('/foo/a.txt')[0].read(), b'') self.assertEqual(self.client.get_stream('/foo/bar/b.txt')[0].read(), b'') with self.assertRaises(FiletrackerError): self.client.get_stream('/c.txt') with self.assertRaises(FiletrackerError): self.client.get_stream('/d.txt')
def main(args=None): parser = argparse.ArgumentParser(description=_DESCRIPTION) parser.add_argument('files', help='file tree to be uploaded') parser.add_argument('url', help='URL of the filetracker server') parser.add_argument('--root', help='the directory that corresponds to the storage root') parser.add_argument('-s', '--silent', action='store_true', help='if set, progress bar is not printed') args = parser.parse_args(args) upload_root = args.files url = args.url storage_root = args.root silent = args.silent if storage_root is None: storage_root = upload_root # Create a client without local cache. client = Client(local_store=None, remote_url=url) # Calculate total size total_size = 0 size_widgets = [ ' [', progress_bar.ShortTimer(), '] ', ' Calculating file size '.ljust(_ACTION_LENGTH), ' ', progress_bar.DataSize(), ' ', progress_bar.BouncingBar(), ] with progress_bar.conditional(show=not silent, widgets=size_widgets) as bar: for cur_dir, _, files in os.walk(upload_root): for file_name in files: total_size += os.path.getsize(os.path.join(cur_dir, file_name)) bar.update(total_size) upload_widgets = [ ' [', progress_bar.ShortTimer(), '] ', ' Uploading files '.ljust(_ACTION_LENGTH), ' ', progress_bar.DataSize(), ' ', progress_bar.Bar(), ' ', progress_bar.Percentage(), ' ', ' (', progress_bar.AdaptiveETA(), ') ', ] processed_size = 0 with progress_bar.conditional(show=not silent, max_value=total_size, widgets=upload_widgets) as bar: for cur_dir, _, files in os.walk(upload_root): for file_name in files: file_path = os.path.join(cur_dir, file_name) remote_path = '/' + os.path.relpath(file_path, storage_root) file_stat = os.stat(file_path) file_size = file_stat.st_size file_version = int(file_stat.st_mtime) remote_name = '{}@{}'.format(remote_path, file_version) try: client.put_file(remote_name, file_path, to_local_store=False) except FiletrackerError as e: print('ERROR when uploading {}:\n{}'.format(file_path, e), file=sys.stderr) processed_size += file_size bar.update(processed_size)
def __init__(self): Client.__init__(self, local_store=DummyDataStore(), remote_store=None)
def main(args=None): parser = argparse.ArgumentParser(description=_DESCRIPTION) parser.add_argument('files', help='file tree to be uploaded') parser.add_argument('url', help='URL of the filetracker server') parser.add_argument( '--root', help='the directory that corresponds to the storage root') parser.add_argument('-s', '--silent', action='store_true', help='if set, progress bar is not printed') args = parser.parse_args(args) upload_root = args.files url = args.url storage_root = args.root silent = args.silent if storage_root is None: storage_root = upload_root # Create a client without local cache. client = Client(local_store=None, remote_url=url) # Calculate total size total_size = 0 size_widgets = [ ' [', progress_bar.ShortTimer(), '] ', ' Calculating file size '.ljust(_ACTION_LENGTH), ' ', progress_bar.DataSize(), ' ', progress_bar.BouncingBar(), ] with progress_bar.conditional(show=not silent, widgets=size_widgets) as bar: for cur_dir, _, files in os.walk(upload_root): for file_name in files: total_size += os.path.getsize(os.path.join(cur_dir, file_name)) bar.update(total_size) upload_widgets = [ ' [', progress_bar.ShortTimer(), '] ', ' Uploading files '.ljust(_ACTION_LENGTH), ' ', progress_bar.DataSize(), ' ', progress_bar.Bar(), ' ', progress_bar.Percentage(), ' ', ' (', progress_bar.AdaptiveETA(), ') ', ] processed_size = 0 with progress_bar.conditional(show=not silent, max_value=total_size, widgets=upload_widgets) as bar: for cur_dir, _, files in os.walk(upload_root): for file_name in files: file_path = os.path.join(cur_dir, file_name) remote_path = '/' + os.path.relpath(file_path, storage_root) file_stat = os.stat(file_path) file_size = file_stat.st_size file_version = int(file_stat.st_mtime) remote_name = '{}@{}'.format(remote_path, file_version) try: client.put_file(remote_name, file_path, to_local_store=False) except FiletrackerError as e: print('ERROR when uploading {}:\n{}'.format(file_path, e), file=sys.stderr) processed_size += file_size bar.update(processed_size)
def __init__(self): Client.__init__(self, local_store=DummyDataStore(), remote_store=None)