def setUpClass(cls):
cls.cache_dir1 = tempfile.mkdtemp()
cls.cache_dir2 = tempfile.mkdtemp()
cls.server_dir = tempfile.mkdtemp()
cls.fallback_server_dir = tempfile.mkdtemp()
cls.temp_dir = tempfile.mkdtemp()
cls.fallback_server_process = Process(target=_start_fallback_server,
args=(cls.fallback_server_dir, ))
cls.fallback_server_process.start()
fallback_url = 'http://127.0.0.1:{}'.format(_TEST_FALLBACK_PORT_NUMBER)
cls.server_process = Process(target=_start_migration_server,
args=(cls.server_dir, fallback_url))
cls.server_process.start()
time.sleep(2) # give servers some time to start
cls.client = Client(
cache_dir=cls.cache_dir1,
remote_url='http://127.0.0.1:{}'.format(_TEST_PRIMARY_PORT_NUMBER),
)
cls.fallback_client = Client(cache_dir=cls.cache_dir2,
remote_url=fallback_url)
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar'))
os.makedirs(os.path.join(self.temp_dir, 'new_root'))
self.server_process = Process(
target=_start_server, args=(os.path.join(self.temp_dir, 'new_root'),)
)
self.server_process.start()
time.sleep(2)
self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER)
self.client = Client(local_store=None, remote_url=self.server_url)
def main():
usage = "usage: %prog [options] command [command-specific options]"
commands = [s for s in globals() if s.startswith('cmd_')]
commands = sorted([s[4:] for s in commands])
epilog = """
Options specified above are filled from environment
(FILETRACKER_DIR, FILETRACKER_URL, FILETRACKER_PUBLIC_URL)
if not specified on the command line.
Each command has its own --help text.
Supported commands: %s.""" % ', '.join(commands)
parser = OptionParser(usage=usage, epilog=epilog)
parser.disable_interspersed_args()
parser.add_option(
'-r',
'--remote-url',
dest='remote_url',
default=None,
help="URL of remote (central) Filetracker server",
)
parser.add_option(
'-c',
'--cache-dir',
dest='cache_dir',
default=None,
help="Path to the local cache directory",
)
parser.add_option('-v',
'--verbose',
dest='verbose',
default=0,
action='count',
help="Be verbose")
options, args = parser.parse_args()
if not args:
parser.error("Missing command. Try --help for list of available "
"commands.")
cmd = globals().get('cmd_' + args[0],
lambda *a: parser.error("Unknown command: " + args[0]))
level = logging.WARNING
if options.verbose:
level = logging.DEBUG
logging.basicConfig(
format="%(asctime)-15s %(name)s %(levelname)s: %(message)s",
level=level)
client = Client(remote_url=options.remote_url, cache_dir=options.cache_dir)
cmd(client, *args[1:])
def setUpClass(cls):
cls.cache_dir = tempfile.mkdtemp()
cls.server_dir = tempfile.mkdtemp()
cls.temp_dir = tempfile.mkdtemp()
cls.server_process = Process(target=_start_server,
args=(cls.server_dir, ))
cls.server_process.start()
time.sleep(2) # give server some time to start
cls.client = Client(
cache_dir=cls.cache_dir,
remote_url='http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER))
def _scan_disk(self):
logger.info("Scanning disk...")
dirs = []
for glob_cache_dir in self.glob_cache_dirs:
dirs.extend(glob.glob(glob_cache_dir))
logger.debug("Glob expressions have expanded to %s", dirs)
self.clients = [
Client(cache_dir=dir, remote_store=None) for dir in dirs
]
self.file_index = []
for client in self.clients:
for file_info in client.list_local_files():
self.file_index.append(
CacheCleaner.FileIndexEntry(file_info=file_info,
client=client))
def setUpClass(cls):
cls.server_dir = tempfile.mkdtemp()
cls.temp_dir = tempfile.mkdtemp()
cls.server_process = Process(target=_start_server,
args=(cls.server_dir, ))
cls.server_process.start()
time.sleep(1) # give server some time to start
cls.clients = []
for _ in range(_PARALLEL_CLIENTS):
client = Client(
local_store=None,
remote_url='http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER))
# A hack to avoid negotiating version (making a request), because
# it slows down client spawn time significantly.
client.remote_store._protocol_ver = 2
cls.clients.append(client)
class MigrateScriptTest(unittest.TestCase):
def setUp(self):
self.temp_dir = tempfile.mkdtemp()
os.makedirs(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar'))
os.makedirs(os.path.join(self.temp_dir, 'new_root'))
self.server_process = Process(
target=_start_server, args=(os.path.join(self.temp_dir, 'new_root'),)
)
self.server_process.start()
time.sleep(2)
self.server_url = 'http://127.0.0.1:{}'.format(_TEST_PORT_NUMBER)
self.client = Client(local_store=None, remote_url=self.server_url)
def tearDown(self):
self.server_process.terminate()
shutil.rmtree(self.temp_dir)
def test_should_upload_files_with_correct_relative_root(self):
_touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'a.txt'))
_touch(os.path.join(self.temp_dir, 'old_root', 'foo', 'bar', 'b.txt'))
_touch(os.path.join(self.temp_dir, 'old_root', 'c.txt'))
_touch(os.path.join(self.temp_dir, 'old_root', 'd.txt'))
migrate.main(
[
os.path.join(self.temp_dir, 'old_root', 'foo'),
self.server_url,
'--root',
os.path.join(self.temp_dir, 'old_root'),
'-s',
]
)
self.assertEqual(self.client.get_stream('/foo/a.txt')[0].read(), b'')
self.assertEqual(self.client.get_stream('/foo/bar/b.txt')[0].read(), b'')
with self.assertRaises(FiletrackerError):
self.client.get_stream('/c.txt')
with self.assertRaises(FiletrackerError):
self.client.get_stream('/d.txt')
def main(args=None):
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument('files', help='file tree to be uploaded')
parser.add_argument('url', help='URL of the filetracker server')
parser.add_argument('--root',
help='the directory that corresponds to the storage root')
parser.add_argument('-s', '--silent', action='store_true',
help='if set, progress bar is not printed')
args = parser.parse_args(args)
upload_root = args.files
url = args.url
storage_root = args.root
silent = args.silent
if storage_root is None:
storage_root = upload_root
# Create a client without local cache.
client = Client(local_store=None, remote_url=url)
# Calculate total size
total_size = 0
size_widgets = [
' [', progress_bar.ShortTimer(), '] ',
' Calculating file size '.ljust(_ACTION_LENGTH),
' ', progress_bar.DataSize(), ' ',
progress_bar.BouncingBar(),
]
with progress_bar.conditional(show=not silent,
widgets=size_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
total_size += os.path.getsize(os.path.join(cur_dir, file_name))
bar.update(total_size)
upload_widgets = [
' [', progress_bar.ShortTimer(), '] ',
' Uploading files '.ljust(_ACTION_LENGTH),
' ', progress_bar.DataSize(), ' ',
progress_bar.Bar(),
' ', progress_bar.Percentage(), ' ',
' (', progress_bar.AdaptiveETA(), ') ',
]
processed_size = 0
with progress_bar.conditional(show=not silent,
max_value=total_size,
widgets=upload_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
file_path = os.path.join(cur_dir, file_name)
remote_path = '/' + os.path.relpath(file_path, storage_root)
file_stat = os.stat(file_path)
file_size = file_stat.st_size
file_version = int(file_stat.st_mtime)
remote_name = '{}@{}'.format(remote_path, file_version)
try:
client.put_file(remote_name, file_path, to_local_store=False)
except FiletrackerError as e:
print('ERROR when uploading {}:\n{}'.format(file_path, e),
file=sys.stderr)
processed_size += file_size
bar.update(processed_size)
def __init__(self):
Client.__init__(self, local_store=DummyDataStore(), remote_store=None)
def main(args=None):
parser = argparse.ArgumentParser(description=_DESCRIPTION)
parser.add_argument('files', help='file tree to be uploaded')
parser.add_argument('url', help='URL of the filetracker server')
parser.add_argument(
'--root', help='the directory that corresponds to the storage root')
parser.add_argument('-s',
'--silent',
action='store_true',
help='if set, progress bar is not printed')
args = parser.parse_args(args)
upload_root = args.files
url = args.url
storage_root = args.root
silent = args.silent
if storage_root is None:
storage_root = upload_root
# Create a client without local cache.
client = Client(local_store=None, remote_url=url)
# Calculate total size
total_size = 0
size_widgets = [
' [',
progress_bar.ShortTimer(),
'] ',
' Calculating file size '.ljust(_ACTION_LENGTH),
' ',
progress_bar.DataSize(),
' ',
progress_bar.BouncingBar(),
]
with progress_bar.conditional(show=not silent,
widgets=size_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
total_size += os.path.getsize(os.path.join(cur_dir, file_name))
bar.update(total_size)
upload_widgets = [
' [',
progress_bar.ShortTimer(),
'] ',
' Uploading files '.ljust(_ACTION_LENGTH),
' ',
progress_bar.DataSize(),
' ',
progress_bar.Bar(),
' ',
progress_bar.Percentage(),
' ',
' (',
progress_bar.AdaptiveETA(),
') ',
]
processed_size = 0
with progress_bar.conditional(show=not silent,
max_value=total_size,
widgets=upload_widgets) as bar:
for cur_dir, _, files in os.walk(upload_root):
for file_name in files:
file_path = os.path.join(cur_dir, file_name)
remote_path = '/' + os.path.relpath(file_path, storage_root)
file_stat = os.stat(file_path)
file_size = file_stat.st_size
file_version = int(file_stat.st_mtime)
remote_name = '{}@{}'.format(remote_path, file_version)
try:
client.put_file(remote_name,
file_path,
to_local_store=False)
except FiletrackerError as e:
print('ERROR when uploading {}:\n{}'.format(file_path, e),
file=sys.stderr)
processed_size += file_size
bar.update(processed_size)
def __init__(self):
Client.__init__(self, local_store=DummyDataStore(), remote_store=None)