def create_collection(args):
data = get_collections_data()
try:
validate_collection_name(args.collection, data['collections'])
except InvalidCollectionName as e:
exit_msg(str(e))
if len(data['collections']):
validate_collections(data['collections'])
validate_collection_data_dir(args.collection)
try:
init_collection_settings(data['collections'], args, data)
create_pg_dir(args.collection)
settings_dir = create_settings_dir(args.collection)
ordered_collections = OrderedDict(
sorted(data['collections'].items(), key=lambda t: t[0]))
write_collection_docker_file(args.collection, settings_dir,
data['collections'][args.collection])
write_env_file(settings_dir, data['collections'][args.collection])
write_python_settings_file(args.collection, settings_dir,
data['collections'][args.collection])
write_global_docker_file(ordered_collections, args.dev
or bool(data['dev_instances']))
write_collections_settings(data)
except Exception as e:
print('Error creating collection: %s' % e)
cleanup(args.collection)
raise
write_instructions(args)
def remove_collection(args):
data = get_collections_data()
try:
validate_collection_name(args.collection,
data['collections'],
new=False)
except InvalidCollectionName as e:
exit_msg(str(e))
if args.collection not in data['collections']:
print('Invalid collection %s' % args.collection)
exit(1)
validate_collections(data['collections'], exit_on_errors=False)
if data['dev_instances'] == 1 and data['collections'][
args.collection]['for_dev']:
data['dev_instances'] = 0
if not args.skip_index:
remove_index(args.collection)
del data['collections'][args.collection]
write_global_docker_file(data['collections'], bool(data['dev_instances']))
write_collections_settings(data['collections'])
cleanup(args.collection)
remove_pg_dir(args.collection)
if args.remove_blobs:
remove_blobs(args.collection, args.yes)
print('Restart docker-compose:')
print(' $ docker-compose down --remove-orphans && docker-compose up -d')
def remove_collection(args):
data = get_collections_data()
if args.collection not in data['collections']:
print('Invalid collection %s' % args.collection)
exit(1)
validate_collections(data['collections'], exit_on_errors=False)
if data['dev_instances'] == 1 and data['collections'][
args.collection]['for_dev']:
data['dev_instances'] = 0
if data['stats_clients'] == 1 and \
data['collections'][args.collection]['env'][DOCKER_HOOVER_SNOOP_STATS]:
data['stats_clients'] = 0
if not args.skip_index:
remove_index(args.collection)
del data['collections'][args.collection]
write_global_docker_file(data['collections'], bool(data['dev_instances']),
bool(data['stats_clients']))
cleanup(args.collection)
remove_pg_dir(args.collection)
if args.remove_blobs:
remove_blobs(args.collection, args.yes)
print('Restart docker-compose:')
print(' $ docker-compose down --remove-orphans && docker-compose up -d')
def rename_collection(args):
data = get_collections_data()
try:
validate_collection_name(args.collection,
data['collections'],
new=False)
validate_collection_name(args.new_name, data['collections'], new=True)
except InvalidCollectionName as e:
exit_msg(str(e))
validate_collections(data['collections'])
index_file_name, index_file_path = export_index(args.collection)
renamed_index_file_path = index_file_path.parent.parent / args.new_name / index_file_name
docker_remove_index(args.collection)
ensure_docker_setup_stopped()
paths_to_rename = [
(collections_path / args.collection, collections_path / args.new_name),
(volumes_path / f'snoop-pg--{args.collection}',
volumes_path / f'snoop-pg--{args.new_name}'),
(blobs_path / args.collection, blobs_path / args.new_name),
(index_file_path, renamed_index_file_path)
]
rename_multiple(paths_to_rename)
data['collections'][args.new_name] = data['collections'][args.collection]
del data['collections'][args.collection]
settings_dir = create_settings_dir(args.new_name)
write_collections_settings(data['collections'])
write_env_file(settings_dir, data['collections'][args.new_name])
write_python_settings_file(args.new_name, settings_dir,
data['collections'][args.new_name])
shutil.rmtree(join(dirname(settings_dir), args.collection))
write_collection_docker_file(args.new_name, settings_dir,
data['collections'][args.new_name])
write_global_docker_file(data['collections'], bool(data['dev_instances']))
import_index(args.collection.lower(), args.new_name, index_file_name)
docker_rename_collection(args)
def update_settings(args):
data = get_collections_data()
collections = data['collections']
if len(collections):
validate_collections(collections)
collections_names = set(collections.keys())
indexing, disable_indexing = read_collections_arg(args.autoindex,
args.manual_indexing,
collections_names)
update_collections_settings(data, {'autoindex': not disable_indexing},
indexing)
profiling, remove_profiling = read_collections_arg(args.profiling,
args.no_profiling,
collections_names)
update_collections_settings(data, {'profiling': not remove_profiling},
profiling)
tracing, disable_tracing = read_collections_arg(args.tracing,
args.no_tracing,
collections_names)
update_collections_settings(data, {'tracing': not disable_tracing},
tracing)
for_dev, remove_dev = read_collections_arg(args.dev, args.remove_dev,
collections_names)
update_collections_settings(data, {'for_dev': not remove_dev}, for_dev)
if args.snoop_image:
for settings in data['collections'].values():
settings['image'] = args.snoop_image
write_env_files(collections)
write_python_settings_files(collections)
dev_instances = write_collections_docker_files(collections)
write_global_docker_file(collections, bool(dev_instances))
write_collections_settings(collections)
print('Restart docker-compose:')
print(' $ docker-compose down && docker-compose up -d')
def update_settings(args):
collections = get_collections_data()['collections']
if len(collections):
validate_collections(collections)
collections_names = set(collections.keys())
profiling, remove_profiling = read_collections_arg(args.profiling, args.no_profiling,
collections_names)
for_dev, remove_dev = read_collections_arg(args.dev, args.remove_dev, collections_names)
indexing, disable_indexing = read_collections_arg(args.autoindex, args.manual_indexing,
collections_names)
stats, disable_stats = read_collections_arg(args.enable_stats, args.disable_stats, collections_names)
stats_clients = write_env_files(collections, stats, disable_stats)
write_python_settings_files(collections, profiling, remove_profiling, for_dev, remove_dev)
dev_instances = write_collections_docker_files(collections, args.snoop_image, profiling,
remove_profiling, for_dev, remove_dev,
indexing, disable_indexing, stats, disable_stats)
write_global_docker_file(collections, bool(dev_instances), bool(stats_clients))
print('Restart docker-compose:')
print(' $ docker-compose down && docker-compose up -d')
def create_collection(args):
data = get_collections_data(args.collection)
if len(data['collections']):
validate_collections(data['collections'])
validate_collection_data_dir(args.collection)
try:
create_pg_dir(args.collection)
settings_dir = create_settings_dir(args.collection)
data['collections'][args.collection] = {
'profiling': args.profiling,
'for_dev': args.dev
}
ordered_collections = OrderedDict(
sorted(data['collections'].items(), key=lambda t: t[0]))
stats = args.stats or data['collections'].get('env', {}).get(
DOCKER_HOOVER_SNOOP_STATS, False)
write_collection_docker_file(args.collection, args.snoop_image,
settings_dir, data['snoop_port'],
args.profiling, args.dev, data['pg_port'],
not args.manual_indexing, stats,
data['flower_port'])
write_env_file(settings_dir, {'DOCKER_HOOVER_SNOOP_STATS': args.stats})
write_python_settings_file(args.collection, settings_dir,
args.profiling, args.dev)
write_global_docker_file(ordered_collections, args.dev
or bool(data['dev_instances']), stats)
except Exception as e:
print('Error creating collection: %s' % e)
cleanup(args.collection)
raise
write_instructions(args)
def test_write_global_docker_file(monkeypatch, data_dir_path,
templates_dir_path, tmpdir):
monkeypatch.setattr(c, 'docker_file_name', 'docker-compose.override.yml')
monkeypatch.setattr(c, 'settings_dir_name', str(tmpdir / 'settings'))
monkeypatch.setattr(c, 'root_dir', tmpdir)
collection = 'FL1'
collections = OrderedDict(((collection, {
'profiling': False,
'for_dev': False,
'autoindex': True,
'image': 'snoop2'
}), ))
os.makedirs(os.path.join(c.settings_dir_name, collection))
write_collection_docker_file(collection, 'snoop2',
os.path.join(c.settings_dir_name, collection),
45025)
write_global_docker_file(collections)
with open(str(tmpdir / c.docker_file_name)) as docker_file, \
open(str(data_dir_path / 'docker-compose.override-uppercase.yml')) as test_file:
global_settings = yaml.load(docker_file)
test_settings = yaml.load(test_file)
assert global_settings == test_settings
collection = 'fl2'
collections = OrderedDict(((collection, {
'profiling': False,
'for_dev': False,
'autoindex': True,
'image': 'snoop2'
}), ))
os.makedirs(os.path.join(c.settings_dir_name, collection))
write_collection_docker_file(collection, 'snoop2',
os.path.join(c.settings_dir_name, collection),
45025)
write_global_docker_file(collections)
with open(str(tmpdir / c.docker_file_name)) as docker_file, \
open(str(data_dir_path / 'docker-compose.override-lowercase.yml')) as test_file:
global_settings = yaml.load(docker_file)
test_settings = yaml.load(test_file)
assert global_settings == test_settings
collection = 'FL3'
collections = OrderedDict(((collection, {
'profiling': False,
'for_dev': True,
'autoindex': True,
'image': 'snoop2'
}), ))
os.makedirs(os.path.join(c.settings_dir_name, collection))
write_collection_docker_file(collection,
'snoop2',
os.path.join(c.settings_dir_name, collection),
45025,
for_dev=True)
write_global_docker_file(collections, for_dev=True)
with open(str(tmpdir / c.docker_file_name)) as docker_file, \
open(str(data_dir_path / 'docker-compose.override-dev.yml')) as test_file:
global_settings = yaml.load(docker_file)
test_settings = yaml.load(test_file)
assert global_settings == test_settings
write_collection_docker_file(collection,
'snoop2',
os.path.join(c.settings_dir_name, collection),
45025,
stats=True)
write_global_docker_file(collections, stats=True)
with open(str(tmpdir / c.docker_file_name)) as docker_file, \
open(str(data_dir_path / 'docker-compose.override-stats.yml')) as test_file:
global_settings = yaml.load(docker_file)
test_settings = yaml.load(test_file)
assert global_settings == test_settings