def find_files_to_process():
files_from_crawler = list(flattened(recursive_listdir(DOWNLOAD_DIR)))
files_to_process = []
files_to_ignore = []
for path in files_from_crawler:
try:
import_date = find_date(path)
size = os.path.getsize(path)
files_to_process.append((path, import_date, os.path.getsize(path)))
except ValueError:
files_to_ignore.append(path)
def _import_date((_1, import_date, _2)):
return import_date
def _size((_1, _2, size)):
return size
bytes_accumulator = Accumulator()
files_to_process.sort(key=_import_date)
files_to_process = [(f, bytes_accumulator(_size(f)))
for f in files_to_process]
bytes_to_process = bytes_accumulator.getvalue()
return (bytes_to_process, files_to_process, files_to_ignore)
def __init__(self, name, basepath, template='cartridge'):
self.name = name
self.basepath = basepath
self.template = template
# create project and save its path
self.path = create_project(basepath, name, template)
# save tarantool_enterprise_is_used() result to variable
tarantool_is_enterprise = tarantool_enterprise_is_used()
# files that should be delivered in the result package
project_files = recursive_listdir(self.path)
self.distribution_files = filter_out_files_removed_on_pack(project_files)
self.distribution_files.add('VERSION')
if tarantool_is_enterprise:
self.distribution_files.update({'tarantool', 'tarantoolctl'})
# project rockspec name and path
self.rockspec_name = '{}-scm-1.rockspec'.format(self.name)
self.rockspec_path = os.path.join(self.path, self.rockspec_name)
# rocks that should be delivered in the result package
self.rocks_content = get_base_project_rocks(self.name, self.rockspec_name)
# keys that should be mentioned in the package VERSION file
self.version_file_keys = {
'TARANTOOL',
self.name,
# default application dependencies
'cartridge',
'luatest',
}
if tarantool_is_enterprise:
self.version_file_keys.add('TARANTOOL_SDK')
def test_build(cartridge_cmd, light_project, tmpdir):
project = light_project
project_files_before = recursive_listdir(project.path)
cmd = [cartridge_cmd, "build", project.path]
process = subprocess.run(cmd, cwd=tmpdir)
assert process.returncode == 0, "Error during building the project"
# check that all expected rocks was installed
files = recursive_listdir(project.path)
assert '.rocks' in files
assert all([rock in files for rock in project.rocks_content])
project_files_after = recursive_listdir(project.path)
# check that nothing was deleted
assert all([f in project_files_after for f in project_files_before])
def test_from(cartridge_cmd, app_template, tmpdir):
APPNAME = "myapp"
cmd = [
cartridge_cmd,
"create",
"--from",
app_template['path'],
"--name",
APPNAME,
tmpdir,
]
rc, output = run_command_and_get_output(cmd)
assert rc == 0
app_path = os.path.join(tmpdir, APPNAME)
assert not os.path.exists(os.path.join(app_path, '.rocks'))
files = recursive_listdir(app_path)
files = {f for f in files if not f.startswith('.git')}
def subst(s):
return s.replace('{{ .Name }}',
APPNAME).replace('{{ .StateboardName }}',
'%s-stateboard' % APPNAME)
exp_files = {}
for filepath, fileinfo in app_template['files'].items():
exp_files.update({
subst(filepath): {
'mode':
fileinfo['mode'],
'content':
subst(fileinfo['content'])
if fileinfo.get('content') is not None else None,
}
})
assert set(exp_files.keys()) == files
for filepath, fileinfo in exp_files.items():
created_file_path = os.path.join(app_path, filepath)
created_file_mode = os.stat(created_file_path)[stat.ST_MODE] & 0o777
assert created_file_mode == fileinfo['mode']
if fileinfo.get('content') is None:
continue
with open(created_file_path, 'r') as f:
created_file_content = f.read()
assert created_file_content == fileinfo['content']
def test_pack(docker_image, tmpdir, docker_client):
project = docker_image.project
image_name = docker_image.name
container = docker_client.containers.create(docker_image.name)
container_distribution_dir = '/usr/share/tarantool/{}'.format(project.name)
# check if distribution dir was created
command = '[ -d "{}" ] && echo true || echo false'.format(container_distribution_dir)
output = run_command_on_image(docker_client, image_name, command)
assert output == 'true'
# get distribution dir contents
arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
with open(arhive_path, 'wb') as f:
bits, _ = container.get_archive(container_distribution_dir)
for chunk in bits:
f.write(chunk)
with tarfile.open(arhive_path) as arch:
arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
os.remove(arhive_path)
assert_distribution_dir_contents(
dir_contents=recursive_listdir(os.path.join(tmpdir, 'usr/share/tarantool/', project.name)),
project=project,
)
assert_filemodes(project, tmpdir)
container.remove()
if not tarantool_enterprise_is_used():
# check if tarantool was installed
command = 'yum list installed 2>/dev/null | grep tarantool'
output = run_command_on_image(docker_client, image_name, command)
packages_list = output.split('\n')
assert any(['tarantool' in package for package in packages_list])
# check tarantool version
command = 'tarantool --version'
output = run_command_on_image(docker_client, image_name, command)
m = re.search(r'Tarantool\s+(\d+.\d+)', output)
assert m is not None
installed_version = m.group(1)
m = re.search(r'(\d+.\d+)', tarantool_version())
assert m is not None
expected_version = m.group(1)
assert installed_version == expected_version
def __init__(self,
cartridge_cmd,
name,
basepath,
template='cartridge',
create_func=None):
self.name = name
self.basepath = basepath
self.template = template
self.deprecated_flow_is_used = False
self.vshard_groups_names = None
self.custom_roles = None
if create_func is None:
# create project and save its path
self.path = create_project(cartridge_cmd, basepath, name, template)
else:
self.path = create_func(basepath)
# save tarantool_enterprise_is_used() result to variable
tarantool_is_enterprise = tarantool_enterprise_is_used()
# files that should be delivered in the result package
project_files = recursive_listdir(self.path)
self.distribution_files = filter_out_files_removed_on_pack(
project_files)
self.distribution_files.add('VERSION')
self.distribution_files.add('VERSION.lua')
if tarantool_is_enterprise:
self.distribution_files.update({'tarantool', 'tarantoolctl'})
# project rockspec name and path
self.rockspec_name = '{}-scm-1.rockspec'.format(self.name)
self.rockspec_path = os.path.join(self.path, self.rockspec_name)
# rocks that should be delivered in the result package
self.rocks_content = get_base_project_rocks(self.name,
self.rockspec_name)
# keys that should be mentioned in the package VERSION file
self.version_file_keys = {
'TARANTOOL',
self.name,
# default application dependencies
'cartridge',
}
if tarantool_is_enterprise:
self.version_file_keys.add('TARANTOOL_SDK')
self.image_runtime_requirements_filepath = None
def test_tgz_pack(project, tgz_archive, tmpdir):
with tarfile.open(name=tgz_archive['name']) as tgz_arch:
# usr/share/tarantool is added to coorectly run assert_filemodes
distribution_dir = os.path.join(tmpdir, 'usr/share/tarantool',
project['name'])
os.makedirs(distribution_dir, exist_ok=True)
tgz_arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
assert_dir_contents(files_list=recursive_listdir(distribution_dir),
exp_files_list=project['distribution_files_list'],
exp_rocks_content=project['rocks_content'])
validate_version_file(project, distribution_dir)
assert_filemodes(project, tmpdir)
def test_project_without_stateboard(cartridge_cmd,
project_without_dependencies, pack_format,
tmpdir):
project = project_without_dependencies
STATEBOARD_ENTRYPOINT_NAME = 'stateboard.init.lua'
# remove stateboard entrypoint from project
os.remove(os.path.join(project.path, STATEBOARD_ENTRYPOINT_NAME))
project.distribution_files.remove(STATEBOARD_ENTRYPOINT_NAME)
cmd = [
cartridge_cmd,
"pack",
pack_format,
project.path,
]
# call cartridge pack
process = subprocess.run(cmd, cwd=tmpdir)
assert process.returncode == 0
# packing should succeed with warning
rc, output = run_command_and_get_output(cmd, cwd=tmpdir)
assert rc == 0
assert "App directory doesn't contain stateboard entrypoint script" in output
# extract files from archive
archive_path = find_archive(tmpdir, project.name, pack_format)
extract_dir = os.path.join(tmpdir, 'extract')
os.makedirs(extract_dir)
if pack_format == 'rpm':
extract_rpm(archive_path, extract_dir)
elif pack_format == 'deb':
extract_deb(archive_path, extract_dir)
with tarfile.open(
name=os.path.join(extract_dir, 'data.tar.xz')) as data_arch:
data_arch.extractall(path=extract_dir)
# check that stateboard unit file wasn't delivered
systemd_dir = (os.path.join(extract_dir, 'etc/systemd/system'))
assert os.path.exists(systemd_dir)
systemd_files = recursive_listdir(systemd_dir)
assert len(systemd_files) == 2
assert '{}-stateboard.service'.format(project.name) not in systemd_files
def test_building_without_path_specifying(cartridge_cmd,
project_without_dependencies):
project = project_without_dependencies
# say `cartridge build` in project directory
cmd = [
cartridge_cmd,
"build",
]
process = subprocess.run(cmd, cwd=project.path)
assert process.returncode == 0, 'Building project failed'
# check that all expected rocks was installed
files = recursive_listdir(project.path)
assert '.rocks' in files
assert all([rock in files for rock in project.rocks_content])
def test_project_without_stateboard(cartridge_cmd,
project_without_dependencies, pack_format,
tmpdir):
project = project_without_dependencies
STATEBOARD_ENTRYPOINT_NAME = 'stateboard.init.lua'
# remove stateboard entrypoint from project
os.remove(os.path.join(project.path, STATEBOARD_ENTRYPOINT_NAME))
project.distribution_files.remove(STATEBOARD_ENTRYPOINT_NAME)
cmd = [
cartridge_cmd,
"pack",
pack_format,
project.path,
]
if platform.system() == 'Darwin':
cmd.append('--use-docker')
# call cartridge pack
process = subprocess.run(cmd, cwd=tmpdir)
assert process.returncode == 0
# packing should succeed with warning
rc, output = run_command_and_get_output(cmd, cwd=tmpdir)
assert rc == 0
assert "App directory doesn't contain stateboard entrypoint script" in output
# extract files from archive
archive_path = find_archive(tmpdir, project.name, pack_format)
extract_dir = os.path.join(tmpdir, 'extract')
extract_app_files(archive_path, pack_format, extract_dir)
# check that stateboard unit file wasn't delivered
systemd_dir = (os.path.join(extract_dir, 'etc/systemd/system'))
assert os.path.exists(systemd_dir)
systemd_files = recursive_listdir(systemd_dir)
assert len(systemd_files) == 2
assert '{}-stateboard.service'.format(project.name) not in systemd_files
def test_tgz(tgz_archive, tmpdir):
project = tgz_archive.project
# archive files should be extracted to the empty directory
# to correctly check archive contents
extract_dir = os.path.join(tmpdir, 'extract')
os.makedirs(extract_dir)
with tarfile.open(name=tgz_archive.filepath) as tgz_arch:
# usr/share/tarantool is added to correctly run assert_filemodes
distribution_dir = os.path.join(extract_dir, 'usr/share/tarantool',
project.name)
os.makedirs(distribution_dir, exist_ok=True)
tgz_arch.extractall(
path=os.path.join(extract_dir, 'usr/share/tarantool'))
assert_distribution_dir_contents(
dir_contents=recursive_listdir(distribution_dir), project=project)
validate_version_file(project, distribution_dir)
assert_filemodes(project, extract_dir)
def find_files_to_process():
files_from_crawler = list(flattened(recursive_listdir(DOWNLOAD_DIR)))
files_to_process = []
files_to_ignore = []
for path in files_from_crawler:
try:
import_date = find_date(path)
size = os.path.getsize(path)
files_to_process.append((path,
import_date,
os.path.getsize(path)))
except ValueError:
files_to_ignore.append(path)
def _import_date((_1, import_date, _2)): return import_date
def _size((_1, _2, size)): return size
bytes_accumulator = Accumulator()
files_to_process.sort(key=_import_date)
files_to_process = [(f, bytes_accumulator(_size(f)))
for f in files_to_process]
bytes_to_process = bytes_accumulator.getvalue()
return (bytes_to_process, files_to_process, files_to_ignore)
except (ValueError, ImportError), err:
return False
re_agency = re.compile('^[0-9]*[A-Z]+')
def extract_prefix(filename):
prefix_match = re_agency.match(filename.upper())
if not prefix_match is None:
prefix = prefix_match.group()
return fix_prefix(prefix)
else:
return None
files_to_process = filter(
filename_has_date,
map(os.path.basename, flattened(recursive_listdir(DOWNLOAD_DIR))))
prefixes = map(extract_prefix, files_to_process)
def unique(iterable):
def combine(accum, item):
accum[item] = None
return accum
return reduce(combine, iterable, {}).keys()
def frequency(iterable):
def combine(frequencies, item):
cnt = frequencies.get(item, 0)
frequencies[item] = cnt + 1
return frequencies
try:
import_date = find_date(filename)
return True
except (ValueError, ImportError), err:
return False
re_agency = re.compile('^[0-9]*[A-Z]+')
def extract_prefix(filename):
prefix_match = re_agency.match(filename.upper())
if not prefix_match is None:
prefix = prefix_match.group()
return fix_prefix(prefix)
else:
return None
files_to_process = filter(filename_has_date, map(os.path.basename, flattened(recursive_listdir(DOWNLOAD_DIR))))
prefixes = map(extract_prefix, files_to_process)
def unique(iterable):
def combine(accum, item):
accum[item] = None
return accum
return reduce(combine, iterable, {}).keys()
def frequency(iterable):
def combine(frequencies, item):
cnt = frequencies.get(item, 0)
frequencies[item] = cnt + 1
return frequencies
return reduce(combine, iterable, {})
def test_pack(docker_image, tmpdir, docker_client):
project = docker_image.project
image_name = docker_image.name
container = docker_client.containers.create(docker_image.name)
container_distribution_dir = '/usr/share/tarantool/{}'.format(project.name)
# check if distribution dir was created
command = '[ -d "{}" ] && echo true || echo false'.format(
container_distribution_dir)
output = run_command_on_image(docker_client, image_name, command)
assert output == 'true'
# get distribution dir contents
arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
with open(arhive_path, 'wb') as f:
bits, _ = container.get_archive(container_distribution_dir)
for chunk in bits:
f.write(chunk)
with tarfile.open(arhive_path) as arch:
arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
os.remove(arhive_path)
distribution_dir_contents = recursive_listdir(
os.path.join(tmpdir, 'usr/share/tarantool/', project.name))
# The runtime image is built using Dockerfile.<random-string> in the
# distribution directory
# This dockerfile name should be added to project distribution files set
# to correctly check distribution directory contents
for f in distribution_dir_contents:
if f.startswith('Dockerfile') and f not in [
'Dockerfile.build.cartridge', 'Dockerfile.cartridge'
]:
project.distribution_files.add(f)
break
assert_distribution_dir_contents(
dir_contents=recursive_listdir(
os.path.join(tmpdir, 'usr/share/tarantool/', project.name)),
project=project,
)
assert_filemodes(project, tmpdir)
container.remove()
if project.image_runtime_requirements_filepath is not None:
command = 'ls {}'.format(project.image_runtime_requirements_filepath)
output = run_command_on_image(docker_client, image_name, command)
assert output == project.image_runtime_requirements_filepath
if not tarantool_enterprise_is_used():
# check if tarantool was installed
command = 'yum list installed 2>/dev/null | grep tarantool'
output = run_command_on_image(docker_client, image_name, command)
packages_list = output.split('\n')
assert any(['tarantool' in package for package in packages_list])
# check tarantool version
command = 'tarantool --version'
output = run_command_on_image(docker_client, image_name, command)
m = re.search(r'Tarantool\s+(\d+.\d+)', output)
assert m is not None
installed_version = m.group(1)
m = re.search(r'(\d+.\d+)', tarantool_version())
assert m is not None
expected_version = m.group(1)
assert installed_version == expected_version
def test_docker_pack(project, docker_image, tmpdir, docker_client):
image_name = docker_image['name']
container = docker_client.containers.create(image_name)
container_distribution_dir = '/usr/share/tarantool/{}'.format(project['name'])
# check if distribution dir was created
command = '[ -d "{}" ] && echo true || echo false'.format(container_distribution_dir)
output = run_command_on_image(docker_client, image_name, command)
assert output == 'true'
# get distribution dir contents
arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
with open(arhive_path, 'wb') as f:
bits, _ = container.get_archive(container_distribution_dir)
for chunk in bits:
f.write(chunk)
with tarfile.open(arhive_path) as arch:
arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
os.remove(arhive_path)
assert_dir_contents(
files_list=recursive_listdir(os.path.join(tmpdir, 'usr/share/tarantool/', project['name'])),
exp_files_list=project['distribution_files_list'],
exp_rocks_content=project['rocks_content'],
skip_tarantool_binaries=True
)
assert_filemodes(project, tmpdir)
container.remove()
if tarantool_enterprise_is_used():
# check tarantool and tarantoolctl binaries
command = '[ -d "/usr/share/tarantool/tarantool-enterprise/" ] && echo true || echo false'
output = run_command_on_image(docker_client, image_name, command)
assert output == 'true'
command = 'cd /usr/share/tarantool/tarantool-enterprise/ && find .'
output = run_command_on_image(docker_client, image_name, command)
files_list = output.split('\n')
files_list.remove('.')
dir_contents = [
os.path.normpath(filename)
for filename in files_list
]
assert 'tarantool' in dir_contents
assert 'tarantoolctl' in dir_contents
else:
# check if tarantool was installed
command = 'yum list installed 2>/dev/null | grep tarantool'
output = run_command_on_image(docker_client, image_name, command)
packages_list = output.split('\n')
assert any(['tarantool' in package for package in packages_list])
# check tarantool version
command = 'yum info tarantool'
output = run_command_on_image(docker_client, image_name, command)
m = re.search(r'Version\s+:\s+(\d+)\.(\d+).', output)
assert m is not None
installed_version = m.groups()
m = re.search(r'(\d+)\.(\d+)\.\d+', tarantool_version())
assert m is not None
expected_version = m.groups()
assert installed_version == expected_version