Example #1
0
def find_files_to_process():
    files_from_crawler = list(flattened(recursive_listdir(DOWNLOAD_DIR)))

    files_to_process = []
    files_to_ignore = []
    for path in files_from_crawler:
        try:
            import_date = find_date(path)
            size = os.path.getsize(path)
            files_to_process.append((path, import_date, os.path.getsize(path)))
        except ValueError:
            files_to_ignore.append(path)

    def _import_date((_1, import_date, _2)):
        return import_date

    def _size((_1, _2, size)):
        return size

    bytes_accumulator = Accumulator()
    files_to_process.sort(key=_import_date)
    files_to_process = [(f, bytes_accumulator(_size(f)))
                        for f in files_to_process]
    bytes_to_process = bytes_accumulator.getvalue()

    return (bytes_to_process, files_to_process, files_to_ignore)
Example #2
0
    def __init__(self, name, basepath, template='cartridge'):
        self.name = name
        self.basepath = basepath
        self.template = template
        # create project and save its path
        self.path = create_project(basepath, name, template)

        # save tarantool_enterprise_is_used() result to variable
        tarantool_is_enterprise = tarantool_enterprise_is_used()

        # files that should be delivered in the result package
        project_files = recursive_listdir(self.path)
        self.distribution_files = filter_out_files_removed_on_pack(project_files)
        self.distribution_files.add('VERSION')
        if tarantool_is_enterprise:
            self.distribution_files.update({'tarantool', 'tarantoolctl'})

        # project rockspec name and path
        self.rockspec_name = '{}-scm-1.rockspec'.format(self.name)
        self.rockspec_path = os.path.join(self.path, self.rockspec_name)

        # rocks that should be delivered in the result package
        self.rocks_content = get_base_project_rocks(self.name, self.rockspec_name)

        # keys that should be mentioned in the package VERSION file
        self.version_file_keys = {
            'TARANTOOL',
            self.name,
            # default application dependencies
            'cartridge',
            'luatest',
        }
        if tarantool_is_enterprise:
            self.version_file_keys.add('TARANTOOL_SDK')
Example #3
0
def test_build(cartridge_cmd, light_project, tmpdir):
    project = light_project

    project_files_before = recursive_listdir(project.path)

    cmd = [cartridge_cmd, "build", project.path]
    process = subprocess.run(cmd, cwd=tmpdir)
    assert process.returncode == 0, "Error during building the project"

    # check that all expected rocks was installed
    files = recursive_listdir(project.path)
    assert '.rocks' in files
    assert all([rock in files for rock in project.rocks_content])

    project_files_after = recursive_listdir(project.path)

    # check that nothing was deleted
    assert all([f in project_files_after for f in project_files_before])
def test_from(cartridge_cmd, app_template, tmpdir):
    APPNAME = "myapp"

    cmd = [
        cartridge_cmd,
        "create",
        "--from",
        app_template['path'],
        "--name",
        APPNAME,
        tmpdir,
    ]

    rc, output = run_command_and_get_output(cmd)
    assert rc == 0

    app_path = os.path.join(tmpdir, APPNAME)
    assert not os.path.exists(os.path.join(app_path, '.rocks'))

    files = recursive_listdir(app_path)
    files = {f for f in files if not f.startswith('.git')}

    def subst(s):
        return s.replace('{{ .Name }}',
                         APPNAME).replace('{{ .StateboardName }}',
                                          '%s-stateboard' % APPNAME)

    exp_files = {}
    for filepath, fileinfo in app_template['files'].items():
        exp_files.update({
            subst(filepath): {
                'mode':
                fileinfo['mode'],
                'content':
                subst(fileinfo['content'])
                if fileinfo.get('content') is not None else None,
            }
        })

    assert set(exp_files.keys()) == files

    for filepath, fileinfo in exp_files.items():
        created_file_path = os.path.join(app_path, filepath)
        created_file_mode = os.stat(created_file_path)[stat.ST_MODE] & 0o777
        assert created_file_mode == fileinfo['mode']

        if fileinfo.get('content') is None:
            continue

        with open(created_file_path, 'r') as f:
            created_file_content = f.read()

        assert created_file_content == fileinfo['content']
def test_pack(docker_image, tmpdir, docker_client):
    project = docker_image.project
    image_name = docker_image.name

    container = docker_client.containers.create(docker_image.name)
    container_distribution_dir = '/usr/share/tarantool/{}'.format(project.name)

    # check if distribution dir was created
    command = '[ -d "{}" ] && echo true || echo false'.format(container_distribution_dir)
    output = run_command_on_image(docker_client, image_name, command)
    assert output == 'true'

    # get distribution dir contents
    arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
    with open(arhive_path, 'wb') as f:
        bits, _ = container.get_archive(container_distribution_dir)
        for chunk in bits:
            f.write(chunk)

    with tarfile.open(arhive_path) as arch:
        arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
    os.remove(arhive_path)

    assert_distribution_dir_contents(
        dir_contents=recursive_listdir(os.path.join(tmpdir, 'usr/share/tarantool/', project.name)),
        project=project,
    )

    assert_filemodes(project, tmpdir)
    container.remove()

    if not tarantool_enterprise_is_used():
        # check if tarantool was installed
        command = 'yum list installed 2>/dev/null | grep tarantool'
        output = run_command_on_image(docker_client, image_name, command)

        packages_list = output.split('\n')
        assert any(['tarantool' in package for package in packages_list])

        # check tarantool version
        command = 'tarantool --version'
        output = run_command_on_image(docker_client, image_name, command)

        m = re.search(r'Tarantool\s+(\d+.\d+)', output)
        assert m is not None
        installed_version = m.group(1)

        m = re.search(r'(\d+.\d+)', tarantool_version())
        assert m is not None
        expected_version = m.group(1)

        assert installed_version == expected_version
Example #6
0
    def __init__(self,
                 cartridge_cmd,
                 name,
                 basepath,
                 template='cartridge',
                 create_func=None):
        self.name = name
        self.basepath = basepath
        self.template = template
        self.deprecated_flow_is_used = False
        self.vshard_groups_names = None
        self.custom_roles = None

        if create_func is None:
            # create project and save its path
            self.path = create_project(cartridge_cmd, basepath, name, template)
        else:
            self.path = create_func(basepath)

        # save tarantool_enterprise_is_used() result to variable
        tarantool_is_enterprise = tarantool_enterprise_is_used()

        # files that should be delivered in the result package
        project_files = recursive_listdir(self.path)
        self.distribution_files = filter_out_files_removed_on_pack(
            project_files)
        self.distribution_files.add('VERSION')
        self.distribution_files.add('VERSION.lua')
        if tarantool_is_enterprise:
            self.distribution_files.update({'tarantool', 'tarantoolctl'})

        # project rockspec name and path
        self.rockspec_name = '{}-scm-1.rockspec'.format(self.name)
        self.rockspec_path = os.path.join(self.path, self.rockspec_name)

        # rocks that should be delivered in the result package
        self.rocks_content = get_base_project_rocks(self.name,
                                                    self.rockspec_name)

        # keys that should be mentioned in the package VERSION file
        self.version_file_keys = {
            'TARANTOOL',
            self.name,
            # default application dependencies
            'cartridge',
        }
        if tarantool_is_enterprise:
            self.version_file_keys.add('TARANTOOL_SDK')

        self.image_runtime_requirements_filepath = None
Example #7
0
def test_tgz_pack(project, tgz_archive, tmpdir):
    with tarfile.open(name=tgz_archive['name']) as tgz_arch:
        # usr/share/tarantool is added to coorectly run assert_filemodes
        distribution_dir = os.path.join(tmpdir, 'usr/share/tarantool',
                                        project['name'])
        os.makedirs(distribution_dir, exist_ok=True)

        tgz_arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
        assert_dir_contents(files_list=recursive_listdir(distribution_dir),
                            exp_files_list=project['distribution_files_list'],
                            exp_rocks_content=project['rocks_content'])

        validate_version_file(project, distribution_dir)
        assert_filemodes(project, tmpdir)
def test_project_without_stateboard(cartridge_cmd,
                                    project_without_dependencies, pack_format,
                                    tmpdir):
    project = project_without_dependencies

    STATEBOARD_ENTRYPOINT_NAME = 'stateboard.init.lua'

    # remove stateboard entrypoint from project
    os.remove(os.path.join(project.path, STATEBOARD_ENTRYPOINT_NAME))
    project.distribution_files.remove(STATEBOARD_ENTRYPOINT_NAME)

    cmd = [
        cartridge_cmd,
        "pack",
        pack_format,
        project.path,
    ]

    # call cartridge pack
    process = subprocess.run(cmd, cwd=tmpdir)
    assert process.returncode == 0

    # packing should succeed with warning
    rc, output = run_command_and_get_output(cmd, cwd=tmpdir)
    assert rc == 0
    assert "App directory doesn't contain stateboard entrypoint script" in output

    # extract files from archive
    archive_path = find_archive(tmpdir, project.name, pack_format)
    extract_dir = os.path.join(tmpdir, 'extract')
    os.makedirs(extract_dir)

    if pack_format == 'rpm':
        extract_rpm(archive_path, extract_dir)
    elif pack_format == 'deb':
        extract_deb(archive_path, extract_dir)
        with tarfile.open(
                name=os.path.join(extract_dir, 'data.tar.xz')) as data_arch:
            data_arch.extractall(path=extract_dir)

    # check that stateboard unit file wasn't delivered
    systemd_dir = (os.path.join(extract_dir, 'etc/systemd/system'))
    assert os.path.exists(systemd_dir)

    systemd_files = recursive_listdir(systemd_dir)

    assert len(systemd_files) == 2
    assert '{}-stateboard.service'.format(project.name) not in systemd_files
Example #9
0
def test_building_without_path_specifying(cartridge_cmd,
                                          project_without_dependencies):
    project = project_without_dependencies

    # say `cartridge build` in project directory
    cmd = [
        cartridge_cmd,
        "build",
    ]
    process = subprocess.run(cmd, cwd=project.path)
    assert process.returncode == 0, 'Building project failed'

    # check that all expected rocks was installed
    files = recursive_listdir(project.path)
    assert '.rocks' in files
    assert all([rock in files for rock in project.rocks_content])
Example #10
0
def test_project_without_stateboard(cartridge_cmd,
                                    project_without_dependencies, pack_format,
                                    tmpdir):
    project = project_without_dependencies

    STATEBOARD_ENTRYPOINT_NAME = 'stateboard.init.lua'

    # remove stateboard entrypoint from project
    os.remove(os.path.join(project.path, STATEBOARD_ENTRYPOINT_NAME))
    project.distribution_files.remove(STATEBOARD_ENTRYPOINT_NAME)

    cmd = [
        cartridge_cmd,
        "pack",
        pack_format,
        project.path,
    ]

    if platform.system() == 'Darwin':
        cmd.append('--use-docker')

    # call cartridge pack
    process = subprocess.run(cmd, cwd=tmpdir)
    assert process.returncode == 0

    # packing should succeed with warning
    rc, output = run_command_and_get_output(cmd, cwd=tmpdir)
    assert rc == 0
    assert "App directory doesn't contain stateboard entrypoint script" in output

    # extract files from archive
    archive_path = find_archive(tmpdir, project.name, pack_format)
    extract_dir = os.path.join(tmpdir, 'extract')
    extract_app_files(archive_path, pack_format, extract_dir)

    # check that stateboard unit file wasn't delivered
    systemd_dir = (os.path.join(extract_dir, 'etc/systemd/system'))
    assert os.path.exists(systemd_dir)

    systemd_files = recursive_listdir(systemd_dir)

    assert len(systemd_files) == 2
    assert '{}-stateboard.service'.format(project.name) not in systemd_files
def test_tgz(tgz_archive, tmpdir):
    project = tgz_archive.project

    # archive files should be extracted to the empty directory
    # to correctly check archive contents
    extract_dir = os.path.join(tmpdir, 'extract')
    os.makedirs(extract_dir)

    with tarfile.open(name=tgz_archive.filepath) as tgz_arch:
        # usr/share/tarantool is added to correctly run assert_filemodes
        distribution_dir = os.path.join(extract_dir, 'usr/share/tarantool',
                                        project.name)
        os.makedirs(distribution_dir, exist_ok=True)

        tgz_arch.extractall(
            path=os.path.join(extract_dir, 'usr/share/tarantool'))
        assert_distribution_dir_contents(
            dir_contents=recursive_listdir(distribution_dir), project=project)

        validate_version_file(project, distribution_dir)
        assert_filemodes(project, extract_dir)
Example #12
0
def find_files_to_process():
    files_from_crawler = list(flattened(recursive_listdir(DOWNLOAD_DIR)))

    files_to_process = []
    files_to_ignore = []
    for path in files_from_crawler:
        try:
            import_date = find_date(path)
            size = os.path.getsize(path)
            files_to_process.append((path, 
                                     import_date,
                                     os.path.getsize(path)))
        except ValueError:
            files_to_ignore.append(path)

    def _import_date((_1, import_date, _2)): return import_date
    def _size((_1, _2, size)): return size
    bytes_accumulator = Accumulator()
    files_to_process.sort(key=_import_date)
    files_to_process = [(f, bytes_accumulator(_size(f)))
                        for f in files_to_process]
    bytes_to_process = bytes_accumulator.getvalue()

    return (bytes_to_process, files_to_process, files_to_ignore)
Example #13
0
        except (ValueError, ImportError), err:
            return False

    re_agency = re.compile('^[0-9]*[A-Z]+')

    def extract_prefix(filename):
        prefix_match = re_agency.match(filename.upper())
        if not prefix_match is None:
            prefix = prefix_match.group()
            return fix_prefix(prefix)
        else:
            return None

    files_to_process = filter(
        filename_has_date,
        map(os.path.basename, flattened(recursive_listdir(DOWNLOAD_DIR))))
    prefixes = map(extract_prefix, files_to_process)

    def unique(iterable):
        def combine(accum, item):
            accum[item] = None
            return accum

        return reduce(combine, iterable, {}).keys()

    def frequency(iterable):
        def combine(frequencies, item):
            cnt = frequencies.get(item, 0)
            frequencies[item] = cnt + 1
            return frequencies
Example #14
0
        try:
            import_date = find_date(filename)
            return True
        except (ValueError, ImportError), err:
            return False

    re_agency = re.compile('^[0-9]*[A-Z]+')
    def extract_prefix(filename):
        prefix_match = re_agency.match(filename.upper())
        if not prefix_match is None:
            prefix = prefix_match.group()
            return fix_prefix(prefix)
        else:
            return None

    files_to_process = filter(filename_has_date, map(os.path.basename, flattened(recursive_listdir(DOWNLOAD_DIR))))
    prefixes = map(extract_prefix, files_to_process)

    def unique(iterable):
        def combine(accum, item):
            accum[item] = None
            return accum
        return reduce(combine, iterable, {}).keys()

    def frequency(iterable):
        def combine(frequencies, item):
            cnt = frequencies.get(item, 0)
            frequencies[item] = cnt + 1
            return frequencies
        return reduce(combine, iterable, {})
Example #15
0
def test_pack(docker_image, tmpdir, docker_client):
    project = docker_image.project
    image_name = docker_image.name

    container = docker_client.containers.create(docker_image.name)
    container_distribution_dir = '/usr/share/tarantool/{}'.format(project.name)

    # check if distribution dir was created
    command = '[ -d "{}" ] && echo true || echo false'.format(
        container_distribution_dir)
    output = run_command_on_image(docker_client, image_name, command)
    assert output == 'true'

    # get distribution dir contents
    arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
    with open(arhive_path, 'wb') as f:
        bits, _ = container.get_archive(container_distribution_dir)
        for chunk in bits:
            f.write(chunk)

    with tarfile.open(arhive_path) as arch:
        arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
    os.remove(arhive_path)

    distribution_dir_contents = recursive_listdir(
        os.path.join(tmpdir, 'usr/share/tarantool/', project.name))

    # The runtime image is built using Dockerfile.<random-string> in the
    #   distribution directory
    # This dockerfile name should be added to project distribution files set
    #   to correctly check distribution directory contents
    for f in distribution_dir_contents:
        if f.startswith('Dockerfile') and f not in [
                'Dockerfile.build.cartridge', 'Dockerfile.cartridge'
        ]:
            project.distribution_files.add(f)
            break

    assert_distribution_dir_contents(
        dir_contents=recursive_listdir(
            os.path.join(tmpdir, 'usr/share/tarantool/', project.name)),
        project=project,
    )

    assert_filemodes(project, tmpdir)
    container.remove()

    if project.image_runtime_requirements_filepath is not None:
        command = 'ls {}'.format(project.image_runtime_requirements_filepath)
        output = run_command_on_image(docker_client, image_name, command)
        assert output == project.image_runtime_requirements_filepath

    if not tarantool_enterprise_is_used():
        # check if tarantool was installed
        command = 'yum list installed 2>/dev/null | grep tarantool'
        output = run_command_on_image(docker_client, image_name, command)

        packages_list = output.split('\n')
        assert any(['tarantool' in package for package in packages_list])

        # check tarantool version
        command = 'tarantool --version'
        output = run_command_on_image(docker_client, image_name, command)

        m = re.search(r'Tarantool\s+(\d+.\d+)', output)
        assert m is not None
        installed_version = m.group(1)

        m = re.search(r'(\d+.\d+)', tarantool_version())
        assert m is not None
        expected_version = m.group(1)

        assert installed_version == expected_version
Example #16
0
def test_docker_pack(project, docker_image, tmpdir, docker_client):
    image_name = docker_image['name']
    container = docker_client.containers.create(image_name)

    container_distribution_dir = '/usr/share/tarantool/{}'.format(project['name'])

    # check if distribution dir was created
    command = '[ -d "{}" ] && echo true || echo false'.format(container_distribution_dir)
    output = run_command_on_image(docker_client, image_name, command)
    assert output == 'true'

    # get distribution dir contents
    arhive_path = os.path.join(tmpdir, 'distribution_dir.tar')
    with open(arhive_path, 'wb') as f:
        bits, _ = container.get_archive(container_distribution_dir)
        for chunk in bits:
            f.write(chunk)

    with tarfile.open(arhive_path) as arch:
        arch.extractall(path=os.path.join(tmpdir, 'usr/share/tarantool'))
    os.remove(arhive_path)

    assert_dir_contents(
        files_list=recursive_listdir(os.path.join(tmpdir, 'usr/share/tarantool/', project['name'])),
        exp_files_list=project['distribution_files_list'],
        exp_rocks_content=project['rocks_content'],
        skip_tarantool_binaries=True
    )

    assert_filemodes(project, tmpdir)
    container.remove()

    if tarantool_enterprise_is_used():
        # check tarantool and tarantoolctl binaries
        command = '[ -d "/usr/share/tarantool/tarantool-enterprise/" ] && echo true || echo false'
        output = run_command_on_image(docker_client, image_name, command)
        assert output == 'true'

        command = 'cd /usr/share/tarantool/tarantool-enterprise/ && find .'
        output = run_command_on_image(docker_client, image_name, command)

        files_list = output.split('\n')
        files_list.remove('.')

        dir_contents = [
            os.path.normpath(filename)
            for filename in files_list
        ]

        assert 'tarantool' in dir_contents
        assert 'tarantoolctl' in dir_contents
    else:
        # check if tarantool was installed
        command = 'yum list installed 2>/dev/null | grep tarantool'
        output = run_command_on_image(docker_client, image_name, command)

        packages_list = output.split('\n')
        assert any(['tarantool' in package for package in packages_list])

        # check tarantool version
        command = 'yum info tarantool'
        output = run_command_on_image(docker_client, image_name, command)

        m = re.search(r'Version\s+:\s+(\d+)\.(\d+).', output)
        assert m is not None
        installed_version = m.groups()

        m = re.search(r'(\d+)\.(\d+)\.\d+', tarantool_version())
        assert m is not None
        expected_version = m.groups()

        assert installed_version == expected_version