def test_parse_string_text(): ret = dockerfile.parse_string('FROM ubuntu:xenial\n' 'CMD ["echo", "☃"]\n', ) assert ret == ( dockerfile.Command( cmd='FROM', sub_cmd=None, json=False, value=('ubuntu:xenial', ), start_line=1, end_line=1, original='FROM ubuntu:xenial', flags=(), ), dockerfile.Command( cmd='CMD', sub_cmd=None, json=True, value=('echo', '☃'), start_line=2, end_line=2, original='CMD ["echo", "☃"]', flags=(), ), )
def test_parse_string_success(): ret = dockerfile.parse_string( 'FROM ubuntu:xenial\n' 'RUN echo hi > /etc/hi.conf\n' 'CMD ["echo"]\n' 'ONBUILD ADD foo bar\n' 'ONBUILD RUN ["cat", "bar"]\n' ) assert ret == ( dockerfile.Command( cmd='from', sub_cmd=None, json=False, value=('ubuntu:xenial',), start_line=1, original='FROM ubuntu:xenial', ), dockerfile.Command( cmd='run', sub_cmd=None, json=False, value=('echo hi > /etc/hi.conf',), start_line=2, original='RUN echo hi > /etc/hi.conf', ), dockerfile.Command( cmd='cmd', sub_cmd=None, json=True, value=('echo',), start_line=3, original='CMD ["echo"]', ), dockerfile.Command( cmd='onbuild', sub_cmd='add', json=False, value=('foo', 'bar'), start_line=4, original='ONBUILD ADD foo bar', ), dockerfile.Command( cmd='onbuild', sub_cmd='run', json=True, value=('cat', 'bar'), start_line=5, original='ONBUILD RUN ["cat", "bar"]', ), )
def checkcode(): ori_code = request.json['code'] language = request.json['language'] code = ori_code.replace('\r\n', '\n').replace('\r', '\n') assert language in ('shell', 'dockerfile'), 'Only shell or dockerfile is accepted' if language == 'dockerfile': try: dockerfile_commands = dockerfile.parse_string(ori_code) for command in dockerfile_commands: if command.cmd == 'run': markers, command_range = visitor.start( command.original.split(' ', 1)[-1].strip()) if len(markers) > 0: pre_lines = command.start_line - 1 # 'run ' takes at least 4 cols pre_cols = 4 while command.original[pre_cols] in (' ', '\t'): pre_cols += 1 assert pre_cols <= 200, "Should be an error. It is very unlikely to have pre_cols larger than 200" markers, command_range = refine_markers_and_command_range( pre_lines=pre_lines, pre_cols=pre_cols, markers=markers, command_range=command_range) return jsonify({ "error": { "code": ori_code, "markers": markers }, "commandRange": command_range }) except: pass return jsonify({ "error": { "code": ori_code, "markers": [] }, "commandRange": {} }) else: markers, command_range = visitor.start(code) return jsonify({ "error": { "code": ori_code, "markers": markers }, "commandRange": command_range })
def _parse_content(self, input: str, expected: str) -> None: input_text = textwrap.dedent(input) expected_text = textwrap.dedent(expected) expected_text = f"# compiled by docker-optimizer\n" \ f"# https://github.com/bmustiata/docker-optimizer\n" \ f"{expected_text}" commands: List[DockerCommand] = dockerfile.parse_string(input_text) optimized_commands = optimize_docker_commands(commands) with io.StringIO() as output: write_docker_commands(output, optimized_commands) self.assertEqual(output.getvalue().strip(), expected_text.strip())
def create_docker_setup( *, commands: Iterable[str]=None, context_files: Dict[str, Optional[str]]=None, image_name: str=_RANDOM_NAME, tags: List[str]=None, always_upload: bool=False, from_image_name: str=EXAMPLE_FROM_IMAGE_NAME) -> Tuple[str, DockerBuildConfiguration]: """ Creates a Docker setup. :param commands: commands to put in the Dockerfile. If `None` and `from_image_name` is set, FROM will be set :param context_files: dictionary where the key is the name of the context file and the value is its content :param image_name: name of the image to setup a build configuration for :param tags: list of strings to tag the built image with :param always_upload: always upload the docker image, even if it has not just been built :param from_image_name: the image that the setup one is based off (FROM added to commands if not `None`) :return: tuple where the first element is the directory that acts as the context and the second is the associated build configuration """ if from_image_name is not None: from_command = f"{FROM_DOCKER_COMMAND} {from_image_name}" if commands is None: commands = (from_command, ) else: commands = (from_command, *commands) parsed_commands = dockerfile.parse_string("\n".join(commands)) if len([command.cmd for command in parsed_commands if command.cmd.lower() == FROM_DOCKER_COMMAND.lower()]) != 1: raise ValueError(f"Exactly one \"{FROM_DOCKER_COMMAND}\" command is expected: {commands}") context_files = context_files if context_files is not None else {} image_name = image_name if image_name != _RANDOM_NAME else f"{name_generator()}" temp_directory = mkdtemp() dockerfile_location = os.path.join(temp_directory, DOCKERFILE_PATH) with open(dockerfile_location, "w") as file: for command in commands: file.write(f"{command}\n") for location, value in context_files.items(): absolute_location = os.path.join(temp_directory, location) os.makedirs(os.path.dirname(absolute_location), exist_ok=True) with open(absolute_location, "w") as file: if value is None: value = "" file.write(value) return temp_directory, DockerBuildConfiguration( image_name, dockerfile_location, tags=tags, always_upload=always_upload)
def run(self, terms, variables, **kwargs): if not HAS_DOCKERFILE: raise errors.AnsibleError('dockerfile module is missing') if len(terms) == 2: terms[1] = listify_lookup_plugin_terms(terms[1], templar=self._templar, loader=self._loader) terms[1] = map(lambda cmd: unicode(cmd.lower()), terms[1]) if not isinstance(terms, list) or not 1 <= len(terms) <= 2: raise errores.AnsibleError('dockerfile lookup expects a list of one or two items'); try: df = dockerfile.parse_string(unicode(terms[0])) except (dockerfile.GoIOError, dockerfile.GoParseError) as e: if isinstance(e, dockerfile.GoIOError): raise_from(errors.AnsibleError("Dockerfile couldn't be opened"), e) else: raise_from( errors.AnsibleError("Dockerfile couldn't be parsed"), e) return [command._asdict() for command in df if len(terms) == 1 or command.cmd in terms[1]]
def from_string(cls, dockerfile_contents: str) -> ParsedDockerfile: return cls("<text>", parse_string(dockerfile_contents))
def parse(cls, dockerfile_contents: str) -> "ParsedDockerfile": return cls(parse_string(dockerfile_contents))
def convert(self): """Parse the Dockerfile and output JSON to STDOUT.""" parsed_dockerfile = dockerfile.parse_string(u"" + self.dockerfile) json = simplejson.dumps(parsed_dockerfile) print(json)
dockerfile_ast['file_sha'] = item[1].split('/')[-1].replace( '.Dockerfile', '' ).strip() return json.dumps(dockerfile_ast) except Exception as ex: return None pool = multiprocessing.Pool() all_lines = [] for line in sys.stdin: with open(line.strip()) as dfh: try: all_lines.append(( dockerfile.parse_string(dfh.read()), line.strip() )) except Exception: continue results = pool.imap(process, all_lines, chunksize=500) with lzma.open('/mnt/outputs/{}.jsonl.xz'.format(sys.argv[1]), mode='wt') as out_file: for result in tqdm.tqdm(results, total=len(all_lines), desc="Generating"): if result is None: continue out_file.write('{}\n'.format(result))
def test_parse_string_success(): ret = dockerfile.parse_string( 'FROM ubuntu:xenial\n' 'RUN echo hi > /etc/hi.conf\n' 'CMD ["echo"]\n' 'HEALTHCHECK --retries=5 CMD echo hi\n' 'ONBUILD ADD foo bar\n' 'ONBUILD RUN ["cat", "bar"]\n', ) assert ret == ( dockerfile.Command( cmd='FROM', sub_cmd=None, json=False, flags=(), value=('ubuntu:xenial', ), start_line=1, end_line=1, original='FROM ubuntu:xenial', ), dockerfile.Command( cmd='RUN', sub_cmd=None, json=False, flags=(), value=('echo hi > /etc/hi.conf', ), start_line=2, end_line=2, original='RUN echo hi > /etc/hi.conf', ), dockerfile.Command( cmd='CMD', sub_cmd=None, json=True, flags=(), value=('echo', ), start_line=3, end_line=3, original='CMD ["echo"]', ), dockerfile.Command( cmd='HEALTHCHECK', sub_cmd=None, json=False, flags=('--retries=5', ), value=('CMD', 'echo hi'), start_line=4, end_line=4, original='HEALTHCHECK --retries=5 CMD echo hi', ), dockerfile.Command( cmd='ONBUILD', sub_cmd='ADD', json=False, flags=(), value=('foo', 'bar'), start_line=5, end_line=5, original='ONBUILD ADD foo bar', ), dockerfile.Command( cmd='ONBUILD', sub_cmd='RUN', json=True, flags=(), value=('cat', 'bar'), start_line=6, end_line=6, original='ONBUILD RUN ["cat", "bar"]', ), )
def test_parse_string_parse_error(): with pytest.raises(dockerfile.GoParseError): dockerfile.parse_string('FROM ubuntu:xenial\n' 'CMD ["echo", 1]\n', )
def to_instruction(parsed_instruction): """Map a parsed `dockerfile.Command` to a `dockerphile.structures` type. `dockerfile.Command` contains the original parsed string (with line continuations and all comments removed) and also contains a `value` attribute storing the part of the Dockerfile instruction non-inclusive of Dockerfile keywords. Known limitations include: (1) no parsing of comment lines; (2) no parsing of the `escape` parser directive; and (2) no direct parsing of the optional arguments for the HEALTHCHECK or COPY commands. Args: parsed_instruction: An instance of `dockerfile.Command`. Returns: Either an instance of a `dockerphile.structures` namedtuple representing the parsed command using `dockerphile` types, or a list of such types. Return value is None for any underlying commands that are not supported by `dockerphile` even if parsed correctly, such as `MAINTAINER` (since `dockerphile` requires following Docker best practices to use `LABEL` for a maintainer label. Raises: DockerphileError: raised if incorrect instruction type is provided or if the parsed `dockerfile.Command` object contains invalid Dockerfile syntax. dockerfile.GoParseError: raised if underlying `dockerfile` library Go parser encounters an unhandled parser error. """ cmd, original = parsed_instruction.cmd, parsed_instruction.original uses_json, value = parsed_instruction.json, parsed_instruction.value if cmd == 'add': return structures.ADD(value) if cmd == 'arg': arg_string = value[0] parsed_arg = re.match(arg_default_value, arg_string) if parsed_arg is not None: key, default_value = parsed_arg.groups() return structures.ARG(key, default_value=default_value) else: return structures.ARG(arg_string) if cmd == 'cmd': return structures.CMD(**{ ('exec_form' if uses_json else 'shell_form'): value }) if cmd == 'copy': parsed_from = re.match(copy_from, original) from_ = parsed_from.groups()[0] if parsed_from is not None else None return structures.COPY(value, from_=from_) if cmd == 'entrypoint': return structures.ENTRYPOINT(**{ ('exec_form' if uses_json else 'shell_form'): value }) if cmd == 'env': if not value: raise DockerphileError("ENV command has no key or value.") if len(value) == 2: return structures.ENV(value[0], value[1]) try: result = [structures.ENV(value[k], value[k + 1]) for k in range(len(value))[::2]] except IndexError: raise DockerphileError("Unpaired index when parsing " "%s to list of ENV types." % value) return result if cmd == 'expose': return structures.EXPOSE(value) if cmd == 'from': parsed_from = re.match(from_as, original) if parsed_from is not None: image, as_ = parsed_from.groups() return structures.FROM(image, as_=as_) if not value: raise DockerphileError("FROM instruction requires nonempty base " "image.") return structures.FROM(value[0]) if cmd == 'healthcheck': if value and value[0] != 'CMD': raise DockerphileError("Invalid CMD specifier for HEALTHCHECK: " "%s" % value[0]) options = { 'interval': re.match(healthcheck_interval, original), 'timeout': re.match(healthcheck_timeout, original), 'start_period': re.match(healthcheck_start_period, original), 'retries': re.match(healthcheck_retries, original) } kwargs = { arg_name: (None if arg_match is None else arg_match.groups()[0]) for arg_name, arg_match in options.items() } if len(value) == 2: kwargs['cmd'] = value[1] elif len(value) > 2: kwargs['cmd'] = value[1:] return structures.HEALTHCHECK(**kwargs) if cmd == 'label': if not value: raise DockerphileError("LABEL command has no key or value.") if len(value) == 2: return structures.LABEL(value[0], value[1]) try: result = [structures.LABEL(value[k], value[k + 1]) for k in range(len(value))[::2]] except IndexError: msg = "Unpaired index when parsing %s to list of LABEL types." raise DockerphileError(msg % value) return result if cmd == 'maintainer': return None if cmd == 'onbuild': if parsed_instruction.sub_cmd == 'onbuild': raise DockerphileError("ONBUILD is not allowed as subcommand of " "ONBUILD.") sub_cmd_str = original.replace('ONBUILD', '') parsed_sub_cmd = parse_string(sub_cmd_str)[0] return structures.ONBUILD(to_instruction(parsed_sub_cmd)) if cmd == 'run': return structures.RUN(**{ ('exec_form' if uses_json else 'shell_form'): value }) if cmd == "shell": if not uses_json: raise DockerphileError("SHELL instruction requires used of JSON " "array option format.") if not value: raise DockerphileError("SHELL instruction requires nonempty JSON " "array option format.") return structures.SHELL(value) if cmd == 'stopsignal': if not value: raise DockerphileError("STOPSIGNAL instruction requires nonempty " "value.") return structures.STOPSIGNAL(value[0]) if cmd == 'user': if not value: raise DockerphileError("USER instruction requires nonempty value.") parsed_user_group = re.match(user_group, original) if parsed_user_group is not None: user, group = parsed_user_group.groups() else: user, group = value[0], None return structures.USER(user, group=group) if cmd == 'volume': return structures.VOLUME(value) if cmd == 'workdir': if not value: raise DockerphileError("WORKDIR instruction requires nonempty " "value.") return structures.WORKDIR(value[0]) raise DockerphileError("Unrecognized dockerfile.Command parsed command " "%s" % cmd)