def test_parse_string_text():
ret = dockerfile.parse_string('FROM ubuntu:xenial\n'
'CMD ["echo", "☃"]\n', )
assert ret == (
dockerfile.Command(
cmd='FROM',
sub_cmd=None,
json=False,
value=('ubuntu:xenial', ),
start_line=1,
end_line=1,
original='FROM ubuntu:xenial',
flags=(),
),
dockerfile.Command(
cmd='CMD',
sub_cmd=None,
json=True,
value=('echo', '☃'),
start_line=2,
end_line=2,
original='CMD ["echo", "☃"]',
flags=(),
),
)
def test_parse_string_success():
ret = dockerfile.parse_string(
'FROM ubuntu:xenial\n'
'RUN echo hi > /etc/hi.conf\n'
'CMD ["echo"]\n'
'ONBUILD ADD foo bar\n'
'ONBUILD RUN ["cat", "bar"]\n'
)
assert ret == (
dockerfile.Command(
cmd='from', sub_cmd=None, json=False, value=('ubuntu:xenial',),
start_line=1, original='FROM ubuntu:xenial',
),
dockerfile.Command(
cmd='run', sub_cmd=None, json=False,
value=('echo hi > /etc/hi.conf',),
start_line=2, original='RUN echo hi > /etc/hi.conf',
),
dockerfile.Command(
cmd='cmd', sub_cmd=None, json=True, value=('echo',),
start_line=3, original='CMD ["echo"]',
),
dockerfile.Command(
cmd='onbuild', sub_cmd='add', json=False, value=('foo', 'bar'),
start_line=4, original='ONBUILD ADD foo bar',
),
dockerfile.Command(
cmd='onbuild', sub_cmd='run', json=True, value=('cat', 'bar'),
start_line=5, original='ONBUILD RUN ["cat", "bar"]',
),
)
def checkcode():
ori_code = request.json['code']
language = request.json['language']
code = ori_code.replace('\r\n', '\n').replace('\r', '\n')
assert language in ('shell',
'dockerfile'), 'Only shell or dockerfile is accepted'
if language == 'dockerfile':
try:
dockerfile_commands = dockerfile.parse_string(ori_code)
for command in dockerfile_commands:
if command.cmd == 'run':
markers, command_range = visitor.start(
command.original.split(' ', 1)[-1].strip())
if len(markers) > 0:
pre_lines = command.start_line - 1
# 'run ' takes at least 4 cols
pre_cols = 4
while command.original[pre_cols] in (' ', '\t'):
pre_cols += 1
assert pre_cols <= 200, "Should be an error. It is very unlikely to have pre_cols larger than 200"
markers, command_range = refine_markers_and_command_range(
pre_lines=pre_lines,
pre_cols=pre_cols,
markers=markers,
command_range=command_range)
return jsonify({
"error": {
"code": ori_code,
"markers": markers
},
"commandRange": command_range
})
except:
pass
return jsonify({
"error": {
"code": ori_code,
"markers": []
},
"commandRange": {}
})
else:
markers, command_range = visitor.start(code)
return jsonify({
"error": {
"code": ori_code,
"markers": markers
},
"commandRange": command_range
})
def _parse_content(self, input: str, expected: str) -> None:
input_text = textwrap.dedent(input)
expected_text = textwrap.dedent(expected)
expected_text = f"# compiled by docker-optimizer\n" \
f"# https://github.com/bmustiata/docker-optimizer\n" \
f"{expected_text}"
commands: List[DockerCommand] = dockerfile.parse_string(input_text)
optimized_commands = optimize_docker_commands(commands)
with io.StringIO() as output:
write_docker_commands(output, optimized_commands)
self.assertEqual(output.getvalue().strip(), expected_text.strip())
def create_docker_setup(
*, commands: Iterable[str]=None, context_files: Dict[str, Optional[str]]=None,
image_name: str=_RANDOM_NAME, tags: List[str]=None, always_upload: bool=False,
from_image_name: str=EXAMPLE_FROM_IMAGE_NAME) -> Tuple[str, DockerBuildConfiguration]:
"""
Creates a Docker setup.
:param commands: commands to put in the Dockerfile. If `None` and `from_image_name` is set, FROM will be set
:param context_files: dictionary where the key is the name of the context file and the value is its content
:param image_name: name of the image to setup a build configuration for
:param tags: list of strings to tag the built image with
:param always_upload: always upload the docker image, even if it has not just been built
:param from_image_name: the image that the setup one is based off (FROM added to commands if not `None`)
:return: tuple where the first element is the directory that acts as the context and the second is the associated
build configuration
"""
if from_image_name is not None:
from_command = f"{FROM_DOCKER_COMMAND} {from_image_name}"
if commands is None:
commands = (from_command, )
else:
commands = (from_command, *commands)
parsed_commands = dockerfile.parse_string("\n".join(commands))
if len([command.cmd for command in parsed_commands if command.cmd.lower() == FROM_DOCKER_COMMAND.lower()]) != 1:
raise ValueError(f"Exactly one \"{FROM_DOCKER_COMMAND}\" command is expected: {commands}")
context_files = context_files if context_files is not None else {}
image_name = image_name if image_name != _RANDOM_NAME else f"{name_generator()}"
temp_directory = mkdtemp()
dockerfile_location = os.path.join(temp_directory, DOCKERFILE_PATH)
with open(dockerfile_location, "w") as file:
for command in commands:
file.write(f"{command}\n")
for location, value in context_files.items():
absolute_location = os.path.join(temp_directory, location)
os.makedirs(os.path.dirname(absolute_location), exist_ok=True)
with open(absolute_location, "w") as file:
if value is None:
value = ""
file.write(value)
return temp_directory, DockerBuildConfiguration(
image_name, dockerfile_location, tags=tags, always_upload=always_upload)
def run(self, terms, variables, **kwargs):
if not HAS_DOCKERFILE:
raise errors.AnsibleError('dockerfile module is missing')
if len(terms) == 2:
terms[1] = listify_lookup_plugin_terms(terms[1], templar=self._templar, loader=self._loader)
terms[1] = map(lambda cmd: unicode(cmd.lower()), terms[1])
if not isinstance(terms, list) or not 1 <= len(terms) <= 2:
raise errores.AnsibleError('dockerfile lookup expects a list of one or two items');
try:
df = dockerfile.parse_string(unicode(terms[0]))
except (dockerfile.GoIOError, dockerfile.GoParseError) as e:
if isinstance(e, dockerfile.GoIOError):
raise_from(errors.AnsibleError("Dockerfile couldn't be opened"), e)
else:
raise_from( errors.AnsibleError("Dockerfile couldn't be parsed"), e)
return [command._asdict() for command in df if len(terms) == 1 or command.cmd in terms[1]]
def from_string(cls, dockerfile_contents: str) -> ParsedDockerfile:
return cls("<text>", parse_string(dockerfile_contents))
def parse(cls, dockerfile_contents: str) -> "ParsedDockerfile":
return cls(parse_string(dockerfile_contents))
def convert(self):
"""Parse the Dockerfile and output JSON to STDOUT."""
parsed_dockerfile = dockerfile.parse_string(u"" + self.dockerfile)
json = simplejson.dumps(parsed_dockerfile)
print(json)
dockerfile_ast['file_sha'] = item[1].split('/')[-1].replace(
'.Dockerfile', ''
).strip()
return json.dumps(dockerfile_ast)
except Exception as ex:
return None
pool = multiprocessing.Pool()
all_lines = []
for line in sys.stdin:
with open(line.strip()) as dfh:
try:
all_lines.append((
dockerfile.parse_string(dfh.read()),
line.strip()
))
except Exception:
continue
results = pool.imap(process, all_lines, chunksize=500)
with lzma.open('/mnt/outputs/{}.jsonl.xz'.format(sys.argv[1]), mode='wt') as out_file:
for result in tqdm.tqdm(results, total=len(all_lines), desc="Generating"):
if result is None:
continue
out_file.write('{}\n'.format(result))
def test_parse_string_success():
ret = dockerfile.parse_string(
'FROM ubuntu:xenial\n'
'RUN echo hi > /etc/hi.conf\n'
'CMD ["echo"]\n'
'HEALTHCHECK --retries=5 CMD echo hi\n'
'ONBUILD ADD foo bar\n'
'ONBUILD RUN ["cat", "bar"]\n', )
assert ret == (
dockerfile.Command(
cmd='FROM',
sub_cmd=None,
json=False,
flags=(),
value=('ubuntu:xenial', ),
start_line=1,
end_line=1,
original='FROM ubuntu:xenial',
),
dockerfile.Command(
cmd='RUN',
sub_cmd=None,
json=False,
flags=(),
value=('echo hi > /etc/hi.conf', ),
start_line=2,
end_line=2,
original='RUN echo hi > /etc/hi.conf',
),
dockerfile.Command(
cmd='CMD',
sub_cmd=None,
json=True,
flags=(),
value=('echo', ),
start_line=3,
end_line=3,
original='CMD ["echo"]',
),
dockerfile.Command(
cmd='HEALTHCHECK',
sub_cmd=None,
json=False,
flags=('--retries=5', ),
value=('CMD', 'echo hi'),
start_line=4,
end_line=4,
original='HEALTHCHECK --retries=5 CMD echo hi',
),
dockerfile.Command(
cmd='ONBUILD',
sub_cmd='ADD',
json=False,
flags=(),
value=('foo', 'bar'),
start_line=5,
end_line=5,
original='ONBUILD ADD foo bar',
),
dockerfile.Command(
cmd='ONBUILD',
sub_cmd='RUN',
json=True,
flags=(),
value=('cat', 'bar'),
start_line=6,
end_line=6,
original='ONBUILD RUN ["cat", "bar"]',
),
)
def test_parse_string_parse_error():
with pytest.raises(dockerfile.GoParseError):
dockerfile.parse_string('FROM ubuntu:xenial\n' 'CMD ["echo", 1]\n', )
def to_instruction(parsed_instruction):
"""Map a parsed `dockerfile.Command` to a `dockerphile.structures` type.
`dockerfile.Command` contains the original parsed string (with line
continuations and all comments removed) and also contains a `value`
attribute storing the part of the Dockerfile instruction non-inclusive
of Dockerfile keywords.
Known limitations include: (1) no parsing of comment lines; (2) no parsing
of the `escape` parser directive; and (2) no direct parsing of the optional
arguments for the HEALTHCHECK or COPY commands.
Args:
parsed_instruction: An instance of `dockerfile.Command`.
Returns:
Either an instance of a `dockerphile.structures` namedtuple
representing the parsed command using `dockerphile` types, or a list of
such types. Return value is None for any underlying commands that are
not supported by `dockerphile` even if parsed correctly, such as
`MAINTAINER` (since `dockerphile` requires following Docker best
practices to use `LABEL` for a maintainer label.
Raises:
DockerphileError: raised if incorrect instruction type is provided or
if the parsed `dockerfile.Command` object contains invalid
Dockerfile syntax.
dockerfile.GoParseError: raised if underlying `dockerfile` library
Go parser encounters an unhandled parser error.
"""
cmd, original = parsed_instruction.cmd, parsed_instruction.original
uses_json, value = parsed_instruction.json, parsed_instruction.value
if cmd == 'add':
return structures.ADD(value)
if cmd == 'arg':
arg_string = value[0]
parsed_arg = re.match(arg_default_value, arg_string)
if parsed_arg is not None:
key, default_value = parsed_arg.groups()
return structures.ARG(key, default_value=default_value)
else:
return structures.ARG(arg_string)
if cmd == 'cmd':
return structures.CMD(**{
('exec_form' if uses_json else 'shell_form'): value
})
if cmd == 'copy':
parsed_from = re.match(copy_from, original)
from_ = parsed_from.groups()[0] if parsed_from is not None else None
return structures.COPY(value, from_=from_)
if cmd == 'entrypoint':
return structures.ENTRYPOINT(**{
('exec_form' if uses_json else 'shell_form'): value
})
if cmd == 'env':
if not value:
raise DockerphileError("ENV command has no key or value.")
if len(value) == 2:
return structures.ENV(value[0], value[1])
try:
result = [structures.ENV(value[k], value[k + 1])
for k in range(len(value))[::2]]
except IndexError:
raise DockerphileError("Unpaired index when parsing "
"%s to list of ENV types." % value)
return result
if cmd == 'expose':
return structures.EXPOSE(value)
if cmd == 'from':
parsed_from = re.match(from_as, original)
if parsed_from is not None:
image, as_ = parsed_from.groups()
return structures.FROM(image, as_=as_)
if not value:
raise DockerphileError("FROM instruction requires nonempty base "
"image.")
return structures.FROM(value[0])
if cmd == 'healthcheck':
if value and value[0] != 'CMD':
raise DockerphileError("Invalid CMD specifier for HEALTHCHECK: "
"%s" % value[0])
options = {
'interval': re.match(healthcheck_interval, original),
'timeout': re.match(healthcheck_timeout, original),
'start_period': re.match(healthcheck_start_period, original),
'retries': re.match(healthcheck_retries, original)
}
kwargs = {
arg_name: (None if arg_match is None else arg_match.groups()[0])
for arg_name, arg_match in options.items()
}
if len(value) == 2:
kwargs['cmd'] = value[1]
elif len(value) > 2:
kwargs['cmd'] = value[1:]
return structures.HEALTHCHECK(**kwargs)
if cmd == 'label':
if not value:
raise DockerphileError("LABEL command has no key or value.")
if len(value) == 2:
return structures.LABEL(value[0], value[1])
try:
result = [structures.LABEL(value[k], value[k + 1])
for k in range(len(value))[::2]]
except IndexError:
msg = "Unpaired index when parsing %s to list of LABEL types."
raise DockerphileError(msg % value)
return result
if cmd == 'maintainer':
return None
if cmd == 'onbuild':
if parsed_instruction.sub_cmd == 'onbuild':
raise DockerphileError("ONBUILD is not allowed as subcommand of "
"ONBUILD.")
sub_cmd_str = original.replace('ONBUILD', '')
parsed_sub_cmd = parse_string(sub_cmd_str)[0]
return structures.ONBUILD(to_instruction(parsed_sub_cmd))
if cmd == 'run':
return structures.RUN(**{
('exec_form' if uses_json else 'shell_form'): value
})
if cmd == "shell":
if not uses_json:
raise DockerphileError("SHELL instruction requires used of JSON "
"array option format.")
if not value:
raise DockerphileError("SHELL instruction requires nonempty JSON "
"array option format.")
return structures.SHELL(value)
if cmd == 'stopsignal':
if not value:
raise DockerphileError("STOPSIGNAL instruction requires nonempty "
"value.")
return structures.STOPSIGNAL(value[0])
if cmd == 'user':
if not value:
raise DockerphileError("USER instruction requires nonempty value.")
parsed_user_group = re.match(user_group, original)
if parsed_user_group is not None:
user, group = parsed_user_group.groups()
else:
user, group = value[0], None
return structures.USER(user, group=group)
if cmd == 'volume':
return structures.VOLUME(value)
if cmd == 'workdir':
if not value:
raise DockerphileError("WORKDIR instruction requires nonempty "
"value.")
return structures.WORKDIR(value[0])
raise DockerphileError("Unrecognized dockerfile.Command parsed command "
"%s" % cmd)
