def __init__(self,
log,
descriptor,
output,
template=None,
scripts=None,
additional_scripts=None,
without_sources=False,
dist_git=False,
ssl_verify=None):
self.log = log
self.pwd = os.path.realpath(os.path.dirname(
os.path.realpath(__file__)))
self.descriptor = os.path.realpath(descriptor)
self.without_sources = without_sources
self.dist_git = dist_git
self.output = output
self.dockerfile = os.path.join(self.output, "Dockerfile")
self.template = template
self.scripts = scripts
self.additional_scripts = additional_scripts
self.ssl_verify = ssl_verify
if self.dist_git:
self.git = Git(self.log, os.path.dirname(self.descriptor),
self.output)
def __init__(self, log, descriptor, output, template=None, scripts=None, additional_script=None, without_sources=False, dist_git=False, ssl_verify=True):
self.log = log
self.ssl_verify = ssl_verify
self.template = template
self.custom_template = False
self.additional_script = additional_script
if not os.path.exists(descriptor):
raise Error("Descriptor file '%s' could not be found. Please make sure you specified correct path." % descriptor)
self.pwd = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
with open(descriptor, 'r') as stream:
self.cfg = yaml.safe_load(stream)
self.output = output
self.scripts = scripts
self.without_sources = without_sources
self.dist_git = dist_git
self.dockerfile = os.path.join(self.output, "Dockerfile")
if self.dist_git:
self.git = Git(self.log, os.path.dirname(os.path.realpath(descriptor)), self.output)
def __init__(self, log, descriptor, output, template=None, scripts=None, additional_scripts=None, without_sources=False, dist_git=False, ssl_verify=None):
self.log = log
self.pwd = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
self.descriptor = os.path.realpath(descriptor)
self.without_sources = without_sources
self.dist_git = dist_git
self.output = output
self.dockerfile = os.path.join(self.output, "Dockerfile")
self.template = template
self.scripts = scripts
self.additional_scripts = additional_scripts
self.ssl_verify = ssl_verify
if self.dist_git:
self.git = Git(self.log, os.path.dirname(self.descriptor), self.output)
class Generator(object):
def __init__(self,
log,
descriptor,
output,
template=None,
scripts=None,
additional_scripts=None,
without_sources=False,
dist_git=False,
ssl_verify=None):
self.log = log
self.pwd = os.path.realpath(os.path.dirname(
os.path.realpath(__file__)))
self.descriptor = os.path.realpath(descriptor)
self.without_sources = without_sources
self.dist_git = dist_git
self.output = output
self.dockerfile = os.path.join(self.output, "Dockerfile")
self.template = template
self.scripts = scripts
self.additional_scripts = additional_scripts
self.ssl_verify = ssl_verify
if self.dist_git:
self.git = Git(self.log, os.path.dirname(self.descriptor),
self.output)
def _fetch_file(self, location, output=None):
"""
Fetches remote file and saves it under output. If no
output path is provided, a temporary file is created
and path to this file is returned.
SSL verification could be disabled by setting
self.ssl_verify to False.
"""
self.log.debug("Fetching '%s' file..." % location)
if not output:
output = tempfile.mktemp("-dogen")
self.log.debug("Fetched file will be saved as '%s'..." % output)
with open(output, 'wb') as f:
f.write(requests.get(location, verify=self.ssl_verify).content)
return output
def _handle_custom_template(self):
"""
Fetches custom template (if provided) and saves as temporary
file. This file is removed later in the process.
"""
if not self.template:
return
self.log.info("Using custom provided template file: '%s'" %
self.template)
if Tools.is_url(self.template):
self.template = self._fetch_file(self.template)
if not os.path.exists(self.template):
raise Error(
"Template file '%s' could not be found. Please make sure you specified correct path or check if the file was successfully fetched."
% self.template)
def configure(self):
"""
Reads configuration values from the descriptor, if provided.
Some Dogen configuration values can be set in the YAML
descriptor file using the 'dogen' section.
"""
# Fail early if descriptor file is not found
if not os.path.exists(self.descriptor):
raise Error(
"Descriptor file '%s' could not be found. Please make sure you specified correct path."
% self.descriptor)
if not self.scripts:
# If scripts directory is not provided, see if there is a "scripts"
# directory next to the descriptor. If found - assume that's the
# directory containing scripts.
scripts = os.path.join(os.path.dirname(self.descriptor), "scripts")
if os.path.exists(scripts) and os.path.isdir(scripts):
self.scripts = scripts
with open(self.descriptor, 'r') as stream:
self.cfg = yaml.safe_load(stream)
dogen_cfg = self.cfg.get('dogen')
if not dogen_cfg:
return
required_version = dogen_cfg.get('version')
if required_version:
# Check if the current runnig version of Dogen
# is the one the descriptor is expecting.
if required_version != version:
raise Error(
"You try to parse descriptor that requires Dogen version %s, but you run version %s"
% (required_version, version))
ssl_verify = dogen_cfg.get('ssl_verify')
if self.ssl_verify is None and ssl_verify is not None:
self.ssl_verify = ssl_verify
template = dogen_cfg.get('template')
if template and not self.template:
self.template = template
scripts = dogen_cfg.get('scripts')
if scripts and not self.scripts:
self.scripts = scripts
additional_scripts = dogen_cfg.get('additional_scripts')
if additional_scripts and not self.additional_scripts:
self.additional_scripts = additional_scripts
if self.scripts:
if not os.path.exists(self.scripts):
raise Error("Provided scripts directory '%s' does not exists" %
self.scripts)
def _handle_scripts(self):
if not self.cfg.get('scripts'):
return
for script in self.cfg['scripts']:
package = script['package']
src_path = os.path.join(self.scripts, package)
output_path = os.path.join(self.output, "scripts", package)
possible_exec = os.getenv('DOGEN_SCRIPT_EXEC', DEFAULT_SCRIPT_EXEC)
if "exec" not in script and os.path.exists(
os.path.join(src_path, possible_exec)):
script['exec'] = possible_exec
if "user" not in script:
script['user'] = os.getenv('DOGEN_SCRIPT_USER',
DEFAULT_SCRIPT_USER)
# Poor-man's workaround for not copying multiple times the same thing
if not os.path.exists(output_path):
self.log.info("Copying package '%s'..." % package)
shutil.copytree(src=src_path, dst=output_path)
self.log.debug("Done.")
def _handle_additional_scripts(self):
self.log.info("Additional scripts provided, installing them...")
output_scripts = os.path.join(self.output, "scripts")
if not os.path.exists(output_scripts):
os.makedirs(output_scripts)
for f in self.additional_scripts:
self.log.debug("Handling '%s' file..." % f)
if Tools.is_url(f):
self._fetch_file(
f, os.path.join(output_scripts, os.path.basename(f)))
else:
if not (os.path.exists(f) and os.path.isfile(f)):
raise Error(
"File '%s' does not exist. Please make sure you specified correct path to a file when specifying additional scripts."
% f)
self.log.debug(
"Copying '%s' file to target scripts directory..." % f)
shutil.copy(f, output_scripts)
def _handle_custom_repo_files(self):
self.cfg['additional_repos'] = []
repo_files = glob.glob(os.path.join(self.output, "scripts", "*.repo"))
if not repo_files:
return
self.log.debug("Found following additional repo files: %s" %
", ".join(repo_files))
for f in repo_files:
self.cfg['additional_repos'].append(
os.path.splitext(os.path.basename(f))[0])
def run(self):
# Set Dogen settings if provided in descriptor
self.configure()
# Special case for ssl_verify setting. Setting it to None
# in CLI if --skip-ssl-verification is not set to make it
# possible to determine which setting should be used.
# This means that we need to se the ssl_verify to the
# default value of True is not set.
if self.ssl_verify is None:
self.ssl_verify = True
if self.dist_git:
self.git.prepare()
if self.template:
self._handle_custom_template()
# Remove the target scripts directory
shutil.rmtree(os.path.join(self.output, "scripts"), ignore_errors=True)
if self.dist_git:
self.git.clean_scripts()
if not os.path.exists(self.output):
os.makedirs(self.output)
if self.scripts:
self._handle_scripts()
else:
self.log.warn("No scripts will be copied, mistake?")
# Additional scripts (not package scripts)
if self.additional_scripts:
self._handle_additional_scripts()
self._handle_custom_repo_files()
self.render_from_template()
self.handle_sources()
if self.dist_git:
self.git.update()
self.log.info("Finished!")
def render_from_template(self):
if self.template:
template_file = self.template
else:
self.log.debug("Using dogen provided template file")
template_file = os.path.join(self.pwd, "templates",
"template.jinja")
self.log.info("Rendering Dockerfile...")
loader = FileSystemLoader(os.path.dirname(template_file))
env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)
env.globals['helper'] = TemplateHelper()
template = env.get_template(os.path.basename(template_file))
with open(self.dockerfile, 'w') as f:
f.write(template.render(self.cfg).encode('utf-8'))
self.log.debug("Done")
if self.template and Tools.is_url(self.template):
self.log.debug("Removing temporary template file...")
os.remove(self.template)
def handle_sources(self):
if not 'sources' in self.cfg or self.without_sources:
return
files = []
for source in self.cfg['sources']:
url = source['url']
basename = os.path.basename(url)
files.append(basename)
filename = ("%s/%s" % (self.output, basename))
passed = False
try:
if os.path.exists(filename):
self.check_sum(filename, source['hash'])
passed = True
except:
passed = False
if not passed:
sources_cache = os.environ.get("DOGEN_SOURCES_CACHE")
if sources_cache:
self.log.info("Using '%s' as cached location for sources" %
sources_cache)
url = "%s/%s" % (sources_cache, basename)
self.log.info("Downloading '%s'..." % url)
with open(filename, 'wb') as f:
f.write(requests.get(url, verify=self.ssl_verify).content)
self.check_sum(filename, source['hash'])
if self.dist_git:
self.git.update_lookaside_cache(files)
def check_sum(self, filename, checksum):
self.log.info("Checking '%s' hash..." % os.path.basename(filename))
filesum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
if filesum != checksum:
raise Exception(
"The md5sum computed for the '%s' file ('%s') doesn't match the '%s' value"
% (filename, filesum, checksum))
self.log.debug("Hash is correct.")
class Generator(object):
def __init__(self, log, descriptor, output, template=None, scripts=None, additional_scripts=None, without_sources=False, dist_git=False, ssl_verify=None):
self.log = log
self.pwd = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
self.descriptor = os.path.realpath(descriptor)
self.without_sources = without_sources
self.dist_git = dist_git
self.output = output
self.dockerfile = os.path.join(self.output, "Dockerfile")
self.template = template
self.scripts = scripts
self.additional_scripts = additional_scripts
self.ssl_verify = ssl_verify
if self.dist_git:
self.git = Git(self.log, os.path.dirname(self.descriptor), self.output)
def _fetch_file(self, location, output=None):
"""
Fetches remote file and saves it under output. If no
output path is provided, a temporary file is created
and path to this file is returned.
SSL verification could be disabled by setting
self.ssl_verify to False.
"""
self.log.debug("Fetching '%s' file..." % location)
if not output:
output = tempfile.mktemp("-dogen")
self.log.debug("Fetched file will be saved as '%s'..." % output)
with open(output, 'wb') as f:
f.write(requests.get(location, verify=self.ssl_verify).content)
return output
def _handle_custom_template(self):
"""
Fetches custom template (if provided) and saves as temporary
file. This file is removed later in the process.
"""
if not self.template:
return
self.log.info("Using custom provided template file: '%s'" % self.template)
if Tools.is_url(self.template):
self.template = self._fetch_file(self.template)
if not os.path.exists(self.template):
raise Error("Template file '%s' could not be found. Please make sure you specified correct path or check if the file was successfully fetched." % self.template)
def configure(self):
"""
Reads configuration values from the descriptor, if provided.
Some Dogen configuration values can be set in the YAML
descriptor file using the 'dogen' section.
"""
# Fail early if descriptor file is not found
if not os.path.exists(self.descriptor):
raise Error("Descriptor file '%s' could not be found. Please make sure you specified correct path." % self.descriptor)
if not self.scripts:
# If scripts directory is not provided, see if there is a "scripts"
# directory next to the descriptor. If found - assume that's the
# directory containing scripts.
scripts = os.path.join(os.path.dirname(self.descriptor), "scripts")
if os.path.exists(scripts) and os.path.isdir(scripts):
self.scripts = scripts
with open(self.descriptor, 'r') as stream:
self.cfg = yaml.safe_load(stream)
dogen_cfg = self.cfg.get('dogen')
if not dogen_cfg:
return
required_version = dogen_cfg.get('version')
if required_version:
# Check if the current runnig version of Dogen
# is the one the descriptor is expecting.
if required_version != version:
raise Error("You try to parse descriptor that requires Dogen version %s, but you run version %s" % (required_version, version))
ssl_verify = dogen_cfg.get('ssl_verify')
if self.ssl_verify is None and ssl_verify is not None:
self.ssl_verify = ssl_verify
template = dogen_cfg.get('template')
if template and not self.template:
self.template = template
scripts = dogen_cfg.get('scripts')
if scripts and not self.scripts:
self.scripts = scripts
additional_scripts = dogen_cfg.get('additional_scripts')
if additional_scripts and not self.additional_scripts:
self.additional_scripts = additional_scripts
if self.scripts:
if not os.path.exists(self.scripts):
raise Error("Provided scripts directory '%s' does not exists" % self.scripts)
def _handle_scripts(self):
if not self.cfg.get('scripts'):
return
for script in self.cfg['scripts']:
package = script['package']
src_path = os.path.join(self.scripts, package)
output_path = os.path.join(self.output, "scripts", package)
possible_exec = os.getenv('DOGEN_SCRIPT_EXEC', DEFAULT_SCRIPT_EXEC)
if "exec" not in script and os.path.exists(os.path.join(src_path, possible_exec)):
script['exec'] = possible_exec
if "user" not in script:
script['user'] = os.getenv('DOGEN_SCRIPT_USER', DEFAULT_SCRIPT_USER)
# Poor-man's workaround for not copying multiple times the same thing
if not os.path.exists(output_path):
self.log.info("Copying package '%s'..." % package)
shutil.copytree(src=src_path, dst=output_path)
self.log.debug("Done.")
def _handle_additional_scripts(self):
self.log.info("Additional scripts provided, installing them...")
output_scripts = os.path.join(self.output, "scripts")
if not os.path.exists(output_scripts):
os.makedirs(output_scripts)
for f in self.additional_scripts:
self.log.debug("Handling '%s' file..." % f)
if Tools.is_url(f):
self._fetch_file(f, os.path.join(output_scripts, os.path.basename(f)))
else:
if not (os.path.exists(f) and os.path.isfile(f)):
raise Error("File '%s' does not exist. Please make sure you specified correct path to a file when specifying additional scripts." % f)
self.log.debug("Copying '%s' file to target scripts directory..." % f)
shutil.copy(f, output_scripts)
def _handle_custom_repo_files(self):
self.cfg['additional_repos'] = []
repo_files = glob.glob(os.path.join(self.output, "scripts", "*.repo"))
if not repo_files:
return
self.log.debug("Found following additional repo files: %s" % ", ".join(repo_files))
for f in repo_files:
self.cfg['additional_repos'].append(os.path.splitext(os.path.basename(f))[0])
def run(self):
# Set Dogen settings if provided in descriptor
self.configure()
# Special case for ssl_verify setting. Setting it to None
# in CLI if --skip-ssl-verification is not set to make it
# possible to determine which setting should be used.
# This means that we need to se the ssl_verify to the
# default value of True is not set.
if self.ssl_verify is None:
self.ssl_verify = True
if self.dist_git:
self.git.prepare()
if self.template:
self._handle_custom_template()
# Remove the target scripts directory
shutil.rmtree(os.path.join(self.output, "scripts"), ignore_errors=True)
if self.dist_git:
self.git.clean_scripts()
if not os.path.exists(self.output):
os.makedirs(self.output)
if self.scripts:
self._handle_scripts()
else:
self.log.warn("No scripts will be copied, mistake?")
# Additional scripts (not package scripts)
if self.additional_scripts:
self._handle_additional_scripts()
self._handle_custom_repo_files()
self.render_from_template()
self.handle_sources()
if self.dist_git:
self.git.update()
self.log.info("Finished!")
def render_from_template(self):
if self.template:
template_file = self.template
else:
self.log.debug("Using dogen provided template file")
template_file = os.path.join(self.pwd, "templates", "template.jinja")
self.log.info("Rendering Dockerfile...")
loader = FileSystemLoader(os.path.dirname(template_file))
env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)
env.globals['helper'] = TemplateHelper()
template = env.get_template(os.path.basename(template_file))
with open(self.dockerfile, 'w') as f:
f.write(template.render(self.cfg).encode('utf-8'))
self.log.debug("Done")
if self.template and Tools.is_url(self.template):
self.log.debug("Removing temporary template file...")
os.remove(self.template)
def handle_sources(self):
if not 'sources' in self.cfg or self.without_sources:
return
files = []
for source in self.cfg['sources']:
url = source['url']
basename = os.path.basename(url)
files.append(basename)
filename = ("%s/%s" %(self.output, basename))
passed = False
try:
if os.path.exists(filename):
self.check_sum(filename, source['hash'])
passed = True
except:
passed = False
if not passed:
sources_cache = os.environ.get("DOGEN_SOURCES_CACHE")
if sources_cache:
self.log.info("Using '%s' as cached location for sources" % sources_cache)
url = "%s/%s" % (sources_cache, basename)
self.log.info("Downloading '%s'..." % url)
with open(filename, 'wb') as f:
f.write(requests.get(url, verify=self.ssl_verify).content)
self.check_sum(filename, source['hash'])
if self.dist_git:
self.git.update_lookaside_cache(files)
def check_sum(self, filename, checksum):
self.log.info("Checking '%s' hash..." % os.path.basename(filename))
filesum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
if filesum != checksum:
raise Exception("The md5sum computed for the '%s' file ('%s') doesn't match the '%s' value" % (filename, filesum, checksum))
self.log.debug("Hash is correct.")
class Generator(object):
def __init__(self, log, descriptor, output, template=None, scripts=None, additional_script=None, without_sources=False, dist_git=False, ssl_verify=True):
self.log = log
self.ssl_verify = ssl_verify
self.template = template
self.custom_template = False
self.additional_script = additional_script
if not os.path.exists(descriptor):
raise Error("Descriptor file '%s' could not be found. Please make sure you specified correct path." % descriptor)
self.pwd = os.path.realpath(os.path.dirname(os.path.realpath(__file__)))
with open(descriptor, 'r') as stream:
self.cfg = yaml.safe_load(stream)
self.output = output
self.scripts = scripts
self.without_sources = without_sources
self.dist_git = dist_git
self.dockerfile = os.path.join(self.output, "Dockerfile")
if self.dist_git:
self.git = Git(self.log, os.path.dirname(os.path.realpath(descriptor)), self.output)
def is_url(self, location):
""" Checks if provided path is a URL """
return bool(urlparse.urlparse(location).netloc)
def fetch_file(self, location, output=None):
"""
Fetches remote file and saves it under output. If no
output path is provided, a temporary file is created
and path to this file is returned.
SSL verification could be disabled by setting
self.ssl_verify to True.
"""
self.log.debug("Fetching '%s' file..." % location)
if not output:
output = tempfile.mktemp("-dogen")
self.log.debug("File will be saved as '%s'..." % output)
with open(output, 'wb') as f:
f.write(requests.get(location, verify=self.ssl_verify).content)
return output
def handle_custom_template(self):
self.log.info("Using custom provided template file: '%s'" % self.template)
self.custom_template = True
if self.is_url(self.template):
self.template = self.fetch_file(self.template)
if not os.path.exists(self.template):
raise Error("Template file '%s' could not be found. Please make sure you specified correct path or check if the file was successfully fetched." % self.template)
def run(self):
if self.dist_git:
self.git.prepare()
if self.template:
self.handle_custom_template()
else:
self.log.debug("Using dogen provided template file")
self.template = os.path.join(self.pwd, "templates", "template.jinja")
# Remove the scripts directory
shutil.rmtree(os.path.join(self.output, "scripts"), ignore_errors=True)
if self.dist_git:
self.git.clean_scripts()
if not os.path.exists(self.output):
os.makedirs(self.output)
try:
for scripts in self.cfg['scripts']:
package = scripts['package']
output_path = os.path.join(self.output, "scripts", package)
try:
# Poor-man's workaround for not copying multiple times the same thing
if not os.path.exists(output_path):
self.log.info("Copying package '%s'..." % package)
shutil.copytree(src=os.path.join(self.scripts, package), dst=output_path)
self.log.debug("Done.")
except Exception, ex:
self.log.exception("Cannot copy package %s" % package, ex)
except KeyError:
pass
# Additional scripts (not package scripts)
if self.additional_script:
self.log.info("Additional scripts provided, installing them...")
output_scripts = os.path.join(self.output, "scripts")
if not os.path.exists(output_scripts):
os.makedirs(output_scripts)
for f in self.additional_script:
self.log.debug("Handling '%s' file..." % f)
if self.is_url(f):
self.fetch_file(f, os.path.join(output_scripts, os.path.basename(f)))
else:
if not (os.path.exists(f) and os.path.isfile(f)):
raise Error("File '%s' does not exist. Please make sure you specified correct path to a file when specifying additional scripts." % f)
self.log.debug("Copying '%s' file to target scripts directory..." % f)
shutil.copy(f, output_scripts)
self.render_from_template()
self.handle_sources()
if self.dist_git:
self.git.update()
def render_from_template(self):
self.log.info("Rendering Dockerfile...")
loader = FileSystemLoader(os.path.dirname(self.template))
env = Environment(loader=loader, trim_blocks=True, lstrip_blocks=True)
env.globals['helper'] = TemplateHelper()
template = env.get_template(os.path.basename(self.template))
with open(self.dockerfile, 'w') as f:
f.write(template.render(self.cfg).encode('utf-8'))
self.log.debug("Done")
if self.custom_template:
self.log.debug("Removing temporary template file...")
os.remove(self.template)
def handle_sources(self):
if not 'sources' in self.cfg or self.without_sources:
return
files = []
for source in self.cfg['sources']:
url = source['url']
basename = os.path.basename(url)
files.append(basename)
filename = ("%s/%s" %(self.output, basename))
passed = False
try:
if os.path.exists(filename):
self.check_sum(filename, source['hash'])
passed = True
except:
passed = False
if not passed:
sources_cache = os.environ.get("DOGEN_SOURCES_CACHE")
if sources_cache:
self.log.info("Using '%s' as cached location for sources" % sources_cache)
url = "%s/%s" % (sources_cache, basename)
self.log.info("Downloading '%s'..." % url)
with open(filename, 'wb') as f:
f.write(requests.get(url, verify=self.ssl_verify).content)
self.check_sum(filename, source['hash'])
if self.dist_git:
self.git.update_lookaside_cache(files)
def check_sum(self, filename, checksum):
self.log.info("Checking '%s' hash..." % os.path.basename(filename))
filesum = hashlib.md5(open(filename, 'rb').read()).hexdigest()
if filesum != checksum:
raise Exception("The md5sum computed for the '%s' file ('%s') doesn't match the '%s' value" % (filename, filesum, checksum))
self.log.debug("Hash is correct.")
