def __init__(self,
project,
force=False,
endpoint_name='foo',
service_name=None):
# custom init
self.custom_project = project
self.force_yes = force
self.original_name = endpoint_name
self.service_name = service_name
names = []
self.class_name = ''
endpoint_name = endpoint_name.lstrip('/')
for piece in endpoint_name.lower().replace(' ', '-').split('-'):
if not piece.isalnum():
log.exit("Only alpha-numeric chars are allowed: {}", piece)
else:
names.append(piece)
self.class_name += piece.capitalize()
self.endpoint_dir = '_'.join(names)
self.endpoint_name = self.endpoint_dir.replace('_', '')
self.specs_file = 'specs.yaml'
# setting the base dir for all scaffold things inside the project
self.backend_dir = Path(PROJECT_DIR, self.custom_project, BACKEND_DIR)
self.base_backend_dir = Path(SUBMODULES_DIR, BACKEND_DIR,
BACKEND_PACKAGE)
def clone(online_url,
path,
branch='master',
do=False,
check=True,
expand_path=True):
if expand_path:
local_path = os.path.join(os.curdir, SUBMODULES_DIR, path)
else:
local_path = path
local_path_exists = os.path.exists(local_path)
if local_path_exists:
log.debug("Path {} already exists", local_path)
gitobj = Repo(local_path)
elif do:
gitobj = Repo.clone_from(url=online_url, to_path=local_path)
log.info("Cloned repo {}@{} as {}", online_url, branch, path)
else:
log.exit("Repo {} missing as {}. You should init your project".format(
online_url, local_path))
if do:
switch_branch(gitobj, branch)
if check:
compare_repository(gitobj, branch, online_url=online_url, path=path)
return gitobj
def exec_command(self,
service,
user=None,
command=None,
disable_tty=False,
nofailure=False):
"""
Execute a command on a running container
"""
shell_command, shell_args = self.split_command(command)
options = {
'SERVICE': service,
'COMMAND': shell_command,
'ARGS': shell_args,
'--index': '1',
'--user': user,
'-T': disable_tty,
'--env': None,
'--workdir': None,
# '-d': False,
'--detach': False,
'--privileged': False,
}
if shell_command is not None:
log.debug("Command: {}({}+{})", service.lower(), shell_command,
shell_args)
try:
out = self.command('exec_command', options, nofailure=nofailure)
except NoSuchService:
if nofailure:
raise AttributeError("Cannot find service: {}".format(service))
else:
log.exit("Cannot find a running container called {}", service)
else:
return out
def config(self):
try:
compose_output_tuple = get_config_from_options('.', self.options)
# NOTE: for compatibility with docker-compose > 1.13
# services is always the second element
services_list = compose_output_tuple[1]
except conferrors.ConfigurationError as e:
log.exit("Wrong compose configuration:\n{}", e)
else:
return services_list
def fetch(path, gitobj, fetch_remote='origin'):
for remote in gitobj.remotes:
if remote.name != fetch_remote:
log.verbose("Skipping fetch of remote {} on {}", remote, path)
continue
log.verbose("Fetching {} on {}", remote, path)
try:
remote.fetch()
except GitCommandError as e:
log.exit(str(e))
def info(self):
infos = '\n'
base_endpoint = False
endpoint = self.endpoint_name
# look inside extended swagger definition
backend = self.backend_dir
needle = self.find_swagger(endpoint, backend)
# or look inside base swagger definition of rapydo
if needle is None:
backend = self.base_backend_dir
needle = self.find_swagger(endpoint, backend)
base_endpoint = True
python_file_dir = Path(backend, 'resources')
else:
python_file_dir = Path(backend, ENDPOINTS_CODE_DIR)
if needle is None:
log.exit('No endpoint "{}" found in current swagger definition',
endpoint)
current_dir = Path.cwd()
uri = Path(needle.get('baseuri', '/api'), endpoint)
infos += 'Endpoint path:\t{}\n'.format(uri)
swagger_dir = Path(current_dir, backend, SWAGGER_DIR,
needle.get('swagger'))
infos += 'Swagger path:\t{}/\n'.format(swagger_dir)
infos += 'Labels:\t\t{}\n'.format(", ".join(needle.get('labels')))
python_file_path = Path(current_dir, python_file_dir,
needle.get('file') + '.py')
infos += 'Python file:\t{}\n'.format(python_file_path)
python_class = needle.get('class')
infos += 'Python class:\t{}\n'.format(python_class)
log.info("Informations about '{}':\n{}", endpoint, infos)
if base_endpoint:
log.warning(
"This is a BASE endpoint of the RAPyDo framework.\n" +
"Do not modify it unless your are not a RAPyDo developer.")
with open(str(python_file_path)) as fh:
content = fh.read()
clstest = 'class {}('.format(python_class)
if clstest not in content:
log.critical("Class '{}' definition not found in python file",
python_class)
def check_file_younger_than(gitobj, filename, timestamp):
try:
commits = gitobj.blame(rev='HEAD', file=filename)
except GitCommandError as e:
log.exit("Failed 'blame' operation on {}.\n{}", filename, e)
dates = []
for commit in commits:
current_blame = gitobj.commit(rev=str(commit[0]))
dates.append(current_blame.committed_datetime)
m = max(dates)
return timestamp_from_string(timestamp) < m, timestamp, m
def compare_repository(gitobj,
branch,
online_url,
check_only=False,
path=None):
# origin = gitobj.remote()
# url = list(origin.urls).pop(0)
url = gitobj.remotes.origin.url
if online_url != url:
local_url = urlparse(url)
expected_url = urlparse(online_url)
# Remove username in the URL, if any
# i.e. [email protected] became github.com
local_netloc = local_url.netloc.split("@").pop()
expected_netloc = local_url.netloc.split("@").pop()
if local_url.scheme != expected_url.scheme:
url_match = False
elif local_netloc != expected_netloc:
url_match = False
elif local_url.path != expected_url.path:
url_match = False
else:
url_match = True
if not url_match:
if check_only:
return False
log.exit("""Unmatched local remote
Found: {}\nExpected: {}
Suggestion: remove {} and execute the init command
""".format(url, online_url, gitobj.working_dir))
if branch is None:
# No more checks, we are ok
return True
active_branch = get_active_branch(gitobj)
if active_branch is not None:
if branch != active_branch:
if check_only:
return False
log.exit("""{p}: wrong branch {ab}, expected {b}.
Suggestion:\n\ncd {wdir}; git fetch; git checkout {b}; cd -;\n""".format(
p=path, ab=active_branch, b=branch, wdir=gitobj.working_dir))
return True
def read_composer_yamls(composers):
base_files = []
all_files = []
# YAML CHECK UP
for name, composer in composers.items():
if not composer.pop('if', False):
continue
log.verbose("Composer {}", name)
mandatory = composer.pop('mandatory', False)
base = composer.pop('base', False)
try:
f = composer.get('file')
p = composer.get('path')
compose = load_yaml_file(file=f, path=p, is_optional=not mandatory)
if compose.get('services') is None or len(
compose.get('services', {})) < 1:
# if mandatory:
# log.exit("No service defined in file {}", name)
# else:
# log.verbose("No service defined in {}, skipping", name)
log.verbose("No service defined in {}, skipping", name)
continue
filepath = get_yaml_path(file=f, path=p)
all_files.append(filepath)
if base:
base_files.append(filepath)
except KeyError as e:
# if mandatory:
# log.exit(
# "Composer {}({}) is mandatory.\n{}", name, filepath, e
# )
# else:
# log.debug("Missing '{}' composer", name)
log.exit("Error loading {}: {}", filepath, e)
return all_files, base_files
def load_yaml_file(file, path, keep_order=False, is_optional=False):
"""
Import any data from a YAML file.
"""
filepath = get_yaml_path(file, path=path)
if filepath is None:
if is_optional:
log.info(
"Failed to read YAML file {}/{}: File does not exist",
path,
file,
)
else:
log.exit(
"Failed to read YAML file {}/{}: File does not exist",
path,
file,
)
return {}
with open(filepath) as fh:
try:
if keep_order:
OrderedLoader.add_constructor(
yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG,
construct_mapping)
loader = yaml.load_all(fh, OrderedLoader)
else:
loader = yaml.load_all(fh, yaml.loader.Loader)
docs = list(loader)
if len(docs) == 0:
log.exit("YAML file is empty: {}", filepath)
return docs[0]
except Exception as e:
# # IF dealing with a strange exception string (escaped)
# import codecs
# error, _ = codecs.getdecoder("unicode_escape")(str(error))
log.warning("Failed to read YAML file [{}]: {}", filepath, e)
return {}
def find_templates_override(services, templates):
# Template and vanilla builds involved in override
tbuilds = {}
vbuilds = {}
for service in services:
builder = service.get('build')
if builder is not None:
dpath = builder.get('context')
dockerfile = os.path.join(os.curdir, CONTAINERS_YAML_DIRNAME,
dpath)
dfp = DockerfileParser(dockerfile)
try:
cont = dfp.content
if cont is None:
log.warning("Dockerfile is empty?")
else:
log.verbose("Parsed dockerfile {}", dpath)
except FileNotFoundError as e:
log.exit(e)
if dfp.baseimage is None:
dfp.baseimage = 'unknown_build'
# elif dfp.baseimage.endswith(':template'):
elif dfp.baseimage.startswith('rapydo/'):
if dfp.baseimage not in templates:
log.exit("""Unable to find {} in this project
\nPlease inspect the FROM image in {}/Dockerfile
""".format(dfp.baseimage, dockerfile))
else:
vanilla_img = service.get('image')
template_img = dfp.baseimage
log.verbose("{} overrides {}", vanilla_img, template_img)
tbuilds[template_img] = templates.get(template_img)
vbuilds[vanilla_img] = template_img
return tbuilds, vbuilds
def find_templates_build(base_services):
templates = {}
from controller.dockerizing import Dock
docker = Dock()
for base_service in base_services:
template_build = base_service.get('build')
if template_build is not None:
template_name = base_service.get('name')
template_image = base_service.get('image')
if template_image is None:
log.exit(
"Template builds must have a name, missing for {}".format(
template_name))
else:
if template_image not in templates:
templates[template_image] = {}
templates[template_image]['services'] = []
templates[template_image]['path'] = template_build.get(
'context')
templates[template_image][
'timestamp'] = docker.image_attribute(template_image)
if 'service' not in templates[template_image]:
templates[template_image]['service'] = template_name
else:
templates[template_image]['service'] = name_priority(
templates[template_image]['service'],
template_name,
)
templates[template_image]['services'].append(template_name)
return templates
def command(self, command, options=None, nofailure=False):
# NOTE: debug defaults
# tmp = self.get_defaults(command)
# print("TEST", tmp, type(tmp))
# # exit(1)
compose_handler = self.get_handle()
method = getattr(compose_handler, command)
if options is None:
options = {}
if options.get('SERVICE', None) is None:
options['SERVICE'] = []
log.debug("{}'{}'", compose_log, command)
out = None
# sometimes this import stucks... importing here to avoid unnecessary waits
from docker.errors import APIError
try:
out = method(options=options)
except SystemExit as e:
# NOTE: we check the status here.
# System exit is received also when a normal command finished.
if e.code < 0:
log.warning("Invalid code returned: {}", e.code)
elif e.code > 0:
log.exit("Compose received: system.exit({})",
e.code,
error_code=e.code)
else:
log.verbose("Executed compose {} w/{}", command, options)
except (clierrors.UserError, cerrors.OperationFailedError,
BuildError) as e:
msg = "Failed command execution:\n{}".format(e)
if nofailure:
raise AttributeError(msg)
else:
log.exit(msg)
except APIError as e:
log.exit("Failed docker container:\n{}", e)
except (ProjectError, NoSuchService) as e:
log.exit(str(e))
else:
log.verbose("Executed compose {} w/{}", command, options)
return out
def command_defaults(command):
if command in ['run']:
return Compose.set_defaults(
variables=[
'COMMAND',
'T',
'e',
'entrypoint',
'user',
'label',
'publish',
'service-ports',
'name',
'workdir',
'volume',
'no-deps',
'use-aliases',
],
merge={'--rm': True},
)
else:
log.exit("No default implemented for: {}", command)
def start_containers(
self,
services,
detach=True,
scale=None,
skip_dependencies=False,
abort_on_container_exit=False,
no_recreate=False,
):
"""
Start containers (docker-compose up)
"""
if scale is None:
scale = {}
options = {
'SERVICE': services,
'--no-deps': skip_dependencies,
'--detach': detach,
'--build': None,
'--no-color': False,
'--remove-orphans': False,
'--abort-on-container-exit': abort_on_container_exit,
'--no-recreate': no_recreate,
'--force-recreate': False,
'--always-recreate-deps': False,
'--no-build': False,
'--scale': scale,
}
try:
return self.command('up', options)
except NetworkConfigChangedError as e:
log.exit("{}.\n{} ({})", e,
"Remove previously created networks and try again",
"you can use rapydo clean or docker system prune")
def update(path, gitobj):
unstaged = get_unstaged_files(gitobj)
changed = len(unstaged['changed']) > 0
untracked = len(unstaged['untracked']) > 0
if changed or untracked:
log.critical("Unable to update {} repo, you have unstaged files", path)
print_diff(gitobj, unstaged)
sys.exit(1)
for remote in gitobj.remotes:
if remote.name == 'origin':
try:
branch = gitobj.active_branch
log.info("Updating {} {} (branch {})", remote, path, branch)
remote.pull(branch)
except GitCommandError as e:
log.error("Unable to update {} repo\n{}", path, e)
except TypeError as e:
if TESTING:
log.warning("Unable to update {} repo, {}", path, e)
else:
log.exit("Unable to update {} repo, {}", path, e)
def read_configuration(default_file_path,
base_project_path,
projects_path,
submodules_path,
read_extended=True,
production=False):
"""
Read default configuration
"""
custom_configuration = load_yaml_file(file=PROJECT_CONF_FILENAME,
path=base_project_path,
keep_order=True)
# Verify custom project configuration
project = custom_configuration.get('project')
if project is None:
raise AttributeError("Missing project configuration")
variables = ['title', 'description', 'version', 'rapydo']
for key in variables:
if project.get(key) is None:
log.exit(
"Project not configured, missing key '{}' in file {}/{}",
key,
base_project_path,
PROJECT_CONF_FILENAME,
)
if default_file_path is None:
base_configuration = {}
else:
base_configuration = load_yaml_file(file=PROJECTS_DEFAULTS_FILE,
path=default_file_path,
keep_order=True)
if production:
base_prod_conf = load_yaml_file(file=PROJECTS_PROD_DEFAULTS_FILE,
path=default_file_path,
keep_order=True)
base_configuration = mix_configuration(base_configuration,
base_prod_conf)
if read_extended:
extended_project = project.get('extends')
else:
extended_project = None
if extended_project is None:
# Mix default and custom configuration
return mix_configuration(base_configuration,
custom_configuration), None, None
extends_from = project.get('extends-from', 'projects')
if extends_from == "projects":
extend_path = projects_path
elif extends_from.startswith("submodules/"):
repository_name = (extends_from.split("/")[1]).strip()
if repository_name == '':
log.exit('Invalid repository name in extends-from, name is empty')
extend_path = os.path.join(submodules_path, repository_name,
projects_path)
else:
suggest = "Expected values: 'projects' or 'submodules/${REPOSITORY_NAME}'"
log.exit("Invalid extends-from parameter: {}.\n{}", extends_from,
suggest)
extend_path = os.path.join(extend_path, extended_project)
if not os.path.exists(extend_path):
log.exit("From project not found: {}", extend_path)
extended_configuration = load_yaml_file(file=PROJECT_CONF_FILENAME,
path=extend_path,
keep_order=True)
m1 = mix_configuration(base_configuration, extended_configuration)
return mix_configuration(
m1, custom_configuration), extended_project, extend_path
def __init__(self):
super(Dock, self).__init__()
if not self.is_daemon_alive():
log.exit("Docker daemon not reachable")
