class RecipeBundleFeature(RecipeFeature):
"""Recipe feature for bundling (*e.g.* making a tar archive or a squashfs
image from any previous result of another recipe) whole or parts of the
final recipe target."""
NAME = "bundle"
FEATURED_ATTRIBUTES = {"bundle"}
SCHEMA = schema.Schema({
"bundle": {
"sdk": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"steps": schema.And([str], len),
},
str: object, # do not consider other keys
})
def bundle(self) -> None:
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["bundle"]["sdk"]),
"sdk")
action_out_subpath = os.path.join(self.recipe.out_subpath, "bundle")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Always clear out the previous image result:
if os.path.exists(action_out_path):
debug("clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
with sdk.session(action_name="bundle",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["bundle"]
["env"].items()
},
terminal=terminal,
shared_host_netns=False) as sess:
for cmd in self.recipe.config["bundle"]["steps"]:
info("{!r} bundles recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
def validate_check_config(check_config):
class PrettyReprAnd(schema.And):
def __repr__(self):
return self._error
check_name = PrettyReprAnd(
str,
lambda val: len(val) > 0,
lambda val: not any(w in val for w in string.whitespace),
error='Check name must be a nonzero length string with no whitespace')
timeout_units = ['ns', 'us', 'µs', 'ms', 's', 'm', 'h']
timeout = schema.Regex(
'^\d+(\.\d+)?({})$'.format('|'.join(timeout_units)),
error='Timeout must be a string containing an integer or float followed by a unit: {}'.format(
', '.join(timeout_units)))
check_config_schema = schema.Schema({
schema.Optional('cluster_checks'): {
check_name: {
'description': str,
'cmd': [str],
'timeout': timeout,
},
},
schema.Optional('node_checks'): {
'checks': {
check_name: {
'description': str,
'cmd': [str],
'timeout': timeout,
schema.Optional('roles'): schema.Schema(
['master', 'agent'],
error='roles must be a list containing master or agent or both',
),
},
},
schema.Optional('prestart'): [check_name],
schema.Optional('poststart'): [check_name],
},
})
check_config_obj = validate_json_dictionary(check_config)
try:
check_config_schema.validate(check_config_obj)
except schema.SchemaError as exc:
raise AssertionError(str(exc).replace('\n', ' ')) from exc
if 'node_checks' in check_config_obj.keys():
node_checks = check_config_obj['node_checks']
assert any(k in node_checks.keys() for k in ['prestart', 'poststart']), (
'At least one of prestart or poststart must be defined in node_checks')
assert node_checks['checks'].keys() == set(
node_checks.get('prestart', []) + node_checks.get('poststart', [])), (
'All node checks must be referenced in either prestart or poststart, or both')
return check_config_obj
def instrumented_recipes() -> Dict[str, InstrumentationLevel]:
"""Probes for existence of an "instrumentation.toml" drop-in file at the
root of the `repo` source tree, parses this file and return a
:py:data:`dict` of all the recipe identifiers (as keys) requested to be
instrumented with their instrumentation level as value.
:raise InstrumentationSpecificationError: in case of an incoherence in the
``instrumentation.toml`` file
"""
instru_filepath = os.path.join(repo_root_path(), "instrumentation.toml")
if not os.path.exists(instru_filepath):
return dict()
try:
with open(instru_filepath, "r") as instru_fp:
instru: Dict[str, Any] = toml.load(instru_fp)
except:
raise InstrumentationSpecificationError(line(
"""Cannot open or parse as TOML the "instrumentation.toml" file
placed at the root of the repo source tree."""))
instru_file_schema = schema.Schema({
schema.Optional(level.name.lower(), default=[]): [
schema.Regex(RECIPE_IDENTIFIER_RE.pattern)
] for level in InstrumentationLevel
})
try:
instru = instru_file_schema.validate(instru)
except schema.SchemaError as exc:
raise InstrumentationSpecificationError(line(
"""The "instrumentation.toml" file has an unexpected data
structure. Reason: {!s}""").format(exc))
for level in InstrumentationLevel:
for recipe in instru[level.name.lower()]:
recipe_config_path = os.path.join(repo_root_path(), "products",
recipe, "recipe.toml")
if not os.path.exists(recipe_config_path):
raise InstrumentationSpecificationError(line(
"""The recipe {!r} is not a valid recipe or has no
configuration file in the products
folder.""").format(recipe))
recipes: Dict[str, InstrumentationLevel] = dict()
for level in InstrumentationLevel:
for recipe_id in instru[level.name.lower()]:
if recipe_id in recipes:
raise InstrumentationSpecificationError(line(
"""The recipe {!r} is specified more than once in the
"instrumentation.toml" file.""").format(recipe_id))
recipes[recipe_id] = level
return recipes
def __init__(self, verbose=False):
super(Producer, self).__init__(verbose=verbose)
self.__schema__ = schema.Schema({
'name':
schema.Regex(r'^[a-zA-Z][0-9a-zA-Z_]+$',
error='Invalid name: '
'Expected only alphanumeric characters or "_"'
'and should start with a letter.'),
'version':
ProducerVersion(verbose=verbose)
})
class MD5Checksum(SchemaComponent):
"""A 'validatable' component in a ``schema`` validation which only accepts syntactically valid *MD5* hash.
Args:
verbose (bool): Optional. Default to ``False``. Toggle exhaustive schema offence reporting.
"""
__schema__ = schema.Regex(
r'^[a-f0-9]{32}$',
error='Invalid file checksum provided: Expected a valid MD5 hash.')
class CuckooSandboxFileAnalysisReport(CortexAnalyzerReport):
""" Represents a Cortex CuckooSandbox_File_Analysis_Inet_1_2 analysis JSON
report. """
report_schema = schema.Schema({
"summary": {
"taxonomies": [
schema.Or(
{
"level": schema.Or("info", "malicious", "safe"),
"namespace": "Cuckoo",
"predicate": "Malscore",
"value": schema.Regex(r'^[0-9\./]*$')
}, {
"level": schema.Or("info", "malicious", "safe"),
"namespace": "Cuckoo",
"predicate": "Malfamily",
"value": str,
})
]
},
"full":
schema.Schema({
"signatures": schema.Schema([str]),
},
ignore_extra_keys=True),
"success":
bool,
"artifacts":
CortexAnalyzerReport.report_schema_artifacts,
"operations": []
})
def __init__(self, report):
super().__init__(report)
self.taxonomies = report.get("summary", {}).get("taxonomies", [{}])
@property
def signatures(self):
""" Matched Cuckoo signatures. """
return self.report.get('full', {}).get('Signatures', None)
@property
def malscore(self):
""" Malscore n of 10 (might be bigger). """
for tax in self.taxonomies:
if tax.get('predicate') == 'Malscore':
return float(tax['value'])
return -1
def meta_schema(self) -> schema.Schema:
"""The schema for which the recipe configuration file (``recipe.toml``)
needs to comply for common part (*i.e.* the part of the configuration
common to all the recipes)"""
return schema.Schema({
"features": [
schema.Or(*(featclass.NAME for featclass in
features.RecipeFeature.__subclasses__())),
],
# All recipes have SDK except for SDK recipes
schema.Optional("sdk"): schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
# other keys must be dict and will be validated by recipe features
# validation methods
str: dict,
})
def test_schema():
# 检查数字
print '----------------------int'
print schema.Schema(int).validate(123)
print schema.Schema(int).is_valid(123)
# 检查字符串
print '----------------------str'
# Regex 没有 is_valid 方法
print schema.Regex(r'^foo').validate('foobar')
print schema.Schema(lambda n: "foo" in n).is_valid('foobar')
print 'False:%s ' % schema.Schema(lambda n: "foo" in n).is_valid('fobar')
# 检查字典
print '----------------------dict'
rules = {
'name': schema.And(str, len),
'age': schema.And(schema.Use(int), lambda n: 18 <= n <= 99),
schema.Optional('gender'): schema.And(str, schema.Use(str.lower), lambda s: s in ('squid', 'kid'))}
data = {'name': 'Sue', 'age': '28', 'gender': 'Squid'}
print schema.Schema(rules).validate(data)
print schema.Schema(rules).is_valid(data)
print '----------------------list-dict'
rules = [{
'name': schema.And(str, len),
'age': schema.And(schema.Use(int), lambda n: 18 <= n <= 99),
schema.Optional('gender'): schema.And(str, schema.Use(str.lower), lambda s: s in ('squid', 'kid'))}]
data = [{'name': 'Sue', 'age': '28', 'gender': 'Squid'},
{'name': 'Sam', 'age': '42'},
{'name': 'Sacha', 'age': '20', 'gender': 'KID'}]
print schema.Schema(rules).validate(data)
print schema.Schema(rules).is_valid(data)
class VirusTotalQueryReport(CortexAnalyzerReport):
""" Represents a Cortex VirusTotal_GetReport_3_0 analysis JSON report. """
report_schema = schema.Schema({
"summary": {
"taxonomies": [{
"level": schema.Or("info", "malicious", "safe"),
"namespace": "VT",
"predicate": str,
"value": schema.Regex(r'^[0-9/]*$')
}]
},
"full": {
"response_code": int,
"resource": str,
"verbose_msg": str
},
"success": bool,
"artifacts": CortexAnalyzerReport.report_schema_artifacts,
"operations": []
})
def __init__(self, unvalidated_report):
super().__init__(unvalidated_report)
self.taxonomies_vt = self.get_element_from_list_of_dicts(
self.report.get('summary', {}).get('taxonomies'), 'namespace',
'VT', {})
@property
def n_of_all(self):
""" n of all Virusscanners at VirusTotal have rated this file as
malicious. """
return int(self.taxonomies_vt.get('value', '-1/0').split('/')[0])
@property
def level(self):
""" safe, suspicious, malicious """
return self.taxonomies_vt.get('level', None)
return schema.And(schema.Use(type_), sub_schema)
def _as_d(sub_schema): return _coerce(dict, sub_schema)
def _as_l(sub_schema): return _coerce(list, sub_schema)
_FILE_SCHEMA = schema.Schema(_as_d(
{
"audit-log": _as_l([str]),
"key-custodians": _as_d({
schema.Optional(str): _as_d({
"pwdkm": str,
}),
}),
schema.Optional(schema.Regex("^(?!meta).*$")): _as_d({
# allow string names for security domains,
# but meta is reserved
"meta": _as_d({
"owners": _as_d({str: str}),
"public-key": str,
}),
schema.Optional(schema.Regex("secret-.*")): str,
}),
}))
# NOTE: this is a public class since it must be passed in
@attr.s(frozen=True)
class Creds(object):
"Stores credentials used to open a KeyFile"
"""
Copyright (c) 2020 Nathan Telles
This Source Code Form is subject to the terms of the Mozilla Public
License, v. 2.0. If a copy of the MPL was not distributed with this
file, You can obtain one at http://mozilla.org/MPL/2.0/.
"""
from typing import Any, Dict, ValuesView
import schema as sch
import yaml
from .errors import ConfigIOError, InvalidConfig
SAFE_STR_REG = sch.Regex(r'^[\w\d]+$')
# Regex for validating ip addressses
IP_REG = sch.Regex(r'^(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}'
r'(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)$')
TCP_SCHEMA = sch.Schema({
'type':
'tcp_server',
'address':
IP_REG,
'port':
sch.And(sch.Use(int), lambda n: 0 <= n <= 65535),
sch.Optional('line_ending', default="\n"):
str,
sch.Optional('default_response', default=''):
Sync the docker daemon clock up to current clock.
'''
with self.logger.info('sync_clock'):
cmd = 'date +%s -s @' + str(int(time.time()))
self.executor.docker.run(
CENTOS, cmd, rm=None, privileged=None
).interactive()
_EMPTY = attr.make_class('Empty', [])() # represents a value that should be dropped from the YAML
_DEPENDS_ON_ALL = '*' # special value for depends_on for connecting to everything else
# (mainly for jupyter notebook or similar debugging tool)
SK_CENTRAL_REMOTE = 'USER@GITLAB_HOSTNAME:REPO'
_ENV_SCHEMA = schema.Or(
[schema.Regex('[_a-zA-Z][_a-zA-Z0-9]*(=[^=]+)?')],
{schema.Regex('[_a-zA-Z][_a-zA-Z0-9]*'): schema.Or(str, None)})
_PORT_SCHEMA = schema.Regex(r'\d+(\:\d+)?')
'''
Environment variables in docker-compose:
https://docs.docker.com/compose/compose-file/#/environment
environment:
RACK_ENV: development
SHOW: 'true'
SESSION_SECRET:
environment:
- RACK_ENV=development
class Config(object):
CONFIG = 'config'
SECTION_CORE = 'core'
SECTION_CORE_LOGLEVEL = 'loglevel'
SECTION_CORE_LOGLEVEL_SCHEMA = schema.And(schema.Use(str.lower), lambda l: l in ('info', 'debug', 'warning', 'error'))
SECTION_CORE_REMOTE = 'remote'
# backward compatibility
SECTION_CORE_CLOUD = 'cloud'
SECTION_CORE_CLOUD_SCHEMA = schema.And(schema.Use(str.lower), lambda c: c in ('aws', 'gcp', 'local', ''))
SECTION_CORE_STORAGEPATH = 'storagepath'
SECTION_CORE_SCHEMA = {
schema.Optional(SECTION_CORE_LOGLEVEL, default='info'): schema.And(str, schema.Use(str.lower), SECTION_CORE_LOGLEVEL_SCHEMA),
schema.Optional(SECTION_CORE_REMOTE, default=''): schema.And(str, schema.Use(str.lower)),
# backward compatibility
schema.Optional(SECTION_CORE_CLOUD, default=''): SECTION_CORE_CLOUD_SCHEMA,
schema.Optional(SECTION_CORE_STORAGEPATH, default=''): str,
}
# backward compatibility
SECTION_AWS = 'aws'
SECTION_AWS_STORAGEPATH = 'storagepath'
SECTION_AWS_CREDENTIALPATH = 'credentialpath'
SECTION_AWS_REGION = 'region'
SECTION_AWS_PROFILE = 'profile'
SECTION_AWS_SCHEMA = {
SECTION_AWS_STORAGEPATH: str,
schema.Optional(SECTION_AWS_REGION): str,
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
}
# backward compatibility
SECTION_GCP = 'gcp'
SECTION_GCP_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_GCP_PROJECTNAME = 'projectname'
SECTION_GCP_SCHEMA = {
SECTION_GCP_STORAGEPATH: str,
schema.Optional(SECTION_GCP_PROJECTNAME): str,
}
# backward compatibility
SECTION_LOCAL = 'local'
SECTION_LOCAL_STORAGEPATH = SECTION_AWS_STORAGEPATH
SECTION_LOCAL_SCHEMA = {
SECTION_LOCAL_STORAGEPATH: str,
}
SECTION_REMOTE_REGEX = r'^\s*remote\s*"(.*)"\s*$'
SECTION_REMOTE_FMT = 'remote "{}"'
SECTION_REMOTE_URL = 'url'
SECTION_REMOTE_SCHEMA = {
SECTION_REMOTE_URL: supported_url,
schema.Optional(SECTION_AWS_REGION): str,
schema.Optional(SECTION_AWS_PROFILE, default='default'): str,
schema.Optional(SECTION_AWS_CREDENTIALPATH, default = ''): str,
schema.Optional(SECTION_GCP_PROJECTNAME): str,
}
SCHEMA = {
schema.Optional(SECTION_CORE, default={}): SECTION_CORE_SCHEMA,
schema.Optional(schema.Regex(SECTION_REMOTE_REGEX)): SECTION_REMOTE_SCHEMA,
# backward compatibility
schema.Optional(SECTION_AWS, default={}): SECTION_AWS_SCHEMA,
schema.Optional(SECTION_GCP, default={}): SECTION_GCP_SCHEMA,
schema.Optional(SECTION_LOCAL, default={}): SECTION_LOCAL_SCHEMA,
}
def __init__(self, dvc_dir):
self.dvc_dir = os.path.abspath(os.path.realpath(dvc_dir))
self.config_file = os.path.join(dvc_dir, self.CONFIG)
try:
self._config = configobj.ConfigObj(self.config_file)
# NOTE: schema doesn't support ConfigObj.Section validation, so we
# need to convert our config to dict before passing it to schema.
self._config = self._lower(self._config)
self._config = schema.Schema(self.SCHEMA).validate(self._config)
# NOTE: now converting back to ConfigObj
self._config = configobj.ConfigObj(self._config, write_empty_values=True)
self._config.filename = self.config_file
except Exception as ex:
raise ConfigError(ex)
@staticmethod
def _lower(config):
new_config = {}
for s_key, s_value in config.items():
new_s = {}
for key, value in s_value.items():
new_s[key.lower()] = value
new_config[s_key.lower()] = new_s
return new_config
@staticmethod
def init(dvc_dir):
config_file = os.path.join(dvc_dir, Config.CONFIG)
open(config_file, 'w+').close()
return Config(dvc_dir)
class GitScm(Scm):
SCHEMA = schema.Schema({
'scm': 'git',
'url': str,
schema.Optional('dir'): str,
schema.Optional('if'): str,
schema.Optional('branch'): str,
schema.Optional('tag'): str,
schema.Optional('commit'): str,
schema.Optional('rev'): str,
schema.Optional(schema.Regex('^remote-.*')): str,
schema.Optional('sslVerify'): bool,
})
REMOTE_PREFIX = "remote-"
def __init__(self, spec, overrides=[], secureSSL=None):
super().__init__(spec, overrides)
self.__url = spec["url"]
self.__branch = None
self.__tag = None
self.__commit = None
self.__remotes = {}
if "rev" in spec:
rev = spec["rev"]
if rev.startswith("refs/heads/"):
self.__branch = rev[11:]
elif rev.startswith("refs/tags/"):
self.__tag = rev[10:]
elif len(rev) == 40:
self.__commit = rev
else:
raise ParseError("Invalid rev format: " + rev)
self.__branch = spec.get("branch", self.__branch)
self.__tag = spec.get("tag", self.__tag)
self.__commit = spec.get("commit", self.__commit)
if self.__commit:
# validate commit
if re.match("^[0-9a-f]{40}$", self.__commit) is None:
raise ParseError("Invalid commit id: " + str(self.__commit))
elif not self.__branch and not self.__tag:
# nothing secified at all -> master branch
self.__branch = "master"
self.__dir = spec.get("dir", ".")
# convert remotes into separate dictionary
for key, val in spec.items():
if key.startswith(GitScm.REMOTE_PREFIX):
stripped_key = key[len(GitScm.REMOTE_PREFIX):] # remove prefix
if stripped_key == "origin":
raise ParseError("Invalid remote name: " + stripped_key)
self.__remotes.update({stripped_key: val})
self.__sslVerify = spec.get('sslVerify', secureSSL)
def getProperties(self):
properties = super().getProperties()
properties.update({
'scm':
'git',
'url':
self.__url,
'branch':
self.__branch,
'tag':
self.__tag,
'commit':
self.__commit,
'dir':
self.__dir,
'rev': (self.__commit if self.__commit else
(("refs/tags/" + self.__tag) if self.__tag else
("refs/heads/" + self.__branch))),
'sslVerify':
self.__sslVerify,
})
for key, val in self.__remotes.items():
properties.update({GitScm.REMOTE_PREFIX + key: val})
return properties
def asScript(self):
remotes_array = [
"# create an array of all remotes for this repository",
"declare -A BOB_GIT_REMOTES=( [origin]={URL} )".format(
URL=quote(self.__url)),
]
# add additional remotes to array
for name, url in self.__remotes.items():
remotes_array.append("BOB_GIT_REMOTES[{NAME}]={URL}".format(
NAME=quote(name), URL=quote(url)))
# Assemble generic header including the remote handling
header = super().asScript()
if not self.__sslVerify:
header += "\nexport GIT_SSL_NO_VERIFY=true"
header += "\n" + dedent("""\
if [ ! -d {DIR}/.git ] ; then
git init {DIR}
fi
cd {DIR}
(
{REMOTES_ARRAY}
# remove remotes from array that are already known to Git
while read -r REMOTE_NAME ; do
# check for empty variable in case no remote at all is specified
if [ -z "$REMOTE_NAME" ]; then
continue
fi
# check if existing remote is configured
if [ "${{BOB_GIT_REMOTES[$REMOTE_NAME]+_}}" ]; then
# check if URL has changed
if [ ! "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}" == "$(git ls-remote --get-url $REMOTE_NAME)" ]; then
git remote set-url "$REMOTE_NAME" "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}"
fi
# it is configured, therefore no need to keep in list
unset "BOB_GIT_REMOTES[$REMOTE_NAME]"
fi
done <<< "$(git remote)"
# add all remaining remotes in the array to the repository
for REMOTE_NAME in "${{!BOB_GIT_REMOTES[@]}}" ; do
git remote add "$REMOTE_NAME" "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}"
done
)""").format(REMOTES_ARRAY="\n ".join(remotes_array),
DIR=quote(self.__dir))
if self.__tag or self.__commit:
refSpec = "'+refs/heads/*:refs/remotes/origin/*' "
if self.__tag:
refSpec += quote("refs/tags/{0}:refs/tags/{0}".format(
self.__tag))
return dedent("""\
{HEADER}
# checkout only if HEAD is invalid
if ! git rev-parse --verify -q HEAD >/dev/null ; then
git fetch origin {REFSPEC}
git checkout -q {REF}
fi
""").format(HEADER=header,
REF=self.__commit if self.__commit else "tags/" +
quote(self.__tag),
REFSPEC=refSpec)
else:
return dedent("""\
{HEADER}
git fetch -p origin
if ! git rev-parse --verify -q HEAD >/dev/null ; then
# checkout only if HEAD is invalid
git checkout -b {BRANCH} remotes/origin/{BRANCH}
elif [[ $(git rev-parse --abbrev-ref HEAD) == {BRANCH} ]] ; then
# pull only if on original branch
git merge --ff-only refs/remotes/origin/{BRANCH}
else
echo Warning: not updating {DIR} because branch was changed manually... >&2
fi
""").format(HEADER=header,
BRANCH=quote(self.__branch),
DIR=quote(self.__dir))
def asDigestScript(self):
"""Return forward compatible stable string describing this git module.
The format is "url rev-spec dir" where rev-spec depends on the given reference.
"""
if self.__commit:
return self.__commit + " " + self.__dir
elif self.__tag:
return self.__url + " refs/tags/" + self.__tag + " " + self.__dir
else:
return self.__url + " refs/heads/" + self.__branch + " " + self.__dir
def asJenkins(self, workPath, credentials, options):
scm = ElementTree.Element("scm",
attrib={
"class": "hudson.plugins.git.GitSCM",
"plugin": "[email protected]",
})
ElementTree.SubElement(scm, "configVersion").text = "2"
userconfigs = ElementTree.SubElement(
ElementTree.SubElement(scm, "userRemoteConfigs"),
"hudson.plugins.git.UserRemoteConfig")
url = ElementTree.SubElement(userconfigs, "url")
url.text = self.__url
if credentials:
credentialsId = ElementTree.SubElement(userconfigs,
"credentialsId")
credentialsId.text = credentials
branch = ElementTree.SubElement(
ElementTree.SubElement(ElementTree.SubElement(scm, "branches"),
"hudson.plugins.git.BranchSpec"), "name")
if self.__commit:
branch.text = self.__commit
elif self.__tag:
branch.text = "refs/tags/" + self.__tag
else:
branch.text = "refs/heads/" + self.__branch
ElementTree.SubElement(
scm, "doGenerateSubmoduleConfigurations").text = "false"
ElementTree.SubElement(scm, "submoduleCfg", attrib={"class": "list"})
extensions = ElementTree.SubElement(scm, "extensions")
ElementTree.SubElement(
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.RelativeTargetDirectory"),
"relativeTargetDir").text = os.path.normpath(
os.path.join(workPath, self.__dir))
# remove untracked files and stale branches
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CleanCheckout")
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.PruneStaleBranch")
# set git clone options
shallow = options.get("scm.git.shallow")
timeout = options.get("scm.git.timeout")
if shallow is not None or timeout is not None:
co = ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CloneOption")
if shallow is not None:
try:
shallow = int(shallow)
if shallow < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.shallow' option: " +
str(shallow))
if shallow > 0:
ElementTree.SubElement(co, "shallow").text = "true"
ElementTree.SubElement(co, "noTags").text = "false"
ElementTree.SubElement(co, "reference").text = ""
ElementTree.SubElement(co, "depth").text = str(shallow)
ElementTree.SubElement(co, "honorRefspec").text = "false"
if timeout is not None:
try:
timeout = int(timeout)
if timeout < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.timeout' option: " +
str(shallow))
if timeout > 0:
ElementTree.SubElement(co, "timeout").text = str(timeout)
if isTrue(options.get("scm.ignore-hooks", "0")):
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.IgnoreNotifyCommit")
return scm
def getDirectory(self):
return self.__dir
def isDeterministic(self):
return bool(self.__tag) or bool(self.__commit)
def hasJenkinsPlugin(self):
return True
def callGit(self, workspacePath, *args):
cmdLine = ['git']
cmdLine.extend(args)
cwd = os.path.join(workspacePath, self.__dir)
try:
output = subprocess.check_output(cmdLine,
cwd=cwd,
universal_newlines=True,
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
raise BuildError(
"git error:\n Directory: '{}'\n Command: '{}'\n'{}'".format(
cwd, " ".join(cmdLine), e.output.rstrip()))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
return output.strip()
def status(self, workspacePath):
status = ScmStatus()
try:
onCorrectBranch = False
onTag = False
output = self.callGit(workspacePath, 'ls-remote', '--get-url')
if output != self.__url:
status.add(
ScmTaint.switched,
"> URL: configured: '{}', actual: '{}'".format(
self.__url, output))
if self.__commit:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD')
if output != self.__commit:
status.add(
ScmTaint.switched,
"> commit: configured: '{}', actual: '{}'".format(
self.__commit, output))
elif self.__tag:
output = self.callGit(workspacePath, 'tag', '--points-at',
'HEAD').splitlines()
if self.__tag not in output:
actual = ("'" + ", ".join(output) +
"'") if output else "not on any tag"
status.add(
ScmTaint.switched,
"> tag: configured: '{}', actual: {}".format(
self.__tag, actual))
# Need to check if the tag still exists. Otherwise the "git
# log" command at the end will trip.
try:
self.callGit(workspacePath, 'rev-parse',
'tags/' + self.__tag)
onTag = True
except BuildError:
pass
elif self.__branch:
output = self.callGit(workspacePath, 'rev-parse',
'--abbrev-ref', 'HEAD')
if output != self.__branch:
status.add(
ScmTaint.switched,
"> branch: configured: '{}', actual: '{}'".format(
self.__branch, output))
else:
output = self.callGit(
workspacePath, 'log', '--oneline',
'refs/remotes/origin/' + self.__branch + '..HEAD')
if output:
status.add(
ScmTaint.unpushed_main,
joinLines(
"> unpushed commits on {}:".format(
self.__branch), indent(output, ' ')))
onCorrectBranch = True
# Check for modifications wrt. checked out commit
output = self.callGit(workspacePath, 'status', '--porcelain')
if output:
status.add(ScmTaint.modified,
joinLines("> modified:", indent(output, ' ')))
# The following shows all unpushed commits reachable by any ref
# (local branches, stash, detached HEAD, etc).
# Exclude HEAD if the configured branch is checked out to not
# double-count them. Does not mark the SCM as dirty. Exclude the
# configured tag too if it is checked out. Otherwise the tag would
# count as unpushed if it is not on a remote branch.
what = ['--all', '--not', '--remotes']
if onCorrectBranch: what.append('HEAD')
if onTag: what.append("tags/" + self.__tag)
output = self.callGit(workspacePath, 'log', '--oneline',
'--decorate', *what)
if output:
status.add(
ScmTaint.unpushed_local,
joinLines("> unpushed local commits:",
indent(output, ' ')))
except BuildError as e:
status.add(ScmTaint.error, e.slogan)
return status
def getAuditSpec(self):
return ("git", self.__dir)
def hasLiveBuildId(self):
return True
async def predictLiveBuildId(self, step):
if self.__commit:
return bytes.fromhex(self.__commit)
with stepAction(step, "LS-REMOTE", self.__url, (INFO, TRACE)) as a:
if self.__tag:
# Annotated tags are objects themselves. We need the commit object!
refs = [
"refs/tags/" + self.__tag + '^{}',
"refs/tags/" + self.__tag
]
else:
refs = ["refs/heads/" + self.__branch]
cmdLine = ['git', 'ls-remote', self.__url] + refs
try:
proc = await asyncio.create_subprocess_exec(
*cmdLine,
stdout=asyncio.subprocess.PIPE,
stderr=subprocess.DEVNULL)
try:
stdout, stderr = await proc.communicate()
rc = await proc.wait()
except concurrent.futures.CancelledError:
proc.terminate()
rc = await proc.wait()
if rc != 0:
a.fail("exit {}".format(rc), WARNING)
return None
output = stdout.decode(
locale.getpreferredencoding(False)).strip()
except (subprocess.CalledProcessError, OSError):
a.fail("error")
return None
# have we found anything at all?
if not output:
a.fail("unknown", WARNING)
return None
# See if we got one of our intended refs. Git is generating lines with
# the following format:
#
# <sha1>\t<refname>
#
# Put the output into a dict with the refname as key. Be extra careful
# and strip out lines not matching this pattern.
output = {
commitAndRef[1].strip(): bytes.fromhex(commitAndRef[0].strip())
for commitAndRef in (line.split('\t')
for line in output.split('\n'))
if len(commitAndRef) == 2
}
for ref in refs:
if ref in output: return output[ref]
# uhh, should not happen...
a.fail("unknown", WARNING)
return None
def calcLiveBuildId(self, workspacePath):
if self.__commit:
return bytes.fromhex(self.__commit)
else:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD').strip()
return bytes.fromhex(output)
def getLiveBuildIdSpec(self, workspacePath):
if self.__commit:
return "=" + self.__commit
else:
return "g" + os.path.join(workspacePath, self.__dir)
@staticmethod
def processLiveBuildIdSpec(dir):
try:
return subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=dir,
universal_newlines=True).strip()
except subprocess.CalledProcessError as e:
raise BuildError("Git audit failed: " + str(e))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
# File Name: demo.py
# Description:
"""
import schema
# 检查数字
print '----------------------int'
print schema.Schema(int).validate(123)
print schema.Schema(int).is_valid(123)
# 检查字符串
print '----------------------str'
# Regex 没有 is_valid 方法
print schema.Regex(r'^foo').validate('foobar')
print schema.Schema(lambda n: "foo" in n).is_valid('foobar')
print 'False:%s ' % schema.Schema(lambda n: "foo" in n).is_valid('fobar')
# 检查字典
print '----------------------dict'
rules = {
'name': schema.And(str, len),
'age': schema.And(schema.Use(int), lambda n: 18 <= n <= 99),
schema.Optional('gender'): schema.And(str, schema.Use(str.lower),lambda s: s in ('squid', 'kid'))}
data = {'name': 'Sue', 'age': '28', 'gender': 'Squid'}
print schema.Schema(rules).validate(data)
print schema.Schema(rules).is_valid(data)
assert url_obj.scheme
if schemes is not None:
assert url_obj.scheme in schemes
assert url_obj.host
return url_obj
def parse_valid_web_url(url):
return parse_valid_url(url, schemes=('http', 'https'))
_PROJECT_SCHEMA = schema.Schema(
{
'name': str,
schema.Optional(schema.Regex('\w+_url')): parse_valid_url,
'repo_url': parse_valid_web_url,
'desc': str,
'tags': tuple
},
ignore_extra_keys=False)
class ApatiteError(Exception):
pass
class DuplicateProjectError(ApatiteError):
pass
class RecipeConfigureFeature(RecipeFeature):
"""Recipe feature for configuring root trees built during a recipe root
feature"""
NAME = "configure"
FEATURED_ATTRIBUTES = {"configure"}
SCHEMA = schema.Schema({
"configure": {
"sdk": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"root": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
"steps": schema.And([str], len),
},
str: object, # do not consider other keys
})
def configure(self) -> None:
"""TODO"""
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["configure"]["sdk"]),
"sdk")
# the recipe to configure (i.e. what is the recipe for which we need to
# get image-resulting root tree on which running the configuration
# steps?):
recipe_to_configure = recipe.Recipe(
self.replace_placeholders(self.recipe.config["configure"]["root"]))
# check that image action for the recipe designated by "root" has well
# been run:
image_out_subpath = os.path.join(recipe_to_configure.out_subpath,
"image")
image_out_path = os.path.join(repo_root_path(), image_out_subpath)
if not os.path.exists(os.path.join(image_out_path, "root")):
raise RecipeActionError(
line("""
Could not process a configure action step if the image action
step from the recipe designated by "configure.root" has not
been run before.
"""))
action_out_subpath = os.path.join(self.recipe.out_subpath, "configure")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Always clear out the previous configure result:
if os.path.exists(action_out_path):
debug("clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
# Retrieve the result of the image action to work on it for this
# configuration step:
with ElevatedPrivileges():
debug(
line("""copying resulting root of the image action step for the
proper recipe ({!r}) into the working environment for the
configure action step...""".format(
recipe_to_configure.identifier)))
shutil.copytree(os.path.join(image_out_path, "root"),
os.path.join(action_out_path, "root"),
symlinks=True)
with sdk.session(action_name="configure",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["configure"]
["env"].items()
},
terminal=terminal,
shared_host_netns=False) as sess:
for cmd in self.recipe.config["configure"]["steps"]:
info("{!r} configures recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
class RecipeSdkFeature(RecipeFeature):
"""Recipe feature for creating and proposing SDK objects to other
features."""
NAME = "sdk"
FEATURED_ATTRIBUTES = {"bootstrap", "run", "sdk"}
SCHEMA = schema.Schema({
"runtime": {
"additional_capabilities": [str],
"additional_device_bindings": [str],
"cwd": schema.And(str, len),
"prelude_commands": [str],
"postlude_commands": [str],
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"writable_assets_dirs_at_build": [schema.And(str, len)],
},
"bootstrap": {
"rootfs_archive": str,
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"steps": [str],
},
str: object, # do not consider other keys
})
def bootstrap(self) -> None:
self.sdk.bootstrap(
rootfs_archive=self.replace_placeholders(
self.recipe.config["bootstrap"]["rootfs_archive"],
sdk_context=False),
steps=[
self.replace_placeholders(step, sdk_context=True)
for step in self.recipe.config["bootstrap"]["steps"]
],
env={
key: self.replace_placeholders(value, sdk_context=True)
for key, value in self.recipe.config["bootstrap"]
["env"].items()
},
)
def run(self, command: Optional[str] = None) -> None:
self.sdk.interactive_run(
recipe=self.recipe,
command=command,
env={
key: self.replace_placeholders(value, sdk_context=True)
for key, value in self.recipe.config["runtime"]["env"].items()
},
)
@property
def sdk(self) -> Sdk:
return Sdk(
recipe=self.recipe,
cwd=self.replace_placeholders(self.recipe.config["runtime"]["cwd"],
sdk_context=True),
env={
key: self.replace_placeholders(value, sdk_context=True)
for key, value in self.recipe.config["runtime"]["env"].items()
},
additional_capabilities=(
self.recipe.config["runtime"]["additional_capabilities"]),
additional_device_bindings=(
self.recipe.config["runtime"]["additional_device_bindings"]),
prelude_commands=[
self.replace_placeholders(cmd, sdk_context=True)
for cmd in self.recipe.config["runtime"]["prelude_commands"]
],
postlude_commands=[
self.replace_placeholders(cmd, sdk_context=True)
for cmd in self.recipe.config["runtime"]["postlude_commands"]
],
writable_assets_dirs_at_build=(self.recipe.config["runtime"]
["writable_assets_dirs_at_build"]))
class GitScm(Scm):
SCHEMA = schema.Schema({
'scm':
'git',
'url':
str,
schema.Optional('dir'):
str,
schema.Optional('if'):
schema.Or(str, IfExpression),
schema.Optional('branch'):
str,
schema.Optional('tag'):
str,
schema.Optional('commit'):
str,
schema.Optional('rev'):
str,
schema.Optional(schema.Regex('^remote-.*')):
str,
schema.Optional('sslVerify'):
bool,
schema.Optional('singleBranch'):
bool,
schema.Optional('shallow'):
schema.Or(int, str),
schema.Optional('submodules'):
schema.Or(bool, [str]),
schema.Optional('recurseSubmodules'):
bool,
schema.Optional('shallowSubmodules'):
bool,
})
REMOTE_PREFIX = "remote-"
def __init__(self, spec, overrides=[], secureSSL=None, stripUser=None):
super().__init__(spec, overrides)
self.__url = spec["url"]
self.__branch = None
self.__tag = None
self.__commit = None
self.__remotes = {}
if "rev" in spec:
rev = spec["rev"]
if rev.startswith("refs/heads/"):
self.__branch = rev[11:]
elif rev.startswith("refs/tags/"):
self.__tag = rev[10:]
elif len(rev) == 40:
self.__commit = rev
else:
raise ParseError("Invalid rev format: " + rev)
self.__branch = spec.get("branch", self.__branch)
self.__tag = spec.get("tag", self.__tag)
self.__commit = spec.get("commit", self.__commit)
if self.__commit:
# validate commit
if re.match("^[0-9a-f]{40}$", self.__commit) is None:
raise ParseError("Invalid commit id: " + str(self.__commit))
elif not self.__branch and not self.__tag:
# nothing secified at all -> master branch
self.__branch = "master"
self.__dir = spec.get("dir", ".")
# convert remotes into separate dictionary
for key, val in spec.items():
if key.startswith(GitScm.REMOTE_PREFIX):
stripped_key = key[len(GitScm.REMOTE_PREFIX):] # remove prefix
if stripped_key == "origin":
raise ParseError("Invalid remote name: " + stripped_key)
self.__remotes.update({stripped_key: val})
self.__sslVerify = spec.get('sslVerify', secureSSL)
self.__singleBranch = spec.get('singleBranch')
self.__shallow = spec.get('shallow')
self.__submodules = spec.get('submodules', False)
self.__recurseSubmodules = spec.get('recurseSubmodules', False)
self.__shallowSubmodules = spec.get('shallowSubmodules', True)
self.__stripUser = stripUser
def getProperties(self, isJenkins):
properties = super().getProperties(isJenkins)
properties.update({
'scm':
'git',
'url':
self.__url,
'branch':
self.__branch,
'tag':
self.__tag,
'commit':
self.__commit,
'dir':
self.__dir,
'rev': (self.__commit if self.__commit else
(("refs/tags/" + self.__tag) if self.__tag else
("refs/heads/" + self.__branch))),
'sslVerify':
self.__sslVerify,
'singleBranch':
self.__singleBranch,
'shallow':
self.__shallow,
'submodules':
self.__submodules,
'recurseSubmodules':
self.__recurseSubmodules,
'shallowSubmodules':
self.__shallowSubmodules,
})
for key, val in self.__remotes.items():
properties.update({GitScm.REMOTE_PREFIX + key: val})
return properties
async def invoke(self, invoker, switch=False):
# make sure the git directory exists
if not os.path.isdir(invoker.joinPath(self.__dir, ".git")):
await invoker.checkCommand(["git", "init", self.__dir])
# Shallow implies singleBranch
if self.__singleBranch is None:
singleBranch = self.__shallow is not None
else:
singleBranch = self.__singleBranch
singleBranch = singleBranch and (self.__branch is not None)
# setup and update remotes
remotes = {"origin": self.__url}
remotes.update(self.__remotes)
existingRemotes = await invoker.checkOutputCommand(["git", "remote"],
cwd=self.__dir)
for remote in existingRemotes.split("\n"):
if remote in remotes:
cfgUrl = remotes[remote]
realUrl = await invoker.checkOutputCommand(
["git", "ls-remote", "--get-url", remote], cwd=self.__dir)
if cfgUrl != realUrl:
await invoker.checkCommand(
["git", "remote", "set-url", remote, cfgUrl],
cwd=self.__dir)
del remotes[remote]
# add remaining (new) remotes
for remote, url in remotes.items():
addCmd = ["git", "remote", "add", remote, url]
if singleBranch: addCmd += ["-t", self.__branch]
await invoker.checkCommand(addCmd, cwd=self.__dir)
# relax security if requested
if not self.__sslVerify:
await invoker.checkCommand(
["git", "config", "http.sslVerify", "false"], cwd=self.__dir)
# Calculate refspec that is used internally. For the user a regular
# refspec is kept in the git config.
# Base fetch command with shallow support
fetchCmd = ["git", "-c", "submodule.recurse=0", "fetch", "-p"]
if isinstance(self.__shallow, int):
fetchCmd.append("--depth={}".format(self.__shallow))
elif isinstance(self.__shallow, str):
fetchCmd.append("--shallow-since={}".format(self.__shallow))
fetchCmd.append("origin")
# Calculate appropriate refspec (all/singleBranch/tag)
if singleBranch:
fetchCmd += [
"+refs/heads/{0}:refs/remotes/origin/{0}".format(self.__branch)
]
else:
fetchCmd += ["+refs/heads/*:refs/remotes/origin/*"]
if self.__tag:
fetchCmd.append("refs/tags/{0}:refs/tags/{0}".format(self.__tag))
# do the checkout
if self.__tag or self.__commit:
await self.__checkoutTag(invoker, fetchCmd, switch)
else:
await self.__checkoutBranch(invoker, fetchCmd, switch)
async def __checkoutTag(self, invoker, fetchCmd, switch):
# checkout only if HEAD is invalid
head = await invoker.callCommand(
["git", "rev-parse", "--verify", "-q", "HEAD"],
stdout=False,
cwd=self.__dir)
if head or switch:
await invoker.checkCommand(fetchCmd, cwd=self.__dir)
await invoker.checkCommand([
"git", "checkout", "-q", "--no-recurse-submodules",
self.__commit if self.__commit else "tags/" + self.__tag
],
cwd=self.__dir)
# FIXME: will not be called again if interrupted!
await self.__checkoutSubmodules(invoker)
async def __checkoutBranch(self, invoker, fetchCmd, switch):
await invoker.checkCommand(fetchCmd, cwd=self.__dir)
if await invoker.callCommand(
["git", "rev-parse", "--verify", "-q", "HEAD"],
stdout=False,
cwd=self.__dir):
# checkout only if HEAD is invalid
await invoker.checkCommand([
"git", "checkout", "--no-recurse-submodules", "-b",
self.__branch, "remotes/origin/" + self.__branch
],
cwd=self.__dir)
await self.__checkoutSubmodules(invoker)
elif switch:
# We're switching the ref. There we will actively change the branch which
# is normally forbidden.
assert not self.__submodules
if await invoker.callCommand([
"git", "show-ref", "-q", "--verify",
"refs/heads/" + self.__branch
]):
# Branch does not exist. Create and checkout.
await invoker.checkCommand([
"git", "checkout", "--no-recurse-submodules", "-b",
self.__branch, "remotes/origin/" + self.__branch
],
cwd=self.__dir)
else:
# Branch exists already. Checkout and fast forward...
await invoker.checkCommand([
"git", "checkout", "--no-recurse-submodules", self.__branch
],
cwd=self.__dir)
await invoker.checkCommand([
"git", "-c", "submodule.recurse=0", "merge", "--ff-only",
"refs/remotes/origin/" + self.__branch
],
cwd=self.__dir)
elif (await invoker.checkOutputCommand(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=self.__dir)) == self.__branch:
# pull only if on original branch
preUpdate = await self.__updateSubmodulesPre(invoker)
await invoker.checkCommand([
"git", "-c", "submodule.recurse=0", "merge", "--ff-only",
"refs/remotes/origin/" + self.__branch
],
cwd=self.__dir)
await self.__updateSubmodulesPost(invoker, preUpdate)
else:
invoker.warn("Not updating", self.__dir,
"because branch was changed manually...")
async def __checkoutSubmodules(self, invoker):
if not self.__submodules: return
args = ["git", "submodule", "update", "--init"]
if self.__shallowSubmodules:
args += ["--depth", "1"]
if self.__recurseSubmodules:
args += ["--recursive"]
if isinstance(self.__submodules, list):
args.append("--")
args.extend(self.__submodules)
await invoker.checkCommand(args, cwd=self.__dir)
async def __updateSubmodulesPre(self, invoker, base="."):
"""Query the status of the currently checked out submodules.
Returns a map with the paths of all checked out submodules as keys.
The value will be True if the submodule looks untouched by the user and
is deemed to be updateable. If the value is False the submodule is
different from the expected vanilla checkout state. The list may only
be a sub-set of all known submodules.
"""
if not self.__submodules:
return {}
# List all active and checked out submodules. This way we know the
# state of all submodules and compare them later to the expected state.
args = [
"git", "-C", base, "submodule", "-q", "foreach",
"printf '%s\\t%s\\n' \"$sm_path\" \"$(git rev-parse HEAD)\""
]
checkedOut = await invoker.checkOutputCommand(args, cwd=self.__dir)
checkedOut = {
path: commit
for path, commit in (line.split("\t")
for line in checkedOut.split("\n") if line)
}
if not checkedOut: return {}
# List commits from git tree of all paths for checked out submodules.
# This is what should be checked out.
args = ["git", "-C", base, "ls-tree", "-z", "HEAD"] + sorted(
checkedOut.keys())
allPaths = await invoker.checkOutputCommand(args, cwd=self.__dir)
allPaths = {
normPath(path): attribs.split(' ')[2]
for attribs, path in (p.split('\t') for p in allPaths.split('\0')
if p) if attribs.split(' ')[1] == "commit"
}
# Calculate which paths are in the right state. They must match the
# commit and must be in detached HEAD state.
ret = {}
for path, commit in checkedOut.items():
path = normPath(path)
if allPaths.get(path) != commit:
ret[path] = False
continue
code = await invoker.callCommand(
["git", "symbolic-ref", "-q", "HEAD"],
cwd=os.path.join(self.__dir, base, path))
if code == 0:
ret[path] = False
continue
ret[path] = True
return ret
async def __updateSubmodulesPost(self, invoker, oldState, base="."):
"""Update all submodules that are safe.
Will update all submodules that are either new or have not been touched
by the user. This will be done recursively if that is enabled.
"""
if not self.__submodules:
return {}
if not os.path.exists(invoker.joinPath(self.__dir, base,
".gitmodules")):
return {}
# Sync remote URLs into our config in case they were changed
args = ["git", "-C", base, "submodule", "sync"]
await invoker.checkCommand(args, cwd=self.__dir)
# List all paths as per .gitmodules. This gives us the list of all
# known submodules. Optionally restrict to user specified subset.
args = [
"git", "-C", base, "config", "-f", ".gitmodules", "-z",
"--get-regexp", "path"
]
finishedProc = await invoker.runCommand(args,
cwd=self.__dir,
stdout=True)
allPaths = finishedProc.stdout.rstrip(
) if finishedProc.returncode == 0 else ""
allPaths = [p.split("\n")[1] for p in allPaths.split("\0") if p]
if isinstance(self.__submodules, list):
subset = set(normPath(p) for p in self.__submodules)
allPaths = [p for p in allPaths if normPath(p) in subset]
# Update only new or unmodified paths
updatePaths = [p for p in allPaths if oldState.get(normPath(p), True)]
for p in sorted(set(allPaths) - set(updatePaths)):
invoker.warn("Not updating submodule",
os.path.join(self.__dir, base, p),
"because its HEAD has been switched...")
if not updatePaths:
return
# If we recurse into sub-submodules get their potential state up-front
if self.__recurseSubmodules:
# Explicit loop because of Python 3.5: "'await' expressions in
# comprehensions are not supported".
subMods = {}
for p in updatePaths:
subMods[p] = await self.__updateSubmodulesPre(
invoker, os.path.join(base, p))
# Do the update of safe submodules
args = ["git", "-C", base, "submodule", "update", "--init"]
if self.__shallowSubmodules:
args += ["--depth", "1"]
args.append("--")
args += updatePaths
await invoker.checkCommand(args, cwd=self.__dir)
# Update sub-submodules if requested
if self.__recurseSubmodules:
for p in updatePaths:
await self.__updateSubmodulesPost(invoker, subMods[p],
os.path.join(base, p))
def canSwitch(self, oldSpec):
diff = self._diffSpec(oldSpec)
# Filter irrelevant properties
diff -= {"sslVerify", 'singleBranch', 'shallow', 'shallowSubmodules'}
diff = set(prop for prop in diff if not prop.startswith("remote-"))
# Enabling "submodules" and/or "recurseSubmodules" is ok. The
# additional content will be checked out in invoke().
if not oldSpec.get("submodules", False) and self.__submodules:
diff.discard("submodules")
if not oldSpec.get("recursiveSubmodules",
False) and self.__recurseSubmodules:
diff.discard("recursiveSubmodules")
# Without submodules the recursiveSubmodules property is irrelevant
if not self.__submodules:
diff.discard("recursiveSubmodules")
# For the rest we can try a inline switch. Git does not handle
# vanishing submodules well and neither do we. So if submodules are
# enabled then we do not do an in-place update.
if not diff:
return True
if not diff.issubset({"branch", "tag", "commit", "rev", "url"}):
return False
if self.__submodules:
return False
return True
async def switch(self, invoker, oldSpec):
# Try to checkout new state in old workspace. If something fails the
# old attic logic will take over.
await self.invoke(invoker, True)
return True
def asDigestScript(self):
"""Return forward compatible stable string describing this git module.
The format is "url rev-spec dir" where rev-spec depends on the given reference.
"""
if self.__stripUser:
filt = removeUserFromUrl
else:
filt = lambda x: x
if self.__commit:
ret = self.__commit + " " + self.__dir
elif self.__tag:
ret = filt(
self.__url) + " refs/tags/" + self.__tag + " " + self.__dir
else:
ret = filt(
self.__url) + " refs/heads/" + self.__branch + " " + self.__dir
if self.__submodules:
ret += " submodules"
if isinstance(self.__submodules, list):
ret += "[{}]".format(",".join(self.__submodules))
if self.__recurseSubmodules:
ret += " recursive"
return ret
def asJenkins(self, workPath, credentials, options):
scm = ElementTree.Element("scm",
attrib={
"class": "hudson.plugins.git.GitSCM",
"plugin": "[email protected]",
})
ElementTree.SubElement(scm, "configVersion").text = "2"
userconfigs = ElementTree.SubElement(
ElementTree.SubElement(scm, "userRemoteConfigs"),
"hudson.plugins.git.UserRemoteConfig")
url = ElementTree.SubElement(userconfigs, "url")
url.text = self.__url
if credentials:
credentialsId = ElementTree.SubElement(userconfigs,
"credentialsId")
credentialsId.text = credentials
branch = ElementTree.SubElement(
ElementTree.SubElement(ElementTree.SubElement(scm, "branches"),
"hudson.plugins.git.BranchSpec"), "name")
if self.__commit:
branch.text = self.__commit
elif self.__tag:
branch.text = "refs/tags/" + self.__tag
else:
branch.text = "refs/heads/" + self.__branch
ElementTree.SubElement(
scm, "doGenerateSubmoduleConfigurations").text = "false"
ElementTree.SubElement(scm, "submoduleCfg", attrib={"class": "list"})
extensions = ElementTree.SubElement(scm, "extensions")
ElementTree.SubElement(
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.RelativeTargetDirectory"),
"relativeTargetDir").text = os.path.normpath(
os.path.join(workPath, self.__dir))
# remove untracked files and stale branches
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CleanCheckout")
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.PruneStaleBranch")
# set git clone options
if isinstance(self.__shallow, int):
shallow = str(self.__shallow)
else:
shallow = options.get("scm.git.shallow")
timeout = options.get("scm.git.timeout")
if shallow is not None or timeout is not None:
co = ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CloneOption")
if shallow is not None:
try:
shallow = int(shallow)
if shallow < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.shallow' option: " +
str(shallow))
if shallow > 0:
ElementTree.SubElement(co, "shallow").text = "true"
ElementTree.SubElement(co, "noTags").text = "false"
ElementTree.SubElement(co, "reference").text = ""
ElementTree.SubElement(co, "depth").text = str(shallow)
ElementTree.SubElement(co, "honorRefspec").text = "false"
if timeout is not None:
try:
timeout = int(timeout)
if timeout < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.timeout' option: " +
str(timeout))
if timeout > 0:
ElementTree.SubElement(co, "timeout").text = str(timeout)
if self.__submodules:
assert isinstance(self.__submodules, bool)
sub = ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.SubmoduleOption")
if self.__recurseSubmodules:
ElementTree.SubElement(sub,
"recursiveSubmodules").text = "true"
if self.__shallowSubmodules:
ElementTree.SubElement(sub, "shallow").text = "true"
ElementTree.SubElement(sub, "depth").text = "1"
if timeout is not None:
ElementTree.SubElement(sub, "timeout").text = str(timeout)
if isTrue(options.get("scm.ignore-hooks", "0")):
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.IgnoreNotifyCommit")
return scm
def getDirectory(self):
return self.__dir
def isDeterministic(self):
return bool(self.__tag) or bool(self.__commit)
def hasJenkinsPlugin(self):
# Cloning a subset of submodules is not supported by the Jenkins
# git-plugin. Fall back to our implementation in this case.
return not isinstance(self.__submodules, list)
def callGit(self, workspacePath, *args, check=True):
cmdLine = ['git']
cmdLine.extend(args)
cwd = os.path.join(workspacePath, self.__dir)
try:
output = subprocess.check_output(cmdLine,
cwd=cwd,
universal_newlines=True,
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
if check:
raise BuildError(
"git error:\n Directory: '{}'\n Command: '{}'\n'{}'".
format(cwd, " ".join(cmdLine), e.output.rstrip()))
else:
return ""
except OSError as e:
raise BuildError("Error calling git: " + str(e))
return output.strip()
def status(self, workspacePath):
status = ScmStatus()
try:
onCorrectBranch = False
onTag = False
output = self.callGit(workspacePath, 'ls-remote', '--get-url')
if output != self.__url:
status.add(
ScmTaint.switched,
"> URL: configured: '{}', actual: '{}'".format(
self.__url, output))
if self.__commit:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD')
if output != self.__commit:
status.add(
ScmTaint.switched,
"> commit: configured: '{}', actual: '{}'".format(
self.__commit, output))
elif self.__tag:
output = self.callGit(workspacePath, 'tag', '--points-at',
'HEAD').splitlines()
if self.__tag not in output:
actual = ("'" + ", ".join(output) +
"'") if output else "not on any tag"
status.add(
ScmTaint.switched,
"> tag: configured: '{}', actual: {}".format(
self.__tag, actual))
# Need to check if the tag still exists. Otherwise the "git
# log" command at the end will trip.
try:
self.callGit(workspacePath, 'rev-parse',
'tags/' + self.__tag)
onTag = True
except BuildError:
pass
elif self.__branch:
output = self.callGit(workspacePath, 'rev-parse',
'--abbrev-ref', 'HEAD')
if output != self.__branch:
status.add(
ScmTaint.switched,
"> branch: configured: '{}', actual: '{}'".format(
self.__branch, output))
else:
output = self.callGit(
workspacePath, 'log', '--oneline',
'refs/remotes/origin/' + self.__branch + '..HEAD')
if output:
status.add(
ScmTaint.unpushed_main,
joinLines(
"> unpushed commits on {}:".format(
self.__branch), indent(output, ' ')))
onCorrectBranch = True
# Check for modifications wrt. checked out commit
output = self.callGit(workspacePath, 'status', '--porcelain',
'--ignore-submodules=all')
if output:
status.add(ScmTaint.modified,
joinLines("> modified:", indent(output, ' ')))
# The following shows all unpushed commits reachable by any ref
# (local branches, stash, detached HEAD, etc).
# Exclude HEAD if the configured branch is checked out to not
# double-count them. Does not mark the SCM as dirty. Exclude the
# configured tag too if it is checked out. Otherwise the tag would
# count as unpushed if it is not on a remote branch.
what = ['--all', '--not', '--remotes']
if onCorrectBranch: what.append('HEAD')
if onTag: what.append("tags/" + self.__tag)
output = self.callGit(workspacePath, 'log', '--oneline',
'--decorate', *what)
if output:
status.add(
ScmTaint.unpushed_local,
joinLines("> unpushed local commits:",
indent(output, ' ')))
# Dive into submodules
self.__statusSubmodule(workspacePath, status, self.__submodules)
except BuildError as e:
status.add(ScmTaint.error, e.slogan)
return status
def __statusSubmodule(self, workspacePath, status, shouldExist, base="."):
"""Get the status of submodules and possibly sub-submodules.
The regular "git status" command is not sufficient for our case. In
case the submodule is not initialized "git status" will completely
ignore it. Using "git submodule status" would help but it's output is
not ment to be parsed by tools.
So we first get the list of all possible submodules with their tracked
commit. Then the actual commit is compared and any further
modifications and unpuched commits are checked.
"""
if not os.path.exists(os.path.join(workspacePath, base,
".gitmodules")):
return
# List all paths as per .gitmodules. This gives us the list of all
# known submodules.
allPaths = self.callGit(workspacePath,
"-C",
base,
"config",
"-f",
".gitmodules",
"-z",
"--get-regexp",
"path",
check=False)
allPaths = [p.split("\n")[1] for p in allPaths.split("\0") if p]
if not allPaths:
return
# Fetch the respecive commits as per git ls-tree
allPaths = self.callGit(workspacePath, "-C", base, "ls-tree", "-z",
"HEAD", *allPaths)
allPaths = {
path: attribs.split(' ')[2]
for attribs, path in (p.split('\t') for p in allPaths.split('\0')
if p) if attribs.split(' ')[1] == "commit"
}
# Normalize subset of submodules
if isinstance(shouldExist, list):
shouldExist = set(normPath(p) for p in shouldExist)
elif shouldExist:
shouldExist = set(normPath(p) for p in allPaths.keys())
else:
shouldExist = set()
# Check each submodule for their commit, modifications and unpushed
# stuff. Unconditionally recurse to even see if something is there even
# tough it shouldn't.
for path, commit in sorted(allPaths.items()):
subPath = os.path.join(base, path)
subShouldExist = normPath(path) in shouldExist
if not os.path.exists(os.path.join(workspacePath, subPath,
".git")):
if subShouldExist:
status.add(ScmTaint.modified,
"> submodule not checked out: " + subPath)
elif not dirIsEmpty(os.path.join(workspacePath, subPath)):
status.add(ScmTaint.modified,
"> ignored submodule not empty: " + subPath)
continue
elif not subShouldExist:
status.add(ScmTaint.modified,
"> submodule checked out: " + subPath)
realCommit = self.callGit(workspacePath, "-C", subPath,
"rev-parse", "HEAD")
if commit != realCommit:
status.add(
ScmTaint.switched,
"> submodule '{}' switched commit: configured: '{}', actual: '{}'"
.format(subPath, commit, realCommit))
output = self.callGit(workspacePath, "-C", subPath, 'status',
'--porcelain', '--ignore-submodules=all')
if output:
status.add(
ScmTaint.modified,
joinLines("> submodule '{}' modified:".format(subPath),
indent(output, ' ')))
output = self.callGit(workspacePath, "-C", subPath, 'log',
'--oneline', '--decorate', '--all', '--not',
'--remotes')
if output:
status.add(
ScmTaint.unpushed_local,
joinLines(
"> submodule '{}' unpushed local commits:".format(
subPath), indent(output, ' ')))
self.__statusSubmodule(workspacePath, status,
self.__recurseSubmodules, subPath)
def getAuditSpec(self):
extra = {}
if self.__submodules:
extra['submodules'] = self.__submodules
if self.__recurseSubmodules:
extra['recurseSubmodules'] = True
return ("git", self.__dir, extra)
def hasLiveBuildId(self):
return True
async def predictLiveBuildId(self, step):
if self.__commit:
return bytes.fromhex(self.__commit)
with stepAction(step, "LS-REMOTE", self.__url, (INFO, TRACE)) as a:
if self.__tag:
# Annotated tags are objects themselves. We need the commit object!
refs = [
"refs/tags/" + self.__tag + '^{}',
"refs/tags/" + self.__tag
]
else:
refs = ["refs/heads/" + self.__branch]
cmdLine = ['git', 'ls-remote', self.__url] + refs
try:
stdout = await check_output(cmdLine,
stderr=subprocess.DEVNULL,
universal_newlines=True)
output = stdout.strip()
except subprocess.CalledProcessError as e:
a.fail("exit {}".format(e.returncode), WARNING)
return None
except OSError as e:
a.fail("error ({})".format(e))
return None
# have we found anything at all?
if not output:
a.fail("unknown", WARNING)
return None
# See if we got one of our intended refs. Git is generating lines with
# the following format:
#
# <sha1>\t<refname>
#
# Put the output into a dict with the refname as key. Be extra careful
# and strip out lines not matching this pattern.
output = {
commitAndRef[1].strip(): bytes.fromhex(commitAndRef[0].strip())
for commitAndRef in (line.split('\t')
for line in output.split('\n'))
if len(commitAndRef) == 2
}
for ref in refs:
if ref in output: return output[ref]
# uhh, should not happen...
a.fail("unknown", WARNING)
return None
def calcLiveBuildId(self, workspacePath):
if self.__commit:
return bytes.fromhex(self.__commit)
else:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD').strip()
return bytes.fromhex(output)
def getLiveBuildIdSpec(self, workspacePath):
if self.__commit:
return "=" + self.__commit
else:
return "g" + os.path.join(workspacePath, self.__dir)
@staticmethod
def processLiveBuildIdSpec(dir):
try:
return subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=dir,
universal_newlines=True).strip()
except subprocess.CalledProcessError as e:
raise BuildError("Git audit failed: " + str(e))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
class GitScm(Scm):
SCHEMA = schema.Schema({
'scm': 'git',
'url': str,
schema.Optional('dir'): str,
schema.Optional('if'): schema.Or(str, IfExpression),
schema.Optional('branch'): str,
schema.Optional('tag'): str,
schema.Optional('commit'): str,
schema.Optional('rev'): str,
schema.Optional(schema.Regex('^remote-.*')): str,
schema.Optional('sslVerify'): bool,
schema.Optional('singleBranch'): bool,
schema.Optional('shallow'): schema.Or(int, str)
})
REMOTE_PREFIX = "remote-"
def __init__(self, spec, overrides=[], secureSSL=None):
super().__init__(spec, overrides)
self.__url = spec["url"]
self.__branch = None
self.__tag = None
self.__commit = None
self.__remotes = {}
if "rev" in spec:
rev = spec["rev"]
if rev.startswith("refs/heads/"):
self.__branch = rev[11:]
elif rev.startswith("refs/tags/"):
self.__tag = rev[10:]
elif len(rev) == 40:
self.__commit = rev
else:
raise ParseError("Invalid rev format: " + rev)
self.__branch = spec.get("branch", self.__branch)
self.__tag = spec.get("tag", self.__tag)
self.__commit = spec.get("commit", self.__commit)
if self.__commit:
# validate commit
if re.match("^[0-9a-f]{40}$", self.__commit) is None:
raise ParseError("Invalid commit id: " + str(self.__commit))
elif not self.__branch and not self.__tag:
# nothing secified at all -> master branch
self.__branch = "master"
self.__dir = spec.get("dir", ".")
# convert remotes into separate dictionary
for key, val in spec.items():
if key.startswith(GitScm.REMOTE_PREFIX):
stripped_key = key[len(GitScm.REMOTE_PREFIX):] # remove prefix
if stripped_key == "origin":
raise ParseError("Invalid remote name: " + stripped_key)
self.__remotes.update({stripped_key: val})
self.__sslVerify = spec.get('sslVerify', secureSSL)
self.__singleBranch = spec.get('singleBranch')
self.__shallow = spec.get('shallow')
def getProperties(self):
properties = super().getProperties()
properties.update({
'scm':
'git',
'url':
self.__url,
'branch':
self.__branch,
'tag':
self.__tag,
'commit':
self.__commit,
'dir':
self.__dir,
'rev': (self.__commit if self.__commit else
(("refs/tags/" + self.__tag) if self.__tag else
("refs/heads/" + self.__branch))),
'sslVerify':
self.__sslVerify,
'singleBranch':
self.__singleBranch,
'shallow':
self.__shallow,
})
for key, val in self.__remotes.items():
properties.update({GitScm.REMOTE_PREFIX + key: val})
return properties
async def invoke(self, invoker):
# make sure the git directory exists
if not os.path.isdir(invoker.joinPath(self.__dir, ".git")):
await invoker.checkCommand(["git", "init", self.__dir])
# Shallow implies singleBranch
if self.__singleBranch is None:
singleBranch = self.__shallow is not None
else:
singleBranch = self.__singleBranch
singleBranch = singleBranch and (self.__branch is not None)
# setup and update remotes
remotes = {"origin": self.__url}
remotes.update(self.__remotes)
existingRemotes = await invoker.checkOutputCommand(["git", "remote"],
cwd=self.__dir)
for remote in existingRemotes.split("\n"):
if remote in remotes:
cfgUrl = remotes[remote]
realUrl = await invoker.checkOutputCommand(
["git", "ls-remote", "--get-url", remote], cwd=self.__dir)
if cfgUrl != realUrl:
await invoker.checkCommand(
["git", "remote", "set-url", remote, cfgUrl],
cwd=self.__dir)
del remotes[remote]
# add remaining (new) remotes
for remote, url in remotes.items():
addCmd = ["git", "remote", "add", remote, url]
if singleBranch: addCmd += ["-t", self.__branch]
await invoker.checkCommand(addCmd, cwd=self.__dir)
# relax security if requested
if not self.__sslVerify:
await invoker.checkCommand(
["git", "config", "http.sslVerify", "false"], cwd=self.__dir)
# Calculate refspec that is used internally. For the user a regular
# refspec is kept in the git config.
# Base fetch command with shallow support
fetchCmd = ["git", "fetch", "-p"]
if isinstance(self.__shallow, int):
fetchCmd.append("--depth={}".format(self.__shallow))
elif isinstance(self.__shallow, str):
fetchCmd.append("--shallow-since={}".format(self.__shallow))
fetchCmd.append("origin")
# Calculate appropriate refspec (all/singleBranch/tag)
if singleBranch:
fetchCmd += [
"+refs/heads/{0}:refs/remotes/origin/{0}".format(self.__branch)
]
else:
fetchCmd += ["+refs/heads/*:refs/remotes/origin/*"]
if self.__tag:
fetchCmd.append("refs/tags/{0}:refs/tags/{0}".format(self.__tag))
# do the checkout
if self.__tag or self.__commit:
await self.__checkoutTag(invoker, fetchCmd)
else:
await self.__checkoutBranch(invoker, fetchCmd)
async def __checkoutTag(self, invoker, fetchCmd):
# checkout only if HEAD is invalid
head = await invoker.callCommand(
["git", "rev-parse", "--verify", "-q", "HEAD"],
stdout=False,
cwd=self.__dir)
if head:
await invoker.checkCommand(fetchCmd, cwd=self.__dir)
await invoker.checkCommand([
"git", "checkout", "-q",
self.__commit if self.__commit else "tags/" + self.__tag
],
cwd=self.__dir)
async def __checkoutBranch(self, invoker, fetchCmd):
await invoker.checkCommand(fetchCmd, cwd=self.__dir)
if await invoker.callCommand(
["git", "rev-parse", "--verify", "-q", "HEAD"],
stdout=False,
cwd=self.__dir):
# checkout only if HEAD is invalid
await invoker.checkCommand([
"git", "checkout", "-b", self.__branch,
"remotes/origin/" + self.__branch
],
cwd=self.__dir)
elif (await invoker.checkOutputCommand(
["git", "rev-parse", "--abbrev-ref", "HEAD"],
cwd=self.__dir)) == self.__branch:
# pull only if on original branch
await invoker.checkCommand([
"git", "merge", "--ff-only",
"refs/remotes/origin/" + self.__branch
],
cwd=self.__dir)
else:
invoker.warn("Not updating", self.__dir,
"because branch was changed manually...")
def asDigestScript(self):
"""Return forward compatible stable string describing this git module.
The format is "url rev-spec dir" where rev-spec depends on the given reference.
"""
if self.__commit:
return self.__commit + " " + self.__dir
elif self.__tag:
return self.__url + " refs/tags/" + self.__tag + " " + self.__dir
else:
return self.__url + " refs/heads/" + self.__branch + " " + self.__dir
def asJenkins(self, workPath, credentials, options):
scm = ElementTree.Element("scm",
attrib={
"class": "hudson.plugins.git.GitSCM",
"plugin": "[email protected]",
})
ElementTree.SubElement(scm, "configVersion").text = "2"
userconfigs = ElementTree.SubElement(
ElementTree.SubElement(scm, "userRemoteConfigs"),
"hudson.plugins.git.UserRemoteConfig")
url = ElementTree.SubElement(userconfigs, "url")
url.text = self.__url
if credentials:
credentialsId = ElementTree.SubElement(userconfigs,
"credentialsId")
credentialsId.text = credentials
branch = ElementTree.SubElement(
ElementTree.SubElement(ElementTree.SubElement(scm, "branches"),
"hudson.plugins.git.BranchSpec"), "name")
if self.__commit:
branch.text = self.__commit
elif self.__tag:
branch.text = "refs/tags/" + self.__tag
else:
branch.text = "refs/heads/" + self.__branch
ElementTree.SubElement(
scm, "doGenerateSubmoduleConfigurations").text = "false"
ElementTree.SubElement(scm, "submoduleCfg", attrib={"class": "list"})
extensions = ElementTree.SubElement(scm, "extensions")
ElementTree.SubElement(
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.RelativeTargetDirectory"),
"relativeTargetDir").text = os.path.normpath(
os.path.join(workPath, self.__dir))
# remove untracked files and stale branches
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CleanCheckout")
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.PruneStaleBranch")
# set git clone options
shallow = options.get("scm.git.shallow")
timeout = options.get("scm.git.timeout")
if shallow is not None or timeout is not None:
co = ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CloneOption")
if shallow is not None:
try:
shallow = int(shallow)
if shallow < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.shallow' option: " +
str(shallow))
if shallow > 0:
ElementTree.SubElement(co, "shallow").text = "true"
ElementTree.SubElement(co, "noTags").text = "false"
ElementTree.SubElement(co, "reference").text = ""
ElementTree.SubElement(co, "depth").text = str(shallow)
ElementTree.SubElement(co, "honorRefspec").text = "false"
if timeout is not None:
try:
timeout = int(timeout)
if timeout < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.timeout' option: " +
str(shallow))
if timeout > 0:
ElementTree.SubElement(co, "timeout").text = str(timeout)
if isTrue(options.get("scm.ignore-hooks", "0")):
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.IgnoreNotifyCommit")
return scm
def getDirectory(self):
return self.__dir
def isDeterministic(self):
return bool(self.__tag) or bool(self.__commit)
def hasJenkinsPlugin(self):
return True
def callGit(self, workspacePath, *args):
cmdLine = ['git']
cmdLine.extend(args)
cwd = os.path.join(workspacePath, self.__dir)
try:
output = subprocess.check_output(cmdLine,
cwd=cwd,
universal_newlines=True,
stderr=subprocess.DEVNULL)
except subprocess.CalledProcessError as e:
raise BuildError(
"git error:\n Directory: '{}'\n Command: '{}'\n'{}'".format(
cwd, " ".join(cmdLine), e.output.rstrip()))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
return output.strip()
def status(self, workspacePath):
status = ScmStatus()
try:
onCorrectBranch = False
onTag = False
output = self.callGit(workspacePath, 'ls-remote', '--get-url')
if output != self.__url:
status.add(
ScmTaint.switched,
"> URL: configured: '{}', actual: '{}'".format(
self.__url, output))
if self.__commit:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD')
if output != self.__commit:
status.add(
ScmTaint.switched,
"> commit: configured: '{}', actual: '{}'".format(
self.__commit, output))
elif self.__tag:
output = self.callGit(workspacePath, 'tag', '--points-at',
'HEAD').splitlines()
if self.__tag not in output:
actual = ("'" + ", ".join(output) +
"'") if output else "not on any tag"
status.add(
ScmTaint.switched,
"> tag: configured: '{}', actual: {}".format(
self.__tag, actual))
# Need to check if the tag still exists. Otherwise the "git
# log" command at the end will trip.
try:
self.callGit(workspacePath, 'rev-parse',
'tags/' + self.__tag)
onTag = True
except BuildError:
pass
elif self.__branch:
output = self.callGit(workspacePath, 'rev-parse',
'--abbrev-ref', 'HEAD')
if output != self.__branch:
status.add(
ScmTaint.switched,
"> branch: configured: '{}', actual: '{}'".format(
self.__branch, output))
else:
output = self.callGit(
workspacePath, 'log', '--oneline',
'refs/remotes/origin/' + self.__branch + '..HEAD')
if output:
status.add(
ScmTaint.unpushed_main,
joinLines(
"> unpushed commits on {}:".format(
self.__branch), indent(output, ' ')))
onCorrectBranch = True
# Check for modifications wrt. checked out commit
output = self.callGit(workspacePath, 'status', '--porcelain')
if output:
status.add(ScmTaint.modified,
joinLines("> modified:", indent(output, ' ')))
# The following shows all unpushed commits reachable by any ref
# (local branches, stash, detached HEAD, etc).
# Exclude HEAD if the configured branch is checked out to not
# double-count them. Does not mark the SCM as dirty. Exclude the
# configured tag too if it is checked out. Otherwise the tag would
# count as unpushed if it is not on a remote branch.
what = ['--all', '--not', '--remotes']
if onCorrectBranch: what.append('HEAD')
if onTag: what.append("tags/" + self.__tag)
output = self.callGit(workspacePath, 'log', '--oneline',
'--decorate', *what)
if output:
status.add(
ScmTaint.unpushed_local,
joinLines("> unpushed local commits:",
indent(output, ' ')))
except BuildError as e:
status.add(ScmTaint.error, e.slogan)
return status
def getAuditSpec(self):
return ("git", self.__dir, {})
def hasLiveBuildId(self):
return True
async def predictLiveBuildId(self, step):
if self.__commit:
return bytes.fromhex(self.__commit)
with stepAction(step, "LS-REMOTE", self.__url, (INFO, TRACE)) as a:
if self.__tag:
# Annotated tags are objects themselves. We need the commit object!
refs = [
"refs/tags/" + self.__tag + '^{}',
"refs/tags/" + self.__tag
]
else:
refs = ["refs/heads/" + self.__branch]
cmdLine = ['git', 'ls-remote', self.__url] + refs
try:
stdout = await check_output(cmdLine,
stderr=subprocess.DEVNULL,
universal_newlines=True)
output = stdout.strip()
except subprocess.CalledProcessError as e:
a.fail("exit {}".format(e.returncode), WARNING)
return None
except OSError as e:
a.fail("error ({})".format(e))
return None
# have we found anything at all?
if not output:
a.fail("unknown", WARNING)
return None
# See if we got one of our intended refs. Git is generating lines with
# the following format:
#
# <sha1>\t<refname>
#
# Put the output into a dict with the refname as key. Be extra careful
# and strip out lines not matching this pattern.
output = {
commitAndRef[1].strip(): bytes.fromhex(commitAndRef[0].strip())
for commitAndRef in (line.split('\t')
for line in output.split('\n'))
if len(commitAndRef) == 2
}
for ref in refs:
if ref in output: return output[ref]
# uhh, should not happen...
a.fail("unknown", WARNING)
return None
def calcLiveBuildId(self, workspacePath):
if self.__commit:
return bytes.fromhex(self.__commit)
else:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD').strip()
return bytes.fromhex(output)
def getLiveBuildIdSpec(self, workspacePath):
if self.__commit:
return "=" + self.__commit
else:
return "g" + os.path.join(workspacePath, self.__dir)
@staticmethod
def processLiveBuildIdSpec(dir):
try:
return subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=dir,
universal_newlines=True).strip()
except subprocess.CalledProcessError as e:
raise BuildError("Git audit failed: " + str(e))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
def entrypoint():
args = docopt(__doc__, version=f"midiedlsequencer {__version__}")
if args["--middle-c"] not in ["C3", "C4"]:
print("Error: --middle-c argument must be either C3 or C4.\n")
print(__doc__)
exit(1)
with open(args['-s']) as inf:
samples = yaml.load(inf, Loader=yaml.Loader)
schema.Schema([{
"name": str,
"note": schema.Regex(r"[A-G](#|b)?-?\d+"),
"src": str,
"start": float,
"length": float
}]).validate(samples)
mid = mido.MidiFile(args['-m'])
# MIDI messages represent time in ticks relative to the previous
# message. The following code recovers the absoulute time from
# the beginning of the file in seconds. The MidiFile object computes
# the time difference in seconds as it's iterated through, and returns
# it in a property of the messages yielded by the iterator. I don't
# like that it's stateful and somewhat obscure, but it works.
abs_msgs = []
acc_time = 0.0
for msg in mid:
acc_time += msg.time
abs_msgs.append(msg.copy(time=acc_time))
notes = [m for m in abs_msgs if m.type is 'note_on']
try:
fps = float(args['--fps'])
except ValueError:
print("Error: --fps argument must be a float literal.\n")
print(__doc__)
exit(1)
def range_from_s_fps(start, dur, fps):
return TimeRange(
start_time=RationalTime.from_seconds(start).rescaled_to(fps),
duration=RationalTime.from_seconds(dur).rescaled_to(fps))
for s in samples:
# The following selects the times of the note corresponding to the sample.
midi_note = pretty_midi.note_name_to_number(
s['note']) + (12 if args['--middle-c'] == "C3" else 0)
times = [n.time for n in notes if n.note == midi_note]
# The following creates an EDL timeline with the times computed above.
tl = otio.schema.Timeline(name=s['name'])
track = otio.schema.Track()
# DaVinci Resolve will automatically set the starting timecode for an imported
# EDL timeline to the start of the first clip. This has the undesirable effect
# of removing initial blank time. As a workaround, a clip with duration zero
# is inserted at the beginning of the EDL file: DVR will not place a clip in
# the timeline, but it will set the starting timecode correctly.
zero_clip = otio.schema.Clip(
name=s["src"],
media_reference=otio.schema.ExternalReference(target_url=s["src"]),
source_range=range_from_s_fps(0, 0, fps))
track.append(zero_clip)
clip = otio.schema.Clip(
name=s["src"], # This is the actual clip
media_reference=otio.schema.ExternalReference(target_url=s["src"]),
source_range=range_from_s_fps(s['start'], s['length'], fps))
# Blank time in OTIO timelines is represented with gaps. To accomodate non-contiguous
# samples, gaps with appropriate durations must be inserted between clips.
used_time = 0.0
for t in times:
track.append(
otio.schema.Gap(
source_range=range_from_s_fps(0, t - used_time, fps)))
track.append(clip.deepcopy())
used_time = t + s['length']
tl.tracks.append(track)
otio.adapters.write_to_file(
tl, os.path.join(args['-o'], s['name'] + '.edl'))
# ElastiCluster imports
from elasticluster.utils import string_to_boolean
## custom validators
def validator(fn):
"""
Decorate a function for use as a validator with `schema`_
.. _schema: https://github.com/keleshev/schema
"""
return schema.Use(fn)
alphanumeric = schema.Regex(r'[0-9A-Za-z_]+')
boolean = schema.Use(string_to_boolean)
def _file_name(v):
try:
return os.path.expanduser(v)
except Exception as err:
raise ValueError("invalid file name `{0}`: {1}".format(v, err))
@validator
def existing_file(v):
f = _file_name(v)
class FileInfoAnalyzerReport(CortexAnalyzerReport):
""" Represents a Cortex FileInfo_8_0 analysis JSON report. """
report_schema = schema.Schema({
"summary": {
"taxonomies": [
schema.Schema(
{
"level": schema.Or("info", "malicious", "safe"),
"namespace": "FileInfo",
# "predicate": str,
# "value": str
},
ignore_extra_keys=True)
]
},
"full": {
"results": [{
"submodule_name":
"Basic properties",
"results": [
{
"submodule_section_header": "Hashes",
"submodule_section_content": {
"md5": schema.Regex(r'^[0-9a-z]{32}$'),
"sha1": schema.Regex(r'^[0-9a-z]{40}$'),
"sha256": schema.Regex(r'^[0-9a-z]{64}$'),
"ssdeep": schema.Regex(r'^[0-9A-Za-z:+/]*$'),
}
},
{
# We consume further structures submodule_sections and
# explicitly check the submodule_section_header to not
# be "Hashes" or it will accept "Hashes"-structures with
# malfarmed hashes.
"submodule_section_header":
schema.And(str, lambda s: s != "Hashes"),
"submodule_section_content":
schema.Schema({}, ignore_extra_keys=True)
},
],
"summary": {
"taxonomies": [
schema.Schema(
{
"level": schema.Or("info", "malicious",
"safe"),
"namespace": "FileInfo",
# "predicate": str,
# "value": str
},
ignore_extra_keys=True)
]
}
}]
},
"success": bool,
"artifacts": CortexAnalyzerReport.report_schema_artifacts,
"operations": []
})
def __init__(self, unvalidated_report=None):
"""
@param report: hash with report data from Cortex FileInfo Analyzer
"""
super().__init__(unvalidated_report)
basic_properties = self.get_element_from_list_of_dicts(
self.report.get('full', []).get('results', {}), 'submodule_name',
'Basic properties').get('results', [])
self._hashes = self.get_element_from_list_of_dicts(
basic_properties, 'submodule_section_header',
'Hashes').get('submodule_section_content', {})
@property
def sha256sum(self):
""" Return the sha256 sum. """
return self._hashes.get('sha256')
@property
def md5sum(self):
""" Return the md5 sum. """
return self._hashes.get('md5')
@property
def ssdeepsum(self):
""" Return the ssdeep sum. """
# TODO: think about if we want to compare ssdeep hashes
return self._hashes.get('ssdeep')
class GitScm(Scm):
SCHEMA = schema.Schema({
'scm': 'git',
'url': str,
schema.Optional('dir'): str,
schema.Optional('if'): str,
schema.Optional('branch'): str,
schema.Optional('tag'): str,
schema.Optional('commit'): str,
schema.Optional('rev'): str,
schema.Optional(schema.Regex('^remote-.*')): str,
})
REMOTE_PREFIX = "remote-"
def __init__(self, spec, overrides=[]):
super().__init__(overrides)
self.__recipe = spec['recipe']
self.__url = spec["url"]
self.__branch = None
self.__tag = None
self.__commit = None
self.__remotes = {}
if "rev" in spec:
rev = spec["rev"]
if rev.startswith("refs/heads/"):
self.__branch = rev[11:]
elif rev.startswith("refs/tags/"):
self.__tag = rev[10:]
elif len(rev) == 40:
self.__commit = rev
else:
raise ParseError("Invalid rev format: " + rev)
self.__branch = spec.get("branch", self.__branch)
self.__tag = spec.get("tag", self.__tag)
self.__commit = spec.get("commit", self.__commit)
if self.__commit:
# validate commit
if re.match("^[0-9a-f]{40}$", self.__commit) is None:
raise ParseError("Invalid commit id: " + str(self.__commit))
elif not self.__branch and not self.__tag:
# nothing secified at all -> master branch
self.__branch = "master"
self.__dir = spec.get("dir", ".")
# convert remotes into separate dictionary
for key, val in spec.items():
if key.startswith(GitScm.REMOTE_PREFIX):
stripped_key = key[len(GitScm.REMOTE_PREFIX):] # remove prefix
if stripped_key == "origin":
raise ParseError("Invalid remote name: " + stripped_key)
self.__remotes.update({stripped_key: val})
def getProperties(self):
properties = [{
'recipe':
self.__recipe,
'scm':
'git',
'url':
self.__url,
'branch':
self.__branch,
'tag':
self.__tag,
'commit':
self.__commit,
'dir':
self.__dir,
'rev': (self.__commit if self.__commit else
(("refs/tags/" + self.__tag) if self.__tag else
("refs/heads/" + self.__branch)))
}]
for key, val in self.__remotes.items():
properties[0].update({GitScm.REMOTE_PREFIX + key: val})
return properties
def asScript(self):
remotes_array = [
"# create an array of all remotes for this repository",
"declare -A BOB_GIT_REMOTES=( [origin]={URL} )".format(
URL=quote(self.__url)),
]
# add additional remotes to array
for name, url in self.__remotes.items():
remotes_array.append("BOB_GIT_REMOTES[{NAME}]={URL}".format(
NAME=quote(name), URL=quote(url)))
# create script to handle remotes
remotes_script = dedent("""\
(
{REMOTES_ARRAY}
# remove remotes from array that are already known to Git
while read -r REMOTE_NAME ; do
# check for empty variable in case no remote at all is specified
if [ -z "$REMOTE_NAME" ]; then
continue
fi
# check if existing remote is configured
if [ "${{BOB_GIT_REMOTES[$REMOTE_NAME]+_}}" ]; then
# check if URL has changed
if [ ! "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}" == "$(git ls-remote --get-url $REMOTE_NAME)" ]; then
git remote set-url "$REMOTE_NAME" "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}"
fi
# it is configured, therefore no need to keep in list
unset "BOB_GIT_REMOTES[$REMOTE_NAME]"
fi
done <<< "$(git remote)"
# add all remaining remotes in the array to the repository
for REMOTE_NAME in "${{!BOB_GIT_REMOTES[@]}}" ; do
git remote add "$REMOTE_NAME" "${{BOB_GIT_REMOTES[$REMOTE_NAME]}}"
done
)""").format(REMOTES_ARRAY="\n ".join(remotes_array))
if self.__tag or self.__commit:
return dedent("""\
export GIT_SSL_NO_VERIFY=true
if [ ! -d {DIR}/.git ] ; then
git init {DIR}
fi
cd {DIR}
{REMOTES}
# checkout only if HEAD is invalid
if ! git rev-parse --verify -q HEAD >/dev/null ; then
git fetch -t origin '+refs/heads/*:refs/remotes/origin/*'
git checkout -q {REF}
fi
""").format(URL=self.__url,
REF=self.__commit if self.__commit else "tags/" +
self.__tag,
DIR=self.__dir,
REMOTES=remotes_script)
else:
return dedent("""\
export GIT_SSL_NO_VERIFY=true
if [ -d {DIR}/.git ] ; then
cd {DIR}
if [[ $(git rev-parse --abbrev-ref HEAD) == "{BRANCH}" ]] ; then
git fetch -p origin
git merge --ff-only refs/remotes/origin/{BRANCH}
else
echo "Warning: not updating {DIR} because branch was changed manually..." >&2
fi
else
if ! git clone -b {BRANCH} {URL} {DIR} ; then
rm -rf {DIR}/.git {DIR}/*
exit 1
fi
cd {DIR}
fi
{REMOTES}
""").format(URL=self.__url,
BRANCH=self.__branch,
DIR=self.__dir,
REMOTES=remotes_script)
def asDigestScript(self):
"""Return forward compatible stable string describing this git module.
The format is "url rev-spec dir" where rev-spec depends on the given reference.
"""
if self.__commit:
return self.__commit + " " + self.__dir
elif self.__tag:
return self.__url + " refs/tags/" + self.__tag + " " + self.__dir
else:
return self.__url + " refs/heads/" + self.__branch + " " + self.__dir
def asJenkins(self, workPath, credentials, options):
scm = ElementTree.Element("scm",
attrib={
"class": "hudson.plugins.git.GitSCM",
"plugin": "[email protected]",
})
ElementTree.SubElement(scm, "configVersion").text = "2"
userconfigs = ElementTree.SubElement(
ElementTree.SubElement(scm, "userRemoteConfigs"),
"hudson.plugins.git.UserRemoteConfig")
url = ElementTree.SubElement(userconfigs, "url")
url.text = self.__url
if credentials:
credentialsId = ElementTree.SubElement(userconfigs,
"credentialsId")
credentialsId.text = credentials
branch = ElementTree.SubElement(
ElementTree.SubElement(ElementTree.SubElement(scm, "branches"),
"hudson.plugins.git.BranchSpec"), "name")
if self.__commit:
branch.text = self.__commit
elif self.__tag:
branch.text = "refs/tags/" + self.__tag
else:
branch.text = "refs/heads/" + self.__branch
ElementTree.SubElement(
scm, "doGenerateSubmoduleConfigurations").text = "false"
ElementTree.SubElement(scm, "submoduleCfg", attrib={"class": "list"})
extensions = ElementTree.SubElement(scm, "extensions")
ElementTree.SubElement(
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.RelativeTargetDirectory"),
"relativeTargetDir").text = os.path.normpath(
os.path.join(workPath, self.__dir))
# remove untracked files and stale branches
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.CleanCheckout")
ElementTree.SubElement(
extensions, "hudson.plugins.git.extensions.impl.PruneStaleBranch")
shallow = options.get("scm.git.shallow")
if shallow is not None:
try:
shallow = int(shallow)
if shallow < 0: raise ValueError()
except ValueError:
raise BuildError("Invalid 'git.shallow' option: " +
str(shallow))
if shallow > 0:
co = ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.CloneOption")
ElementTree.SubElement(co, "shallow").text = "true"
ElementTree.SubElement(co, "noTags").text = "false"
ElementTree.SubElement(co, "reference").text = ""
ElementTree.SubElement(co, "depth").text = str(shallow)
ElementTree.SubElement(co, "honorRefspec").text = "false"
if isTrue(options.get("scm.ignore-hooks", "0")):
ElementTree.SubElement(
extensions,
"hudson.plugins.git.extensions.impl.IgnoreNotifyCommit")
return scm
def merge(self, other):
return False
def getDirectories(self):
return {self.__dir: hashString(self.asDigestScript())}
def isDeterministic(self):
return bool(self.__tag) or bool(self.__commit)
def hasJenkinsPlugin(self):
return True
def callGit(self, workspacePath, *args):
cmdLine = ['git']
cmdLine.extend(args)
try:
output = subprocess.check_output(cmdLine,
cwd=os.path.join(
os.getcwd(), workspacePath,
self.__dir),
universal_newlines=True,
stderr=subprocess.STDOUT)
except subprocess.CalledProcessError as e:
raise BuildError(
"git error:\n Directory: '{}'\n Command: '{}'\n'{}'".format(
os.path.join(workspacePath, self.__dir), " ".join(cmdLine),
e.output.rstrip()))
return output
# Get GitSCM status. The purpose of this function is to return the status of the given directory
#
# return values:
# - error: The SCM is in a error state. Use this if git returned a error code.
# - dirty: SCM is dirty. Could be: modified files, switched to another branch/tag/commit/repo, unpushed commits.
# - clean: Same branch/tag/commit as specified in the recipe and no local changes.
# - empty: Directory is not existing.
#
# This function is called when build with --clean-checkout. 'error' and 'dirty' SCMs are moved to attic,
# while empty and clean directories are not.
def status(self, workspacePath, dir):
scmdir = os.path.join(workspacePath, dir)
if not os.path.exists(os.path.join(os.getcwd(), scmdir)):
return 'empty', '', ''
status = 'clean'
shortStatus = ""
longStatus = ""
def setStatus(shortMsg, longMsg, dirty=True):
nonlocal status, shortStatus, longStatus
if (shortMsg not in shortStatus):
shortStatus += shortMsg
longStatus += longMsg
if (dirty):
status = 'dirty'
try:
output = self.callGit(workspacePath, 'ls-remote',
'--get-url').rstrip()
if output != self.__url:
setStatus(
"S",
colorize(
"> URL: configured: '{}' actual: '{}'\n".format(
self.__url, output), "33"))
else:
if self.__commit:
output = self.callGit(workspacePath, 'rev-parse',
'HEAD').rstrip()
if output != self.__commit:
setStatus(
"S",
colorize(
"> commitId: configured: {} actual: {}\n".
format(self.__commit, output), "33"))
elif self.__tag:
output = self.callGit(workspacePath, 'tag', '--points-at',
'HEAD').rstrip().splitlines()
if self.__tag not in output:
actual = ("'" + ", ".join(output) +
"'") if output else "not on any tag"
setStatus(
"S",
colorize(
" > tag: configured: '{}' actual: {}\n".
format(self.__tag, actual), "33"))
elif self.__branch:
output = self.callGit(workspacePath, 'rev-parse',
'--abbrev-ref', 'HEAD').rstrip()
if output != self.__branch:
setStatus(
"S",
colorize(
"> branch: configured: {} actual: {}\n".format(
self.__branch, output), "33"))
else:
output = self.callGit(
workspacePath, 'rev-list',
'origin/' + self.__branch + '..HEAD')
if len(output):
setStatus("U", "")
# do not print detailed status this point.
# git log --branches --not --remotes --decorate will give the same informations.
output = self.callGit(workspacePath, 'status', '--porcelain')
if len(output):
longMsg = colorize("> modified:\n", "33")
for line in output.split('\n'):
if line != "":
longMsg += ' ' + line + '\n'
setStatus("M", longMsg)
# the following shows unpushed commits even on local branches. do not mark the SCM as dirty.
output = self.callGit(workspacePath, 'log', '--branches', '--not',
'--remotes', '--decorate')
if len(output):
longMsg = colorize("> unpushed:\n", "33")
for line in output.split('\n'):
if line != "":
longStatus += ' ' + line + '\n'
setStatus("u", longMsg, False)
except BuildError as e:
print(e)
ret = 'error'
return status, shortStatus, longStatus
def getAuditSpec(self):
return ("git", [self.__dir])
def hasLiveBuildId(self):
return True
def predictLiveBuildId(self):
if self.__commit:
return [bytes.fromhex(self.__commit)]
if self.__tag:
# Annotated tags are objects themselves. We need the commit object!
refs = [
"refs/tags/" + self.__tag + '^{}', "refs/tags/" + self.__tag
]
else:
refs = ["refs/heads/" + self.__branch]
cmdLine = ['git', 'ls-remote', self.__url] + refs
try:
output = subprocess.check_output(cmdLine,
universal_newlines=True,
stderr=subprocess.STDOUT).strip()
except subprocess.CalledProcessError as e:
return [None]
# have we found anything at all?
if not output:
return [None]
# see if we got one of our intended refs
output = {
ref.strip(): bytes.fromhex(commit.strip())
for commit, ref in (line.split('\t')
for line in output.split('\n'))
}
for ref in refs:
if ref in output: return [output[ref]]
# uhh, should not happen...
return [None]
def calcLiveBuildId(self, workspacePath):
if self.__commit:
return [bytes.fromhex(self.__commit)]
else:
output = self.callGit(workspacePath, 'rev-parse', 'HEAD').strip()
return [bytes.fromhex(output)]
def getLiveBuildIdSpec(self, workspacePath):
if self.__commit:
return ["=" + self.__commit]
else:
return ["g" + os.path.join(workspacePath, self.__dir)]
@staticmethod
def processLiveBuildIdSpec(dir):
try:
return subprocess.check_output(["git", "rev-parse", "HEAD"],
cwd=dir,
universal_newlines=True).strip()
except subprocess.CalledProcessError as e:
raise BuildError("Git audit failed: " + str(e))
except OSError as e:
raise BuildError("Error calling git: " + str(e))
class Descriptor(object, metaclass=abc.ABCMeta):
"""Abstract base class for all |Descriptor|.
A |Descriptor| is a class used to extract relevant information from a collection of |Record|.
More specifically, a |Descriptor| is used in two phases:
* In a first time, optional internals are updated and/or accumulated over the entire collection of
|RecordCollection| tuples. Those internals will be used to tailor the |Descriptor| to the description task
specific context (*e.g.* like updating a range of possible value to later normalize or clip a property value).
* In a second time, the descriptor is used to construct a new |RecordCollection| or a tuple of |RecordCollection|
where each enclosed |Record| is added a named property containing the adequate description of the |Record|
from a tuple of |RecordCollection|.
Subclasses must override the two interfaces used to do so: :meth:`update(*record_collections) <update>` and
:meth:`compute(*record_collections) <compute>` as well as the :meth:`reset` method and
the :meth:`_make_interface` private method.
Args:
name (str): The |Descriptor| name.
Attributes:
name (str): The |Descriptor| name.
"""
__interface_schema__ = schema.Schema(
schema.Or(
{
'name': schema.Regex(r'^[(),a-zA-Z_ ]+$'),
'type': 'categorical',
'property': schema.Regex(r'^[a-z][a-z_]+[a-z]$'),
'schema': _categorical_schema
}, {
'name': schema.Regex(r'^[(),a-zA-Z_ ]+$'),
'type': 'continuous',
'property': schema.Regex(r'^[a-z][a-z_]+[a-z]$'),
'schema': _continuous_schema
}))
def __init__(self, name):
self.name = name
@property
def property_name(self):
"""str: The |Record| inserted property name after :meth:`compute` call."""
return '_'.join((_camel_to_snake(self.__class__.__name__),
_camel_to_snake(self.name)))
@property
def __descriptor__(self): # noqa: D401
"""dict: A dictionary which summarizes the internal state of the |Descriptor| instance.
Keys:
name (str): The descriptor name. Note that in some |Descriptor|, it conjointly designates the
descriptor name **and** the name of the property read from each |Record| to describe it.
property (str): The name of the property written out by the |Descriptor| in each |Record|.
type (str): Either ``categorical`` or ``continuous``.
schema (dict, tuple, ColorMap): For ``categorical`` descriptors, it must be a dict mapping each
``category name`` to its relevant value (*e.g.* its index or |Color|). For ``continuous`` descriptors,
it must be a range description (*i.e.* either a ``(start, end)`` tuple or a object with a ``range``
attribute like a |ColorMap|).
See Also:
For more information on how :attr:`__descriptor__` is constructed, please see :ref:`descriptor`.
"""
interface = self._make_interface()
interface.update({
'name': self.name.title().replace('_', ' '),
'property': self.property_name
})
try:
self.__interface_schema__.validate(interface)
except schema.SchemaError as e:
raise ValueError('Invalid __descriptor__ returned:\n'
'{}\n{}'.format(e.errors, e.autos))
return interface
def __repr__(self):
"""Return a python representation of the |Descriptor|."""
return '{}(name={})'.format(self.__class__.__name__, self.name)
def __str__(self):
"""Return the |Descriptor| representation."""
return self.__repr__()
def __eq__(self, other):
"""Return ``True`` if two |Descriptor| have the same attributes and class."""
if self.__class__ == other.__class__:
return dict_equal(self.__dict__, other.__dict__)
return False
def __ne__(self, other):
"""Return ``True`` if two |Descriptor| do not have the same attributes and class."""
return not self == other
@abc.abstractmethod
def update(self, *record_collections):
"""Update internal values from |Record| descriptions.
Args:
*record_collections (|RecordCollection|): |RecordCollection| of which |Record| will be used to update
internals.
"""
raise NotImplementedError
@abc.abstractmethod
def compute(self, *record_collections):
"""Construct new |RecordCollection| where each enclosed |Record| is added a named description property.
Args:
*record_collections (|RecordCollection|): |RecordCollection| used to construct new |RecordCollection| with
described |Record|.
Returns:
(|RecordCollection|, ): A described |RecordCollection| tuple.
"""
raise NotImplementedError
@abc.abstractmethod
def reset(self):
"""Reset |Descriptor| internals to factory values."""
raise NotImplementedError
@abc.abstractmethod
def _make_interface(self):
"""Build a summary of hte internal state of the |Descriptor|.
It is used as a unified interface for all descriptors to describe and explicit their description.
As an example, for a typical |CategoricalDescriptor|, the interface dictionary would contain:
* The |Descriptor| name
* The |Descriptor| property name
* The |Descriptor| type, *i.e.* ``'categorical'`` here
* The category name to category index mapping.
As for a typical |ContinuousDescriptor|, it would contain:
* The |Descriptor| name
* The |Descriptor| property name
* The |Descriptor| type, *i.e.* ``'continuous'`` here
* The value range.
See Also:
The :attr:`__descriptor__` for more information.
Returns:
dict: A valid |Descriptor| :attr:`__descriptor__`.
"""
raise NotImplementedError
class RecipeConfigureFeature(RecipeFeature):
"""Recipe feature for configuring root trees built during a recipe root
feature"""
NAME = "configure"
FEATURED_ATTRIBUTES = {"configure"}
SCHEMA = schema.Schema({
"sdk": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
"configure": {
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"root": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
"steps": schema.And([str], len),
},
str: object, # do not consider other keys
})
def configure(self) -> None:
"""TODO"""
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["sdk"]), "sdk")
# the recipe to configure (i.e. what is the recipe for which we need to
# get image-resulting root tree on which running the configuration
# steps?):
recipe_to_configure = recipe.Recipe(
self.replace_placeholders(self.recipe.config["configure"]["root"]))
# check that image action for the recipe designated by "root" has well
# been run:
image_out_subpath = os.path.join(recipe_to_configure.out_subpath,
"image")
image_out_path = os.path.join(repo_root_path(), image_out_subpath)
if not os.path.exists(os.path.join(image_out_path, "root")):
raise RecipeActionError(
line("""
Could not process a configure action step if the image action
step from the recipe designated by "configure.root" has not
been run before.
"""))
action_out_subpath = os.path.join(self.recipe.out_subpath, "configure")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Always clear out the previous configure result:
if os.path.exists(action_out_path):
info("Clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
# Retrieve the result of the image action to work on it for this
# configuration step:
with ElevatedPrivileges():
info(
line("""Copying resulting root of the image action step for the
proper recipe ({!r}) into the working environment for the
configure action step...""".format(
recipe_to_configure.identifier)))
# WARNING!! Do not use `shutil.copytree()` function for the action
# that follows as this function uses `shutil.copy2()` by default,
# which itself uses `shutil.copystat()`. However, `copystat()`
# copies all node metadata **EXCEPT** the user and mode
# ownerships!!! See the Warning block on top of the page
# https://docs.python.org/3/library/shutil.html.
# Therefore, let's use the good old "cp -a", which seems to be
# available for all flavors of "cp" (but GNU cp is advised for
# this) and seems way quicker than `copytree()`. A pure Python
# workaround would have been too time-consuming.
run([
"cp", "-a",
os.path.join(image_out_path, "root"),
os.path.join(action_out_path, "root")
])
with sdk.session(action_name="configure",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["configure"]
["env"].items()
},
terminal=terminal,
shared_host_netns=False) as sess:
for cmd in self.recipe.config["configure"]["steps"]:
info("{!r} configures recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
def _check_config_file(config):
"""For non-deployment editions, we must load config from a json5 file.
This also gets run for deployments, as this function is responsible for
merging multiple configs into the single root config.
This function also contains all schema validation of json5 files, as
the version stored in the observatory image is separate.
"""
global CONFIG_FOLDER, IMAGE_TAG
# Not a deployment -- requires CONFIG_FOLDER
if CONFIG_FOLDER is None:
assert config is not None
CONFIG_FOLDER = config
else:
assert config is None
import pyjson5, schema as s
config_data = pyjson5.load(open(os.path.join(CONFIG_FOLDER,
'config.json5')))
# Before applying schema, merge in child configs
for child_name in os.listdir(CONFIG_FOLDER):
child_path = os.path.join(CONFIG_FOLDER, child_name)
if not os.path.isdir(child_path):
continue
child_config_path = os.path.join(child_path, 'config.json5')
if not os.path.lexists(child_config_path):
continue
child_config = pyjson5.load(open(child_config_path))
# First traversal -- patch keys and values
nodes = [([], child_config)]
while nodes:
path, src = nodes.pop()
for k, v in src.items():
## Rewrite v
# Check executables; amend cwd
if path and (
'file_detail_views' == path[-1]
or 'decision_views' == path[-1]
or 'parsers' == path[-1]
or 'tasks' == path[-1]):
if 'cwd' in v:
v['cwd'] = f'{child_name}/' + v['cwd']
else:
v['cwd'] = child_name
# Docker build stage patch
if path == ['build', 'stages']:
if 'commands' in v:
v['commands'] = [
vv
.replace('{disttarg}', f'{{disttarg}}/{child_name}')
.replace('{dist}', f'{{dist}}/{child_name}')
for vv in v['commands']]
if isinstance(v, dict):
nodes.append((path + [k], v))
# Second traversal -- merge
nodes = [([], config_data, child_config)]
while nodes:
path, dst, src = nodes.pop()
for k, v in src.items():
## Rewrite k
# Docker build stage patch -- rewrites `k`
if path == ['build', 'stages']:
# Adding a build stage. If not 'base' or 'final', then
# prepend config folder name
if k not in ['base', 'final']:
k = f'{child_name}_{k}'
# New entries only for these
if path in [
['pipelines'],
['file_detail_views'],
['decision_views'],
['parsers'],
]:
# Amend these with the child's name, to allow for copying
k = f'{child_name.replace("_", "-")}-{k}'
assert k not in dst, f'Cannot extend {path} {k}; must be new'
## Check for merge type
# Check if new -- if so, assign and be done
if k not in dst:
# Non-existent key; add it to the dictionary
dst[k] = v
continue
# Do merge
if isinstance(v, dict):
if not isinstance(dst[k], dict):
raise ValueError(f'{path} {k} type does not match base config')
# Dictionary merge
nodes.append((path + [k], dst[k], v))
elif isinstance(v, list):
if not isinstance(dst[k], list):
raise ValueError(f'{path} {k} type does not match base config')
# Add to end.
dst[k].extend(v)
else:
raise ValueError(f'May not extend {path} {k}: base config type {dst[k]}')
# Pull in parser-specific schema
import importlib.util
spec = importlib.util.spec_from_file_location('etl_parse',
os.path.join(faw_dir, 'common', 'pdf-etl-parse', 'parse_schema.py'))
etl_parse = importlib.util.module_from_spec(spec)
spec.loader.exec_module(etl_parse)
schema_views = {
s.Optional('decision_views', default={}): s.Or({}, {
str: {
'label': str,
'type': 'program',
'exec': [s.Or(
s.And(str, lambda x: not x.startswith('<')),
s.And(str, lambda x: x in [
'<filesPath>', '<apiInfo>', '<jsonArguments>', '<mongo>', '<outputHtml>',
'<workbenchApiUrl>']),
)],
s.Optional('cwd', default='.'): str,
'execStdin': s.And(str, lambda x: all([
y.group(0) in ['<referenceDecisions>', '<statsbyfile>']
for y in re.finditer('<[^>]*>', x)]),
error="Must be string with any <'s being one of: "
"<referenceDecisions>, <statsbyfile>"),
},
}),
s.Optional('file_detail_views', default={}): s.Or({}, {
str: {
'label': str,
'type': 'program_to_html',
'exec': [str],
s.Optional('cwd', default='.'): str,
s.Optional('outputMimeType', default='text/html'): str,
},
}),
}
# NOTE -- primary schema validation is here, but NOT for submodules such
# as pdf-etl-parse.
sch = s.Schema({
'name': s.And(str, s.Regex(r'^[a-zA-Z0-9-]+$')),
# parsers validated by pdf-etl-parse
'parsers': etl_parse.schema_get(),
s.Optional('parserDefaultTimeout', default=30): s.Or(float, int),
'decision_default': str,
s.Optional('pipelines', default={}): s.Or({}, {
s.And(str, lambda x: '_' not in x and '.' not in x,
error='Must not have underscore or dot'): {
s.Optional('label'): str,
s.Optional('disabled', default=False): s.Or(True, False),
s.Optional('tasks', default={}): s.Or({},
s.And(
{
s.And(str, lambda x: '_' not in x and '.' not in x,
error='Must not have underscore or dot'): {
s.Optional('disabled', default=False): s.Or(True, False),
'version': str,
'exec': [str],
s.Optional('cwd', default='.'): str,
s.Optional('dependsOn', default=[]): [str],
},
},
lambda x: all([d in x for _, task in x.items() for d in task['dependsOn']]),
error="Task `dependsOn` had invalid name",
)),
s.Optional('parsers', default={}): etl_parse.schema_get(),
**schema_views,
},
}),
'build': {
'stages': {
str: {
s.Optional('from'): str,
s.Optional('commands'): [str],
s.Optional('copy_output'): {
str: s.Or(str, bool),
},
},
},
},
**schema_views,
})
try:
config_data = sch.validate(config_data)
except Exception as e:
traceback.print_exc()
sys.stderr.write('\n'.join([str(v) for v in e.autos]) + '\n')
sys.exit(1)
IMAGE_TAG = config_data['name']
return config_data
class RecipeRootFeature(RecipeFeature):
"""Recipe feature for building rootfs from sources and binary packages.
Provides the 'run' attribute for development and debug."""
NAME = "root"
FEATURED_ATTRIBUTES = {"build", "image", "run"}
SCHEMA = schema.Schema({
"sdk": schema.Regex(RECIPE_IDENTIFIER_RE.pattern),
"root": {
schema.Optional("env", default={}): {
schema.Regex(ENVVAR_FORMAT_RE.pattern,
error="Bad environment variable name"):
str
},
"build_steps": [str],
"image_steps": schema.And([str], len),
},
str: object, # do not consider other keys
})
def build(self,
clear_cache: bool = False,
clear_previous_build: bool = True) -> None:
"""Build a root tree for this recipe from sources and binary packages
cache.
:param clear_cache: whether or not the cache has to clear before
processing root build
:param clear_previous_build: whether or not the previous root build
result has to be cleared
"""
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["sdk"]), "sdk")
action_out_subpath = os.path.join(self.recipe.out_subpath, "build")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Clear out the previous image result when told so:
if os.path.exists(action_out_path) and clear_previous_build:
info("Clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
if not os.path.exists(action_out_path):
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
if clear_cache:
info("Clearing cache for recipe {!r}...".format(
self.recipe.identifier))
with ElevatedPrivileges():
shutil.rmtree(
os.path.join(repo_root_path(), self.recipe.cache_subpath))
with sdk.session(action_name="build",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["root"]
["env"].items()
},
terminal=terminal,
shared_host_netns=True) as sess:
for cmd in self.recipe.config["root"]["build_steps"]:
info("{!r} builds recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
def image(self) -> None:
"""Build a root tree for this recipe only from binary packages cache
and without any build-time dependency."""
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["sdk"]), "sdk")
action_out_subpath = os.path.join(self.recipe.out_subpath, "image")
action_out_path = os.path.join(repo_root_path(), action_out_subpath)
# Always clear out the previous image result:
if os.path.exists(action_out_path):
info("Clearing {!r}...".format(action_out_path))
with ElevatedPrivileges():
shutil.rmtree(action_out_path)
# prepare output directory
os.makedirs(action_out_path)
# hook the SDK container process to the current TTY (if existent):
terminal = is_tty_attached()
with sdk.session(action_name="image",
action_targeted_recipe=self.recipe,
env={
key: self.replace_placeholders(value,
sdk_context=True)
for key, value in self.recipe.config["root"]
["env"].items()
},
terminal=terminal,
shared_host_netns=False) as sess:
for cmd in self.recipe.config["root"]["image_steps"]:
info("{!r} images recipe {!r}, runs:\n {}".format(
sdk.recipe.identifier, self.recipe.identifier, cmd))
sess.run(self.replace_placeholders(cmd, sdk_context=True))
def run(self, command: Optional[str] = None) -> None:
# using getattr to avoid static analyzers from complaining about
# missing attr (but brought by a recipe feature):
sdk = getattr(recipe.Recipe(self.recipe.config["sdk"]), "sdk")
sdk.interactive_run(
recipe=self.recipe,
command=command,
env={
key: self.replace_placeholders(value, sdk_context=True)
for key, value in self.recipe.config["root"]["env"].items()
},
)
from datetime import datetime
import schema as sm
from .client import HttpClient, ClientError
from .provider import ExchangeRatesProvider, ExchangeRateLoadError
from ..models import Currency, Rate
_POSSIBLE_CCY = [ccy.value for ccy in Currency]
_ccy_validator = sm.Or(*_POSSIBLE_CCY)
_rate_validator = sm.And(sm.Use(float), lambda n: n >= 0)
_date_validator = sm.Regex(r"\d{4}-\d{2}-\d{2}")
_ecb_rate_scm = sm.Schema({
"rates": {
_ccy_validator: _rate_validator
},
"base": _ccy_validator,
"date": _date_validator
})
class ECBProvider(ExchangeRatesProvider):
API_ENDPOINT_CONFIG_NAME = "API_ENDPOINT"
@classmethod
def create(cls, endpoint: str):
if not endpoint:
raise ValueError("API endpoint is required")
