def mirror(name, repo):
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
repo_url = get_repo_url(repo)
try:
tar_file = get_repo_name(repo_url) + '.tar'
app.log(name, 'Try fetching tarball %s' % tar_file)
# try tarball first
with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
call(['wget', os.path.join(app.config['tar-url'], tar_file)])
call(['tar', 'xf', tar_file], stderr=fnull)
os.remove(tar_file)
call(['git', 'config', 'remote.origin.url', repo_url])
call(['git', 'config', 'remote.origin.mirror', 'true'])
if call(['git', 'config', 'remote.origin.fetch',
'+refs/*:refs/*']) != 0:
raise BaseException('Did not get a valid git repo')
call(['git', 'fetch', 'origin'])
except:
app.log(name, 'Try git clone from', repo_url)
with open(os.devnull, "w") as fnull:
if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
app.exit(name, 'ERROR: failed to clone', repo)
with app.chdir(tmpdir):
if call(['git', 'rev-parse']):
app.exit(name, 'ERROR: problem mirroring git repo at', tmpdir)
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
try:
os.rename(tmpdir, gitdir)
app.log(name, 'Git repo is mirrored at', gitdir)
except:
pass
def mirror(name, repo):
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
repo_url = get_repo_url(repo)
try:
os.makedirs(tmpdir)
tar_file = get_repo_name(repo_url) + '.tar'
app.log(name, 'Try fetching tarball %s' % tar_file)
# try tarball first
with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
call(['wget', os.path.join(app.config['tar-url'], tar_file)])
call(['tar', 'xf', tar_file])
os.remove(tar_file)
call(['git', 'config', 'remote.origin.url', repo_url])
call(['git', 'config', 'remote.origin.mirror', 'true'])
if call(['git', 'config', 'remote.origin.fetch',
'+refs/*:refs/*']) != 0:
raise BaseException('Did not get a valid git repo')
call(['git', 'fetch', 'origin'])
except:
app.log(name, 'Try git clone from', repo_url)
with open(os.devnull, "w") as fnull:
if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
app.exit(name, 'ERROR: failed to clone', repo)
with app.chdir(tmpdir):
if call(['git', 'rev-parse']):
app.exit(name, 'ERROR: problem mirroring git repo at', tmpdir)
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
try:
os.rename(tmpdir, gitdir)
app.log(name, 'Git repo is mirrored at', gitdir)
except:
pass
def mirror(name, repo):
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
repo_url = get_repo_url(repo)
try:
tar_file = get_repo_name(repo_url) + '.tar'
app.log(name, 'Try fetching tarball %s' % tar_file)
# try tarball first
with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
call(
['wget', os.path.join(app.config['tar-url'], tar_file)],
stdout=fnull,
stderr=fnull)
call(['tar', 'xf', tar_file], stderr=fnull)
os.remove(tar_file)
update_mirror(name, repo, tmpdir)
except:
app.log(name, 'Try git clone from', repo_url)
with open(os.devnull, "w") as fnull:
if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
app.log(name, 'Failed to clone', repo, exit=True)
with app.chdir(tmpdir):
if call(['git', 'rev-parse']):
app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True)
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
try:
shutil.move(tmpdir, gitdir)
app.log(name, 'Git repo is mirrored at', gitdir)
except:
pass
def mirror(name, repo):
tempfile.tempdir = app.config['tmp']
tmpdir = tempfile.mkdtemp()
repo_url = get_repo_url(repo)
try:
tar_file = get_repo_name(repo_url) + '.tar'
app.log(name, 'Try fetching tarball %s' % tar_file)
# try tarball first
with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
call(['wget', os.path.join(app.config['tar-url'], tar_file)])
call(['tar', 'xf', tar_file], stderr=fnull)
os.remove(tar_file)
update_mirror(name, repo, tmpdir)
except:
app.log(name, 'Try git clone from', repo_url)
with open(os.devnull, "w") as fnull:
if call(['git', 'clone', '--mirror', '-n', repo_url, tmpdir]):
app.log(name, 'Failed to clone', repo, exit=True)
with app.chdir(tmpdir):
if call(['git', 'rev-parse']):
app.log(name, 'Problem mirroring git repo at', tmpdir, exit=True)
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
try:
shutil.move(tmpdir, gitdir)
app.log(name, 'Git repo is mirrored at', gitdir)
except:
pass
def mirror(name, repo):
tempfile.tempdir = app.config["tmp"]
tmpdir = tempfile.mkdtemp()
repo_url = get_repo_url(repo)
try:
tar_file = get_repo_name(repo_url) + ".tar"
app.log(name, "Try fetching tarball %s" % tar_file)
# try tarball first
with app.chdir(tmpdir), open(os.devnull, "w") as fnull:
call(["wget", os.path.join(app.config["tar-url"], tar_file)])
call(["tar", "xf", tar_file], stderr=fnull)
os.remove(tar_file)
call(["git", "config", "remote.origin.url", repo_url])
call(["git", "config", "remote.origin.mirror", "true"])
if call(["git", "config", "remote.origin.fetch", "+refs/*:refs/*"]) != 0:
raise BaseException("Did not get a valid git repo")
call(["git", "fetch", "origin"])
except:
app.log(name, "Try git clone from", repo_url)
with open(os.devnull, "w") as fnull:
if call(["git", "clone", "--mirror", "-n", repo_url, tmpdir]):
app.exit(name, "ERROR: failed to clone", repo)
with app.chdir(tmpdir):
if call(["git", "rev-parse"]):
app.exit(name, "ERROR: problem mirroring git repo at", tmpdir)
gitdir = os.path.join(app.config["gits"], get_repo_name(repo))
try:
os.rename(tmpdir, gitdir)
app.log(name, "Git repo is mirrored at", gitdir)
except:
pass
def parse_files(self, directory):
schemas = self.load_schemas()
with chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
path = os.path.join(dirname, filename)
data = self._load(path)
if data is not None:
self.validate_schema(schemas, data)
data['path'] = path[2:]
self._fix_keys(data)
self._tidy_and_insert_recursively(data)
if config.get('mode') == 'parse-only':
with open(config['result-file'], 'w') as f:
f.write(
yaml.dump(self._data,
default_flow_style=False,
Dumper=ExplicitDumper))
log('RESULT', 'Parsed definitions data in yaml format is at',
config['result-file'])
os._exit(0)
def __init__(self, directory="."):
"""Load all definitions from a directory tree."""
self._definitions = {}
self._trees = {}
json_schema = self._load(app.config.get("json-schema"))
definitions_schema = self._load(app.config.get("defs-schema"))
if json_schema and definitions_schema:
import jsonschema as js
js.validate(json_schema, json_schema)
js.validate(definitions_schema, json_schema)
things_have_changed = not self._check_trees()
with app.chdir(directory):
for dirname, dirnames, filenames in os.walk("."):
filenames.sort()
dirnames.sort()
if ".git" in dirnames:
dirnames.remove(".git")
for filename in filenames:
if filename.endswith((".def", ".morph")):
contents = self._load(os.path.join(dirname, filename))
if contents is not None:
if things_have_changed and definitions_schema:
app.log(filename, "Validating schema")
js.validate(contents, definitions_schema)
self._fix_keys(contents)
self._tidy_and_insert_recursively(contents)
self.defaults = defaults.Defaults()
if self._check_trees():
for path in self._definitions:
self._definitions[path]["tree"] = self._trees.get(path)
def get_tree(this):
ref = this['ref']
gitdir = os.path.join(app.settings['gits'], get_repo_name(this['repo']))
if not os.path.exists(gitdir):
try:
url = (app.settings['cache-server-url'] + 'repo='
+ get_repo_url(this['repo']) + '&ref=' + ref)
with urllib2.urlopen(url) as response:
tree = json.loads(response.read().decode())['tree']
return tree
except:
app.log(this, 'WARNING: no tree from cache-server', ref)
mirror(this['name'], this['repo'])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
stderr=fnull):
# can't resolve this ref. is it upstream?
call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
try:
tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.log(this, 'ERROR: could not find tree for ref', ref)
raise SystemExit
def mirror(name, repo):
# try tarball first
gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
repo_url = get_repo_url(repo)
try:
os.makedirs(gitdir)
tar_file = get_repo_name(repo_url) + '.tar'
app.log(name, 'Try fetching tarball %s' % tar_file)
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
call(['wget', app['tar-url']], stdout=fnull, stderr=fnull)
call(['tar', 'xf', tar_file], stdout=fnull, stderr=fnull)
os.remove(tar_file)
call(['git', 'config', 'remote.origin.url', repo_url],
stdout=fnull, stderr=fnull)
call(['git', 'config', 'remote.origin.mirror', 'true'],
stdout=fnull, stderr=fnull)
if call(['git', 'config', 'remote.origin.fetch',
'+refs/*:refs/*'],
stdout=fnull, stderr=fnull) != 0:
raise BaseException('Did not get a valid git repo')
call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
except:
app.log(name, 'Using git clone from ', repo_url)
try:
with open(os.devnull, "w") as fnull:
call(['git', 'clone', '--mirror', '-n', repo_url, gitdir],
stdout=fnull, stderr=fnull)
except:
app.log(name, 'ERROR: failed to clone', repo)
raise SystemExit
app.log(name, 'Git repo is mirrored at', gitdir)
def checkout(name, repo, ref, checkout):
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
# checkout the required version of this from git
with open(os.devnull, "w") as fnull:
# We need to pass '--no-hardlinks' because right now there's nothing to
# stop the build from overwriting the files in the .git directory
# inside the sandbox. If they were hardlinks, it'd be possible for a
# build to corrupt the repo cache. I think it would be faster if we
# removed --no-hardlinks, though.
if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
stdout=fnull, stderr=fnull):
app.exit(name, 'ERROR: git clone failed for', ref)
with app.chdir(checkout):
if call(['git', 'checkout', '--force', ref], stdout=fnull,
stderr=fnull):
app.exit(name, 'ERROR: git checkout failed for', ref)
app.log(name, 'Git checkout %s in %s' % (repo, checkout))
app.log(name, 'Upstream version %s' % get_version(checkout, ref))
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkout)
def do_deployment_manifest(system, configuration):
app.log(system, "Creating deployment manifest in", system['sandbox'])
data = {'configuration': configuration}
metafile = os.path.join(system['sandbox'], 'baserock', 'deployment.meta')
with app.chdir(system['sandbox']), open(metafile, "w") as f:
json.dump(data, f, indent=4, sort_keys=True, encoding='unicode-escape')
f.flush()
def update_mirror(name, repo, gitdir):
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
app.log(name, 'Refreshing mirror for %s' % repo)
repo_url = get_repo_url(repo)
if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
stdout=fnull, stderr=fnull):
app.log(name, 'Git update mirror failed', repo, exit=True)
def checkout(name, repo, ref, checkout):
gitdir = os.path.join(app.config['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
elif not mirror_has_ref(gitdir, ref):
update_mirror(name, repo, gitdir)
# checkout the required version of this from git
with open(os.devnull, "w") as fnull:
# We need to pass '--no-hardlinks' because right now there's nothing to
# stop the build from overwriting the files in the .git directory
# inside the sandbox. If they were hardlinks, it'd be possible for a
# build to corrupt the repo cache. I think it would be faster if we
# removed --no-hardlinks, though.
if call(['git', 'clone', '--no-hardlinks', gitdir, checkout],
stdout=fnull, stderr=fnull):
app.exit(name, 'ERROR: git clone failed for', ref)
with app.chdir(checkout):
if call(['git', 'checkout', '--force', ref], stdout=fnull,
stderr=fnull):
app.exit(name, 'ERROR: git checkout failed for', ref)
app.log(name, 'Git checkout %s in %s' % (repo, checkout))
app.log(name, 'Upstream version %s' % get_version(checkout, ref))
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkout)
def get_tree(this):
ref = this['ref']
gitdir = os.path.join(app.config['gits'], get_repo_name(this['repo']))
if not os.path.exists(gitdir):
try:
url = (app.config['tree-server'] + 'repo=' +
get_repo_url(this['repo']) + '&ref=' + ref)
response = requests.get(url=url)
tree = response.json()['tree']
return tree
except:
app.log(this, 'WARNING: no tree from tree-server', ref)
mirror(this['name'], this['repo'])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
stderr=fnull):
# can't resolve this ref. is it upstream?
app.log(this, 'Fetching from upstream to resolve %s' % ref)
call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
try:
tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.exit(this, 'ERROR: could not find tree for ref', (ref, gitdir))
def __init__(self, directory='.'):
'''Load all definitions from a directory tree.'''
self._definitions = {}
self._trees = {}
schemas = self.load_schemas()
with app.chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
path = os.path.join(dirname, filename)
data = self._load(path)
if data is not None:
self.validate_schema(schemas, data)
data['path'] = path[2:]
self._fix_keys(data)
self._tidy_and_insert_recursively(data)
self.defaults = defaults.Defaults()
caches_are_valid = self._check_trees()
for path in self._definitions:
try:
this = self._definitions[path]
if this.get('ref') and self._trees.get(path):
if this['ref'] == self._trees.get(path)[0]:
this['tree'] = self._trees.get(path)[1]
except:
app.log('DEFINITIONS', 'WARNING: problem with .trees file')
pass
def set_origin_url(gitdir, checkout):
'''Sets the origin url of a checkout to that of its gitdir.
git-lfs requires a remote server in order to fetch binaries, so we set the
origin url to that of the mirror for lfs enabled checkouts.
'''
try:
with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
origin_url = check_output(
['git', 'config', '--get', 'remote.origin.url'], stderr=fnull)
with open(os.devnull, 'w') as fnull, app.chdir(checkout):
check_call(['git', 'config', 'remote.origin.url', origin_url],
stderr=fnull)
except:
app.log('UTILS', 'Setting origin url failed for', checkout, exit=True)
raise
def get_tree(dn):
ref = dn['ref']
gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo']))
if dn['repo'].startswith('file://') or dn['repo'].startswith('/'):
gitdir = dn['repo'].replace('file://', '')
if not os.path.isdir(gitdir):
app.log(dn, 'Git repo not found:', dn['repo'], exit=True)
if not os.path.exists(gitdir):
try:
params = {'repo': get_repo_url(dn['repo']), 'ref': ref}
r = requests.get(url=app.config['tree-server'], params=params)
return r.json()['tree']
except:
if app.config.get('tree-server'):
app.log(dn, 'WARNING: no tree from tree-server for', ref)
mirror(dn['name'], dn['repo'])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
stderr=fnull):
# can't resolve ref. is it upstream?
app.log(dn, 'Fetching from upstream to resolve %s' % ref)
update_mirror(dn['name'], dn['repo'], gitdir)
try:
tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.log(dn, 'No tree for ref', (ref, gitdir), exit=True)
def get_tree(this):
ref = this["ref"]
gitdir = os.path.join(app.config["gits"], get_repo_name(this["repo"]))
if this["repo"].startswith("file://") or this["repo"].startswith("/"):
gitdir = this["repo"].replace("file://", "")
if not os.path.isdir(gitdir):
app.exit(this, "ERROR: git repo not found:", this["repo"])
if not os.path.exists(gitdir):
try:
url = app.config["tree-server"] + "repo=" + get_repo_url(this["repo"]) + "&ref=" + ref
response = requests.get(url=url)
tree = response.json()["tree"]
return tree
except:
if app.config.get("tree-server"):
app.log(this, "WARNING: no tree from tree-server for", ref)
mirror(this["name"], this["repo"])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(["git", "rev-parse", ref + "^{object}"], stdout=fnull, stderr=fnull):
# can't resolve this ref. is it upstream?
app.log(this, "Fetching from upstream to resolve %s" % ref)
call(["git", "fetch", "origin"], stdout=fnull, stderr=fnull)
try:
tree = check_output(["git", "rev-parse", ref + "^{tree}"], universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.exit(this, "ERROR: could not find tree for ref", (ref, gitdir))
def do_manifest(this):
metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
with app.chdir(this['install']), open(metafile, "w") as f:
f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
f.write('elapsed_time: %s\n' % app.elapsed(this['start-time']))
f.flush()
call(['find'], stdout=f, stderr=f)
def update_mirror(name, repo, gitdir):
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
app.log(name, 'Refreshing mirror for %s' % repo)
repo_url = get_repo_url(repo)
if call(['git', 'fetch', repo_url, '+refs/*:refs/*', '--prune'],
stdout=fnull,
stderr=fnull):
app.log(name, 'Git update mirror failed', repo, exit=True)
def checkout(this):
_checkout(this['name'], this['repo'], this['ref'], this['build'])
with app.chdir(this['build']):
if os.path.exists('.gitmodules') or this.get('submodules'):
checkout_submodules(this)
utils.set_mtime_recursively(this['build'])
def do_manifest(this):
metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
with app.chdir(this['install']), open(metafile, "w") as f:
f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
f.flush()
call(['find'], stdout=f, stderr=f)
copyfile(metafile, os.path.join(app.settings['artifacts'],
this['cache'] + '.meta'))
def get_last_tag(gitdir):
try:
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
tag = check_output(['git', 'describe', '--abbrev=0',
'--tags', 'HEAD'], stderr=fnull)[0:-1]
return tag
except:
return None
def checkout(dn):
_checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
with app.chdir(dn['checkout']):
if os.path.exists('.gitmodules') or dn.get('submodules'):
checkout_submodules(dn)
utils.set_mtime_recursively(dn['checkout'])
def checkout(this):
_checkout(this['name'], this['repo'], this['ref'], this['build'])
with app.chdir(this['build']):
if os.path.exists('.gitmodules') or this.get('submodules'):
checkout_submodules(this)
utils.set_mtime_recursively(this['build'])
def checkout(dn):
_checkout(dn['name'], dn['repo'], dn['ref'], dn['checkout'])
with app.chdir(dn['checkout']):
if os.path.exists('.gitmodules') or dn.get('submodules'):
checkout_submodules(dn)
utils.set_mtime_recursively(dn['checkout'])
def do_manifest(this):
metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
with app.chdir(this['install']), open(metafile, "w") as f:
f.write("repo: %s\nref: %s\n" % (this.get('repo'), this.get('ref')))
f.flush()
call(['find'], stdout=f, stderr=f)
copyfile(metafile,
os.path.join(app.config['artifacts'], this['cache'] + '.meta'))
def get_last_tag(gitdir):
try:
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
tag = check_output(
['git', 'describe', '--abbrev=0', '--tags', 'HEAD'],
stderr=fnull)[0:-1]
return tag
except:
return None
def ref_expects_lfs(gitdir, ref):
'''Parses .gitattributes at a ref to determine if git-lfs is required.'''
with open(os.devnull, 'w') as fnull, app.chdir(gitdir):
blob = check_output(
['git', 'ls-tree', ref, '.gitattributes'], stderr=fnull)
if blob:
attributes = check_output(
['git', 'cat-file', 'blob', blob.split()[2]], stderr=fnull)
return bool(re.search('filter=lfs.*-text', attributes))
return False
def save_trees(self):
with app.chdir(app.settings['defdir']):
checksum = check_output('ls -lRA */', shell=True)
checksum = hashlib.md5(checksum).hexdigest()
self.__trees = {'.checksum': checksum}
for name in self.__definitions:
if self.__definitions[name].get('tree') is not None:
self.__trees[name] = self.__definitions[name]['tree']
with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
f.write(yaml.dump(self.__trees, default_flow_style=False))
def get_version(gitdir, ref="HEAD"):
try:
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
described = check_output(["git", "describe", "--tags", "--dirty"], stderr=fnull)[0:-1]
last_tag = check_output(["git", "describe", "--abbrev=0", "--tags", ref], stderr=fnull)[0:-1]
commits = check_output(["git", "rev-list", last_tag + ".." + ref, "--count"])[0:-1]
result = "%s %s (%s + %s commits)" % (ref[:8], described, last_tag, commits)
except:
result = ref[:8] + " (No tag found)"
return result
def detect_format(source):
with app.chdir(source):
for dirname, dirnames, filenames in os.walk('.'):
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith('.morph'):
return 'baserock-morphologies'
for filename in filenames:
if filename.endswith('.cida'):
return 'cida-definitions'
return None
def get_version(gitdir, ref='HEAD'):
try:
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
version = check_output(['git', 'describe', '--tags', '--dirty'],
stderr=fnull)[0:-1]
tag = check_output(['git', 'describe', '--abbrev=0',
'--tags', ref], stderr=fnull)[0:-1]
commits = check_output(['git', 'rev-list', tag + '..' + ref,
'--count'])[0:-1]
result = "%s %s (%s + %s commits)" % (ref[:8], version, tag, commits)
except:
result = ref[:8] + " (No tag found)"
return result
def get_upstream_version(repo, ref):
try:
gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
last_tag = check_output(['git', 'describe', '--abbrev=0',
'--tags', ref], stderr=fnull)[0:-1]
commits = check_output(['git', 'rev-list', last_tag + '..' + ref,
'--count'])
result = "%s (%s + %s commits)" % (ref[:8], last_tag, commits[0:-1])
except:
result = ref[:8] + " " + "(No tag found)"
return result
def get_version(gitdir, ref='HEAD'):
try:
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
described = check_output(['git', 'describe', '--tags', '--dirty'],
stderr=fnull)[0:-1]
last_tag = check_output(['git', 'describe', '--abbrev=0',
'--tags', ref], stderr=fnull)[0:-1]
commits = check_output(['git', 'rev-list', last_tag + '..' + ref,
'--count'])[0:-1]
result = "%s %s (%s + %s commits)" % (ref[:8], described, last_tag,
commits)
except:
result = ref[:8] + " (No tag found)"
return result
def parse_files(self, directory):
with chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
path = os.path.join(dirname, filename)
data = self._load(path)
if data is not None:
data['path'] = self._demorph(path[2:])
self._fix_keys(data)
self._tidy_and_insert_recursively(data)
def _check_trees(self):
try:
with app.chdir(app.settings['defdir']):
checksum = check_output('ls -lRA */', shell=True)
checksum = hashlib.md5(checksum).hexdigest()
with open('.trees') as f:
text = f.read()
self.__trees = yaml.safe_load(text)
if self.__trees.get('.checksum') == checksum:
return True
except:
if os.path.exists('.trees'):
os.remove('.trees')
self.__trees = {}
return False
def save_trees(self):
'''Creates the .trees file for the current working directory
.trees contains a list of git trees for all the definitions, and a
checksum for the state of the working subdirectories
'''
with app.chdir(app.config['defdir']):
checksum = check_output('ls -lRA */', shell=True)
checksum = hashlib.md5(checksum).hexdigest()
self._trees = {'.checksum': checksum}
for name in self._definitions:
if self._definitions[name].get('tree') is not None:
self._trees[name] = self._definitions[name]['tree']
with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
f.write(yaml.dump(self._trees, default_flow_style=False))
def save_trees(self):
"""Creates the .trees file for the current working directory
.trees contains a list of git trees for all the definitions, and a
checksum for the state of the working subdirectories
"""
with app.chdir(app.config["defdir"]):
checksum = check_output("ls -lRA */", shell=True)
checksum = hashlib.md5(checksum).hexdigest()
self._trees = {".checksum": checksum}
for name in self._definitions:
if self._definitions[name].get("tree") is not None:
self._trees[name] = self._definitions[name]["tree"]
with open(os.path.join(os.getcwd(), ".trees"), "w") as f:
f.write(yaml.dump(self._trees, default_flow_style=False))
def checkout(name, repo, ref, checkoutdir):
gitdir = os.path.join(app.settings['gits'], get_repo_name(repo))
if not os.path.exists(gitdir):
mirror(name, repo)
app.log(name, 'Upstream version:', get_upstream_version(repo, ref))
app.log(name, 'Git checkout %s in %s' % (repo, checkoutdir))
# checkout the required version of this from git
with app.chdir(checkoutdir), open(os.devnull, "w") as fnull:
copy_repo(gitdir, checkoutdir)
if call(['git', 'checkout', ref], stdout=fnull, stderr=fnull):
app.log(name, 'ERROR: git checkout failed for', ref)
raise SystemExit
if os.path.exists('.gitmodules'):
checkout_submodules(name, ref)
utils.set_mtime_recursively(checkoutdir)
def save_trees(self):
'''Creates the .trees file for the current working directory
.trees contains a list of git trees for all the definitions, and a
checksum for the state of the working subdirectories
'''
with app.chdir(app.config['defdir']):
checksum = check_output('ls -lRA */', shell=True)
checksum = hashlib.md5(checksum).hexdigest()
self._trees = {'.checksum': checksum}
for name in self._definitions:
if self._definitions[name].get('tree') is not None:
self._trees[name] = [
self._definitions[name]['ref'],
self._definitions[name]['tree']
]
with open(os.path.join(os.getcwd(), '.trees'), 'w') as f:
f.write(yaml.safe_dump(self._trees, default_flow_style=False))
def log_changes(dn, tmpdir, old_defs, ref):
do_git_log = False
old_def = old_defs.get(dn['path'])
log_file = os.path.join(tmpdir, dn['name'])
with open(log_file, 'w') as f:
keys = set(dn) - set(['tree', 'cache'])
for key in keys:
try:
old_value = old_def.get(key)
except:
old_value = None
if dn[key] != old_value:
f.write('[%s] Value changed: %s\n' % (dn['path'], key))
if type(dn[key]) is str:
f.write('%s | %s\n' % (old_value, dn[key]))
if type(dn[key]) is not str and type(dn[key]) is not float:
if old_value:
for x in old_value:
f.write(repr(x))
f.write('\n vvv\n')
if dn[key]:
for x in dn[key]:
f.write(repr(x))
f.write('\n\n')
if dn.get('kind', 'chunk') == 'chunk' and config.get('release-cmd'):
log(dn, 'Logging git change history', tmpdir)
try:
gitdir = os.path.join(config['gits'],
get_repo_name(dn['repo']))
if not os.path.exists(gitdir):
mirror(dn['name'], dn['repo'])
elif not mirror_has_ref(gitdir, ref):
update_mirror(dn['name'], dn['repo'], gitdir)
with chdir(gitdir):
text = dn['ref'] + '..'
if old_def and old_def.get('ref'):
text += old_def['ref']
f.write(check_output(config['release-command'] + [text]))
except:
log(dn, 'WARNING: Failed to log git changes')
if os.stat(log_file).st_size == 0:
os.remove(log_file)
def log_changes(dn, tmpdir, old_defs, ref):
do_git_log = False
old_def = old_defs.get(dn['path'])
log_file = os.path.join(tmpdir, dn['name'])
with open(log_file, 'w') as f:
keys = set(dn) - set(['tree', 'cache'])
for key in keys:
try:
old_value = old_def.get(key)
except:
old_value = None
if dn[key] != old_value:
f.write('[%s] Value changed: %s\n' % (dn['path'], key))
if type(dn[key]) is str:
f.write('%s | %s\n' % (old_value, dn[key]))
if type(dn[key]) is not str and type(dn[key]) is not float:
if old_value:
for x in old_value:
f.write(repr(x))
f.write('\n vvv\n')
if dn[key]:
for x in dn[key]:
f.write(repr(x))
f.write('\n\n')
if dn.get('kind', 'chunk') == 'chunk' and config['release-command']:
log(dn, 'Logging git change history', tmpdir)
try:
gitdir = os.path.join(config['gits'],
get_repo_name(dn['repo']))
if not os.path.exists(gitdir):
mirror(dn['name'], dn['repo'])
elif not mirror_has_ref(gitdir, ref):
update_mirror(dn['name'], dn['repo'], gitdir)
with chdir(gitdir):
text = dn['ref'] + '..'
if old_def and old_def.get('ref'):
text += old_def['ref']
f.write(check_output(config['release-command'] + [text]))
except:
log(dn, 'WARNING: Failed to log git changes')
if os.stat(log_file).st_size == 0:
os.remove(log_file)
def run_extension(this, deployment, step, method):
app.log(this, 'Running %s extension:' % step, method)
extensions = utils.find_extensions()
tempfile.tempdir = tmp = app.settings['tmp']
cmd_tmp = tempfile.NamedTemporaryFile(delete=False)
cmd_bin = extensions[step][method]
if method == 'ssh-rsync':
envlist = ['UPGRADE=yes']
else:
envlist = ['UPGRADE=no']
if 'PYTHONPATH' in os.environ:
envlist.append('PYTHONPATH=%s:%s' % (os.environ['PYTHONPATH'],
app.settings['extsdir']))
else:
envlist.append('PYTHONPATH=%s' % app.settings['extsdir'])
for key, value in deployment.iteritems():
if key.isupper():
envlist.append("%s=%s" % (key, value))
command = ["env"] + envlist + [cmd_tmp.name]
if step in ('write', 'configure'):
command.append(this['sandbox'])
if step in ('write', 'check'):
command.append(deployment['location'])
with app.chdir(app.settings['defdir']):
try:
with open(cmd_bin, "r") as infh:
shutil.copyfileobj(infh, cmd_tmp)
cmd_tmp.close()
os.chmod(cmd_tmp.name, 0o700)
if call(command):
app.log(this, 'ERROR: %s extension failed:' % step, cmd_bin)
raise SystemExit
finally:
os.remove(cmd_tmp.name)
return
def write_chunk_metafile(chunk):
'''Writes a chunk .meta file to the baserock dir of the chunk
The split rules are used to divide up the installed files for the chunk
into artifacts in the 'products' list
'''
log(chunk['name'], 'Splitting', chunk.get('kind'))
rules, splits = compile_rules(chunk)
with chdir(chunk['install']):
for root, dirs, files in os.walk('.', topdown=False):
for name in files + dirs:
path = os.path.join(root, name)[2:]
for artifact, rule in rules:
if rule.match(path) or rule.match(path + '/'):
splits[artifact].append(path)
break
write_metafile(rules, splits, chunk)
def write_chunk_metafile(chunk):
'''Writes a chunk .meta file to the baserock dir of the chunk
The split rules are used to divide up the installed files for the chunk
into artifacts in the 'products' list
'''
log(chunk['name'], 'Splitting', chunk.get('kind'))
rules, splits = compile_rules(chunk)
with chdir(chunk['install']):
for root, dirs, files in os.walk('.', topdown=False):
for name in files + dirs:
path = os.path.join(root, name)[2:]
for artifact, rule in rules:
if rule.match(path) or rule.match(path + '/'):
splits[artifact].append(path)
break
write_metafile(rules, splits, chunk)
def do_manifest(defs, this):
metafile = os.path.join(this['baserockdir'], this['name'] + '.meta')
metadata = {}
metadata['repo'] = this.get('repo')
metadata['ref'] = this.get('ref')
kind = this.get('kind', 'chunk')
if kind == 'chunk':
metadata['products'] = splitting.do_chunk_splits(defs, this, metafile)
elif kind == 'stratum':
metadata['products'] = splitting.do_stratum_splits(defs, this)
if metadata.get('products'):
defs.set_member(this['path'], '_artifacts', metadata['products'])
with app.chdir(this['install']), open(metafile, "w") as f:
yaml.safe_dump(metadata, f, default_flow_style=False)
copyfile(metafile, os.path.join(app.config['artifacts'],
this['cache'] + '.meta'))
def __init__(self, directory='.'):
'''Load all definitions from a directory tree.'''
self._definitions = {}
self._trees = {}
json_schema = self._load(app.config.get('json-schema'))
definitions_schema = self._load(app.config.get('defs-schema'))
if json_schema and definitions_schema:
import jsonschema as js
js.validate(json_schema, json_schema)
js.validate(definitions_schema, json_schema)
things_have_changed = not self._check_trees()
with app.chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
contents = self._load(os.path.join(dirname, filename))
if contents is not None:
if things_have_changed and definitions_schema:
app.log(filename, 'Validating schema')
js.validate(contents, definitions_schema)
self._fix_keys(contents)
self._tidy_and_insert_recursively(contents)
self.defaults = defaults.Defaults()
caches_are_valid = self._check_trees()
for path in self._definitions:
try:
this = self._definitions[path]
if this.get('ref') and self._trees.get(path):
if this['ref'] == self._trees.get(path)[0]:
this['tree'] = self._trees.get(path)[1]
except:
app.log('DEFINITIONS', 'WARNING: problem with .trees file')
pass
def run_extension(dn, deployment, step, method):
app.log(dn, 'Running %s extension:' % step, method)
extensions = utils.find_extensions()
tempfile.tempdir = app.config['tmp']
cmd_tmp = tempfile.NamedTemporaryFile(delete=False)
cmd_bin = extensions[step][method]
envlist = ['UPGRADE=yes'] if method == 'ssh-rsync' else ['UPGRADE=no']
if 'PYTHONPATH' in os.environ:
envlist.append('PYTHONPATH=%s:%s' %
(os.environ['PYTHONPATH'], app.config['extsdir']))
else:
envlist.append('PYTHONPATH=%s' % app.config['extsdir'])
for key, value in deployment.iteritems():
if key.isupper():
envlist.append("%s=%s" % (key, value))
command = ["env"] + envlist + [cmd_tmp.name]
if step in ('write', 'configure'):
command.append(dn['sandbox'])
if step in ('write', 'check'):
command.append(
deployment.get('location') or deployment.get('upgrade-location'))
with app.chdir(app.config['defdir']):
try:
with open(cmd_bin, "r") as infh:
shutil.copyfileobj(infh, cmd_tmp)
cmd_tmp.close()
os.chmod(cmd_tmp.name, 0o700)
if call(command):
app.log(dn, 'ERROR: %s extension failed:' % step, cmd_bin)
raise SystemExit
finally:
os.remove(cmd_tmp.name)
return
def parse_files(self, directory):
with chdir(directory):
for dirname, dirnames, filenames in os.walk('.'):
filenames.sort()
dirnames.sort()
if '.git' in dirnames:
dirnames.remove('.git')
for filename in filenames:
if filename.endswith(('.def', '.morph')):
path = os.path.join(dirname, filename)
data = self._load(path)
if data is not None:
data['path'] = self._demorph(path[2:])
self._fix_keys(data)
self._tidy_and_insert_recursively(data)
for x in self._data:
dn = self._data[x]
for field in dn:
if field not in self.fields:
log(dn, 'Invalid field "%s" in' % field, dn['path'],
exit=True)
def make_deterministic_tar_archive(base_name, root):
'''Make a tar archive of contents of 'root_dir'.
This function takes extra steps to make the output more deterministic,
compared to shutil.make_archive() - it sorts the results to ensure
the ordering of the files in the archive is always the same.
Also this puts the directory last, to workaround a bug in docker/overlayfs
runners - see https://gitlab.com/baserock/ybd/issues/241
FIXME: make this do timestamps
'''
with app.chdir(root), open(base_name + '.tar', 'wb') as f:
with tarfile.TarFile(mode='w', fileobj=f) as f_tar:
directories = [d[0] for d in os.walk('.')]
for d in sorted(directories):
files = [os.path.join(d, f) for f in os.listdir(d)]
for path in sorted(files):
f_tar.add(name=path, recursive=False)
f_tar.add(name=d, recursive=False)
def _check_trees(self):
'''True if the .trees file matches the current working subdirectories
The .trees file lists all git trees for a set of definitions, and a
checksum of the checked-out subdirectories when we calculated them.
If the checksum for the current subdirectories matches, return True
'''
try:
with app.chdir(app.config['defdir']):
checksum = check_output('ls -lRA */', shell=True)
checksum = hashlib.md5(checksum).hexdigest()
with open('.trees') as f:
text = f.read()
self._trees = yaml.safe_load(text)
if self._trees.get('.checksum') == checksum:
return True
except:
self._trees = {}
return False
def get_tree(this):
ref = this['ref']
gitdir = os.path.join(app.config['gits'], get_repo_name(this['repo']))
if this['repo'].startswith('file://') or this['repo'].startswith('/'):
gitdir = this['repo'].replace('file://', '')
if not os.path.isdir(gitdir):
app.exit(this, 'ERROR: git repo not found:', this['repo'])
if not os.path.exists(gitdir):
try:
url = (app.config['tree-server'] + 'repo=' +
get_repo_url(this['repo']) + '&ref=' + ref)
response = requests.get(url=url)
tree = response.json()['tree']
return tree
except:
if app.config.get('tree-server'):
app.log(this, 'WARNING: no tree from tree-server for', ref)
mirror(this['name'], this['repo'])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(['git', 'rev-parse', ref + '^{object}'], stdout=fnull,
stderr=fnull):
# can't resolve this ref. is it upstream?
app.log(this, 'Fetching from upstream to resolve %s' % ref)
call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
try:
tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.exit(this, 'ERROR: could not find tree for ref', (ref, gitdir))
def get_tree(dn):
ref = str(dn['ref'])
gitdir = os.path.join(app.config['gits'], get_repo_name(dn['repo']))
if dn['repo'].startswith('file://') or dn['repo'].startswith('/'):
gitdir = dn['repo'].replace('file://', '')
if not os.path.isdir(gitdir):
app.log(dn, 'Git repo not found:', dn['repo'], exit=True)
if not os.path.exists(gitdir):
try:
params = {'repo': get_repo_url(dn['repo']), 'ref': ref}
r = requests.get(url=app.config['tree-server'], params=params)
return r.json()['tree']
except:
if app.config.get('tree-server'):
app.log(dn, 'WARNING: no tree from tree-server for', ref)
mirror(dn['name'], dn['repo'])
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
if call(['git', 'rev-parse', ref + '^{object}'],
stdout=fnull,
stderr=fnull):
# can't resolve ref. is it upstream?
app.log(dn, 'Fetching from upstream to resolve %s' % ref)
update_mirror(dn['name'], dn['repo'], gitdir)
try:
tree = check_output(['git', 'rev-parse', ref + '^{tree}'],
universal_newlines=True)[0:-1]
return tree
except:
# either we don't have a git dir, or ref is not unique
# or ref does not exist
app.log(dn, 'No tree for ref', (ref, gitdir), exit=True)
def source_date_epoch(checkout):
with app.chdir(checkout):
return check_output(['git', 'log', '-1', '--pretty=%ct'])[:-1]
def fetch(repo):
with app.chdir(repo), open(os.devnull, "w") as fnull:
call(['git', 'fetch', 'origin'], stdout=fnull, stderr=fnull)
def update_mirror(name, repo, gitdir):
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
app.log(name, 'Refreshing mirror for %s' % repo)
if call(['git', 'remote', 'update', 'origin'], stdout=fnull,
stderr=fnull):
app.exit(name, 'ERROR: git update mirror failed', repo)
def mirror_has_ref(gitdir, ref):
with app.chdir(gitdir), open(os.devnull, "w") as fnull:
out = call(['git', 'cat-file', '-t', ref], stdout=fnull, stderr=fnull)
return out == 0