def pull(self, options):
repo = options['repo']
if repo.startswith("file://./"):
repo = repo.replace('.', os.path.abspath('.'), 1)
findrefs = False
result = None
if 'local' in options:
localrepo = options['local']
_, name = os.path.split(localrepo)
else:
_, localrepo = os.path.split(repo)
name = localrepo
localrepo = os.path.join(
'./',
self.env.env['RESULTS'],
localrepo )
fetchRef = None
if 'fetch-ref' in options:
fetchRef = options['fetch-ref']
ref, reftype = GitProvider.splitRepoReference(fetchRef)
self.log.debug("Starting Ref: %s, Reftype: %s", ref, reftype)
GitProvider.fetchRepository(
self.log,
name,
localrepo,
repo,
ref,
reftype,
secure=False )
repoObject = git.Repo(localrepo)
remote, _ = GitProvider.ensureCsmakeRemote(repoObject, repo)
startsha = GitProvider.getSHAFromRef(
self.log,
repoObject,
remote,
ref,
reftype)
if startsha is None:
self.log.error("The reference %s could not be resolved from the given repo", ref)
self.log.failed()
return None
parent = repoObject.commit(startsha)
while True:
if len(parent.parents) == 0:
break
parent = parent.parents[0]
result = parent.hexsha
self.env.addTransPhase(options['env'], result)
self.log.passed()
return result
def default(self, options):
local = options["local"]
name = options['name']
(name, localRepo) = GitProvider.determineRepoPath(local, name)
if 'env' in options:
self.env.env[options['env']] = localRepo
self.log.passed()
def clean(self, options):
local = self.env.doSubstitutions(options["local"])
name = options['name']
(name, path) = GitProvider.determineRepoPath(local, name)
if 'env' in options:
self.env.env[options['env']] = path
try:
shutil.rmtree(path)
except Exception as e:
self.log.info("Could not remove path '%s': %s", path, repr(e))
self.log.passed()
def pull(self, options):
self.log.debug("Options are: %s", str(options))
#TODO: Do check for required parameters
local = options["local"]
name = options['name']
(name, localRepo) = GitProvider.determineRepoPath(local, name)
reporef = None
if 'ref' in options:
reporef = options['ref']
(ref, reftype) = GitProvider.splitRepoReference(reporef)
if ref is None:
msg = "Reference '%s' is not understood" % ref
self.log.error(msg)
self.log.failed()
return (False, "ERROR", msg)
ref = self.env.doSubstitutions(ref)
url = options['URL']
if url.startswith("file://./"):
url = url.replace('.', os.path.abspath('.'), 1)
secure = False
if 'secure' in options:
secure = options['secure'] == 'True'
if 'env' in options:
self.env.env[options['env']] = localRepo
return GitProvider.fetchRepository(self.log,
name,
localRepo,
url,
ref,
reftype,
secure=secure)
def git(innerself):
info = "Build: %s" % build
delta = False
url = None
newref = None
oldref = None
if 'REF' not in old or 'REF' not in new:
info += ", a manifest was missing the git SHA1" % build
delta = True
elif old['REF'] != new['REF']:
if 'URL' in new:
url = new['URL']
elif 'URL' in old:
url = old['URL']
if url is None:
info += ", there is no URL to access the git repo"
delta = True
newref = new['REF']
oldref = old['REF']
else:
info += ", SHA1's are the same"
if 'BRANCH' not in old or 'BRANCH' not in new \
or old['BRANCH'] != new['BRANCH']:
info += ", The branch changed from '%s' to '%s'" % (
old['BRANCH'], new['BRANCH'])
delta = True
#get the sha delta from GitProvider
if delta and url is not None:
_, name = os.path.split(url)
localrepo = os.path.join(self.gitstash, name)
result = None
try:
result = GitProvider.generateSpanLogInfo(
self.log, localrepo, oldref, newref, self.findrefs)
except Exception as e:
self.log.info(
"Span log threw '%s' attempting fetch and trying again",
str(e))
try:
GitProvider.fetchRepository(self.log,
name,
localrepo,
url,
newref,
None,
env=self.shellenv,
secure=False)
result = GitProvider.generateSpanLogInfo(
self.log, localrepo, oldref, newref,
self.findrefs)
except:
self.log.exception(
"Attempt to generate git changelog failed")
info += ", Could not get git changelog due to failures"
if result is not None:
if 'changelog' not in records:
records['changelog'] = {}
records['changelog'][build] = result
if delta:
records['__info'].append(info)
return delta
def build(self, options):
dibenv = self.env.env['__DIBEnv__']
overrides = dibenv['source-repository-overrides']
self.log.devdebug("Overrides will be for: %s", str(overrides))
scriptFilter = re.compile(r'^source-repository-.*[^~]$')
#Gather all the source-repository sections
hooksdir = dibenv['hooksdir']
filelist = os.listdir(hooksdir)
repofiles = []
for fileitem in filelist:
tested = scriptFilter.match(fileitem)
if tested is not None:
currentRepo = os.path.join(hooksdir, fileitem)
if os.path.isfile(currentRepo):
repofiles.append(currentRepo)
#Dig through and modify source-repo definitions
sourceRepos = []
repoEntry = ('name', 'type', 'local', 'URL', 'ref')
self.log.devdebug("Source repo lines are: %s", str(repofiles))
for repopath in repofiles:
repolines = None
try:
with open(repopath) as repofile:
repolines = repofile.readlines()
except:
self.log.exception("source-repository %s failed to open",
repopath)
self.log.failed()
return None
if repolines is None:
self.log.error("source-repository %s failed to open", repopath)
for repoline in repolines:
repoline = repoline.strip()
if len(repoline) == 0 or repoline.startswith('#'):
self.log.warning(
"Empty line or comment character found in source-repository %s",
repopath)
self.log.warning(
"The (scant) source-repositories documentation makes no mention that these are acceptable in a source-repository spec. YMMV"
)
continue
repoparts = repoline.split()
if len(repoparts) == 4:
repoparts.append('*')
if len(repoparts) != 5:
self.log.error(
"source-repository file '%s' did not provide all of the required data: %s",
repoline)
#TODO: look up the element if getfattr is installed
self.log.failed()
return None
repo = dict(zip(repoEntry, repoparts))
self.log.info("Default source-repository '%s': %s", repopath,
str(repoparts))
if 'default-url-base' in options:
defaultURL = self.env.doSubstitutions(
options['default-url-base'])
self.log.info("substituting %s in place of base for %s",
defaultURL, repo['URL'])
parsed = urlparse.urlsplit(repo['URL'])
defaultParsed = urlparse.urlsplit(defaultURL)
fixup = urlparse.urlunsplit(defaultParsed[:2] + parsed[2:])
self.log.info("--- new url is: %s", fixup)
repo['URL'] = fixup
reponame = repo['name']
altreponame = reponame.replace('-', '_')
self.log.devdebug("Repo name to check: '%s'", reponame)
self.log.devdebug("Alternate repo name to check: '%s'",
altreponame)
self.log.devdebug(" Current repo: %s", str(repo))
repokey = None
if reponame in overrides:
repokey = reponame
elif altreponame in overrides:
repokey = altreponame
if repokey is not None:
self.log.devdebug("Name '%s' was in overrides", repokey)
override = overrides[repokey]
self.log.info("Overriding source-repository: %s",
str(override))
if 'URLbase' in override:
self.log.info("overriding with base %s for %s",
override['URLbase'], repo['URL'])
parsed = urlparse.urlsplit(repo['URL'])
overrideParsed = urlparse.urlsplit(override['URLbase'])
fixup = urlparse.urlunsplit(overrideParsed[:2] +
parsed[2:])
self.log.info('--- new url is: %s', fixup)
repo['URL'] = fixup
repo.update(override)
else:
self.log.devdebug("~~~Name was not in overrides~~~")
sourceRepos.append(repo)
reposCompleted = []
if dibenv['logconfig'].has_section('DIBGetSourceRepositories'):
reposCompleted = dibenv['logconfig'].options(
'DIBGetSourceRepositories')
else:
dibenv['logconfig'].add_section('DIBGetSourceRepositories')
#Run through all the parts
tempdir = None
failed = False
try:
for source in sourceRepos:
if failed and not self.settings['keep-going']:
self.log.info(
"Bailing out of source repo grabbing because of failure"
)
break
self.flowcontrol.initFlowControlIssue(
"doNotStartScript",
"Tells DIBGetSourceRepositories to not run a script")
self.flowcontrol.initFlowControlIssue(
"tryScriptAgain",
"Tells DIBGetSourceRepositories to try the script again")
self.flowcontrol.initFlowControlIssue(
"doNotAvoidScript",
"Tells DIBGetSourceRepositories to ignore build avoidance")
self.flowcontrol.initFlowControlIssue(
"doNotDeleteTemp",
"Tells DIBGetSourceRepositories to leave the temp file")
#Find all aspects for each step like RunParts
sourceName = source['name']
aspects = DIBRunParts.lookupAspects(options, sourceName)
self.log.devdebug("Processing source-repository: %s",
sourceName)
if sourceName in reposCompleted:
self.engine.launchAspects(aspects, 'script_skip', 'build',
self, options,
{'source': source})
if not self.flowcontrol.advice("doNotAvoidScript"):
self.log.devdebug(
"----Avoiding repo '%s' as completed", sourceName)
continue
else:
self.log.devdebug(
"++++Repeating repo '%s' even though completed",
sourceName)
self.engine.launchAspects(aspects, 'script_start', 'build',
self, options, {'source': source})
if self.flowcontrol.advice("doNotStartScript"):
self.log.info(
"==== Loading the repo was preempted by an aspect")
continue
pullResult = None
deleteMe = None
tryagain = True
while tryagain:
try:
tryagain = False
imageTargetPath, targetRepo = os.path.split(
source['local'])
#Root slash messes with os.path.join and other functions
if imageTargetPath[0] == '/':
imageTargetPath = imageTargetPath[1:]
targetRoot = self.env.env['TARGET_ROOT']
targetPath = os.path.join(targetRoot, imageTargetPath)
self.log.devdebug("targetPath for injection: %s",
targetPath)
self.log.devdebug("imageTargetPath: %s",
imageTargetPath)
self.log.devdebug("targetRepo: %s", targetRepo)
tempdir = tempfile.mkdtemp()
self.log.devdebug("tempdir: %s", tempdir)
#Create any parts that weren't already created
if not os.path.exists(targetPath):
result = subprocess.call(
['sudo', 'mkdir', '-p', targetPath])
if result != 0:
self.log.error(
"Could not make directory for repo")
failed = True
continue
#GIT REPO
if source['type'] == 'git':
ref = source['ref']
if ref == '*':
ref = 'master'
self.log.info("Fetching %s from git: %s:%s",
sourceName, source['URL'], ref)
reftype = None
if 'reftype' in source:
reftype = source['reftype']
name, localRepo = GitProvider.determineRepoPath(
tempdir, targetRepo)
pullResult = GitProvider.fetchRepository(
self.log, sourceName, localRepo, source['URL'],
ref, reftype, dibenv['shellenv'])
passed, status, msg = pullResult
if not passed:
self.log.error(msg)
failed = True
continue
#NOTE: Believe it or not pbr barfs if you do this
#....sad.
#shutil.rmtree(
# os.path.join(
# tempdir,
# targetRepo,
# '.git' ) )
result = subprocess.call([
'sudo', '-E', 'rsync', '-a',
'--remove-source-files', localRepo, targetPath
],
stdout=self.log.out(),
stderr=self.log.err(),
env=dibenv['shellenv'])
if result != 0:
self.log.error("GIT repo failed to inject")
failed = True
continue
#TAR REPO
elif source['type'] == 'tar':
temptarball = tempfile.mkdtemp() + "/temp.tgz"
self.log.info("Fetching %s from tar: %s:%s",
sourceName, source['URL'],
source['ref'])
result = subprocess.call(
'sudo -E curl %s -o %s' %
(source['URL'], temptarball),
shell=True,
stdout=self.log.out(),
stderr=self.log.err(),
env=dibenv['shellenv'])
if result != 0:
self.log.error("TAR repo failed to download")
failed = True
continue
sha = None
with file(temptarball) as myfile:
sha = self._fileSHA1(myfile)
downloadTime = datetime.utcnow().isoformat()
result = subprocess.call(
'sudo -E tar -C %s -xzf %s' %
(tempdir, temptarball),
shell=True,
stdout=self.log.out(),
stderr=self.log.err(),
env=dibenv['shellenv'])
if result != 0:
self.log.error("TAR repo failed to download")
failed = True
continue
repofiles = []
sourcepull = os.path.join(tempdir, source['ref'],
'.')
targetdrop = os.path.join(targetPath, targetRepo)
proc = subprocess.Popen([
'sudo', '-E', 'rsync', '-av',
'--remove-source-files', sourcepull, targetdrop
],
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
env=dibenv['shellenv'])
out, err = proc.communicate()
if proc.returncode != 0:
self.log.error("TAR repo failed to inject")
failed = True
continue
repofiles = []
lines = out.split('\n')
start = False
for line in lines:
line = line.strip()
if not start:
start = line == './'
continue
else:
if len(line) == 0:
break
repofiles.append(line)
pullResult = (targetdrop, repofiles, sha,
downloadTime, temptarball)
deleteMe = temptarball
#FILE
elif source['type'] == 'file':
self.log.info("Pulling '%s' as a file", sourceName)
filepath = os.path.join(targetPath, targetRepo)
cmd = [
'sudo', '-E', 'curl', source['URL'], '-o',
filepath
]
self.log.devdebug('File command is: %s', str(cmd))
if 'http_proxy' in dibenv['shellenv']:
self.log.debug(
"http_proxy is: %s",
dibenv['shellenv']['http_proxy'])
if 'HTTPS_PROXY' in dibenv['shellenv']:
self.log.debug(
"HTTPS_PROXY is: %s",
dibenv['shellenv']['HTTPS_PROXY'])
if 'NO_PROXY' in dibenv['shellenv']:
self.log.debug("NO_PROXY is: %s",
dibenv['shellenv']['NO_PROXY'])
result = subprocess.call(cmd,
stdout=self.log.out(),
stderr=self.log.err(),
env=dibenv['shellenv'])
if result != 0:
self.log.error(
"FILE repo failed to download and inject")
failed = True
continue
sha = None
with file(filepath) as myfile:
sha = self._fileSHA1(myfile)
pullResult = (filepath, sha,
datetime.utcnow().isoformat())
else:
self.log.error("Incorrect repo type")
failed = True
continue
except Exception as e:
self.log.exception(
"Repo failed to load with exception")
self.engine.launchAspects(aspects, 'script_exception',
'build', self, options, {
'__ex__': e,
'source': source
})
tryagain = self.flowcontrol.advice("tryScriptAgain")
if not tryagain:
failed = True
self.log.info("Not trying repo again")
raise e
finally:
if not tryagain:
if failed:
self.engine.launchAspects(
aspects, "script_failed", 'build', self,
options, {'source': source})
self.log.error("Repo '%s' failed with errors",
sourceName)
tryagain = self.flowcontrol.advice(
"tryScriptAgain")
else:
dibenv['logconfig'].set(
'DIBGetSourceRepositories', sourceName, '')
self.engine.launchAspects(
aspects, "script_passed", "build", self,
options, {
'source': source,
'pulled': pullResult
})
tryagain = self.flowcontrol.advice(
"tryScriptAgain")
try:
if deleteMe is not None:
result = subprocess.call(
['sudo', '-E', 'rm', '-rf', deleteMe],
stdout=self.log.out(),
stderr=self.log.err(),
env=dibenv['shellenv'])
deleteMe = None
if result != 0:
raise ValueError("Result: %d" % result)
except:
self.log.exception(
"Failed to delete temporary tar: %s",
deleteMe)
self.engine.launchAspects(aspects, 'script_end', 'build', self,
options, {'source': source})
except:
failed = True
self.log.exception("Source repo loading exited on exception")
finally:
if tempdir is not None and \
not self.flowcontrol.advice('doNotDeleteTemp'):
shutil.rmtree(tempdir)
with file(dibenv['logfile'], 'w') as f:
dibenv['logconfig'].write(f)
if failed:
self.log.failed()
else:
self.log.passed()
return not failed
def build(self, options):
if self.yieldsfiles is None:
self.log.warning(
"**yields-files was not specified in section. Nothing to do..."
)
self.log.warning(" No changelog will be produced")
self.log.passed()
return None
filespecs = []
for index in self.yieldsfiles:
location = index['location']
if not location.startswith('/') and \
not location.startswith('./'):
location = os.path.join(self.env.env['RESULTS'],
index['location'])
filespecs.append((location, index['type']))
repoURL = options['repo']
findrefs = False
nochange = False
nochangeString = "Unknown reason"
if len(options['old']) == 0 or options['old'] == '<unknown>':
self.log.info("There is no valid previous version")
nochangeString = "No valid previous version"
nochange = True
if options['old'] == options['new']:
self.log.info("The old and new versions are the same, no change")
nochangeString = "Old and new versions identical"
nochange = True
if nochange:
self._formatOutput(None, filespecs, {'errortext': nochangeString})
self.log.passed()
return True
if 'local' in options:
localrepo = options['local']
_, name = os.path.split(localrepo)
else:
_, localrepo = os.path.split(repoURL)
name = localrepo
localrepo = os.path.join(self.env.env['RESULTS'], localrepo)
newref, newreftype = GitProvider.splitRepoReference(options['new'])
oldref, oldreftype = GitProvider.splitRepoReference(options['old'])
try:
repo = git.Repo(localrepo)
except git.exc.NoSuchPathError:
GitProvider.fetchRepository(self.log,
name,
localrepo,
repoURL,
newref,
newreftype,
secure=False)
repo = git.Repo(localrepo)
remote, _ = GitProvider.ensureCsmakeRemote(repo, repoURL)
try:
new = GitProvider.getSHAFromRef(self.log, repo, remote, newref,
newreftype)
old = GitProvider.getSHAFromRef(self.log, repo, remote, oldref,
oldreftype)
except Exception as e:
self.log.info(
"getSHA threw '%s' attempting to lookup ref, retrying", str(e))
GitProvider.fetchRepository(self.log,
name,
localrepo,
repo,
newref,
newreftype,
secure=False)
new = GitProvider.getSHAFromRef(self.log, repo, remote, newref,
newreftype)
old = GitProvider.getSHAFromRef(self.log, repo, remote, oldref,
oldreftype)
try:
result = GitProvider.generateSpanLogInfo(self.log, localrepo, old,
new, findrefs)
except Exception as e:
self.log.info(
"Span log threw '%s' attempting to fetch and try again",
str(e))
try:
GitProvider.fetchRepository(self.log,
name,
localrepo,
repo,
newref,
newreftype,
secure=False)
result = GitProvider.generateSpanLogInfo(
self.log, localrepo, old, new, findrefs)
except:
self.log.exception("Attempt to generate git changelog failed")
self.log.failed()
return None
self.log.devdebug("result == %s", str(result))
self._formatOutput(result, filespecs, {'URL': repoURL})
self.log.passed()
def pull(self, options):
repo = options['repo']
if repo.startswith("file://./"):
repo = repo.replace('.', os.path.abspath('.'), 1)
findrefs = False
result = None
if 'local' in options:
localrepo = options['local']
_, name = os.path.split(localrepo)
else:
_, localrepo = os.path.split(repo)
name = localrepo
localrepo = os.path.join(
'./',
self.env.env['RESULTS'],
localrepo )
fetchRef = None
if 'fetch-ref' in options:
fetchRef = options['fetch-ref']
ref, reftype = GitProvider.splitRepoReference(fetchRef)
self.log.debug("Ref: %s, Reftype: %s", ref, reftype)
GitProvider.fetchRepository(
self.log,
name,
localrepo,
repo,
ref,
reftype,
secure=False )
repoObject = git.Repo(localrepo)
dateOfCurrent = repoObject.commit().authored_date
tagList = []
self.log.debug("Current date: %d", dateOfCurrent)
for tag in repoObject.tags:
#Is the tag a lightweight object?
if tag.object.__class__ is git.Commit:
#Yes. Deal with the missing data.
tagObject = {
'name':str(tag),
'commit':tag.object,
'date':tag.object.authored_date,
'authored_date':tag.object.authored_date,
'tag_sha':tag.object.hexsha,
'commit_sha':tag.object.hexsha
}
else:
tagObject = {
'name':str(tag),
'commit':tag.object.object,
'date':tag.object.tagged_date,
'authored_date':tag.object.object.authored_date,
'tag_sha':tag.object.hexsha,
'commit_sha':tag.object.object.hexsha
}
self.log.debug(
"Tag: %s, Date: %d",
tagObject['name'],
tagObject['authored_date'])
if fnmatch.fnmatch(tagObject['name'], options['format']) \
and tagObject['authored_date'] <= dateOfCurrent:
tagList.append(tagObject)
if len(tagList) == 0:
self.log.failed()
self.log.error("No tags found")
return self.default(options)
tagList.sort(key=lambda x: x['date'], reverse=True)
tag = tagList[0]
self.log.info("Tag found: %s (%s, %s)", tag['name'], tag['tag_sha'], tag['commit_sha'])
result = tag['commit_sha']
self.env.addTransPhase(options['env'], result)
self.log.passed()
return result