def __init__(self, name, svn_root, svn_username, svn_password,
config_file):
self.name = name
self.svn_root = svn_root
self.svn_username = svn_username
self.svn_password = svn_password
self.client = Client()
self.client.callback_get_login = self._credentials
self.parser = ConfigParser()
self.config_file = config_file
def testPythonSvn():
log = logging.getLogger("testPythonSvn")
log.notice(
'If this script immediately exists with "Segmentation fault" or "Illegal instruction", see http://wiki/wiki/python-svn'
)
c = Client()
try:
c.mkdir('svn+ssh://svn/wm/sandbox/bug25523', 'testing mkdir')
except ClientError:
# already exists
pass
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" %
(svn_name, V8_SVN_URL))
else:
print(
"INFO: we will try to checkout and build a private v8 build from <%s>."
% V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r: return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print(
"INFO: we will try to use the system 'svn' command to checkout/update V8 code"
)
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
cmdline = ' '.join(args)
exec_cmd(cmdline, "checkout or update Google V8 code from SVN")
def makemovie(url, size):
client = Client()
images = []
r1 = ""
log = client.log(url)
versions = sorted([l['revision'].number for l in log])
for rev in versions:
(r2, diff) = getrev(client, url, rev)
if r2 and not diff:
#really wish I had multiple dispatch here
images.append(gen_one(r2, size))
elif r2 and diff:
images.extend(gen_diff(r1, r2, diff, size))
r1 = r2
raw_input('<enter> to continue')
def checkout_v8(self):
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r: return
print "ERROR: Failed to export from V8 svn repository"
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print "INFO: we will try to use the system 'svn' command to checkout/update V8 code"
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
try:
proc = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr)
proc.communicate()
if proc.returncode != 0:
print "WARN: fail to checkout or update Google v8 code from SVN, error code: ", proc.returncode
except Exception, e:
print "ERROR: fail to invoke 'svn' command, please install it first: %s" % e
sys.exit(-1)
def unrelease(project, version, options):
"""Undoes the project release for the specified project/version"""
log = logging.getLogger("unrelease")
client = Client()
# calculate version with and without patch number
matchPatch = re.compile(r'([1-9][0-9.]+)p([1-9][0-9.]*)')
m = matchPatch.match(version)
if m:
baseVersion = m.group(1)
fullVersion = version
patch = m.group(2)
else:
baseVersion = version
fullVersion = version
patch = None
basePath = '/ext/build/%s/%s' % (project, baseVersion)
releaseLinkPath = basePath + '/releases/' + fullVersion
if not os.path.exists(releaseLinkPath):
raise Exception('Could not find link to release %s for %s at %s' %
(version, project, releaseLinkPath))
# get canonical path
releasePath = os.path.realpath(releaseLinkPath)
# sanity check
if releasePath.find('releases') == -1:
raise Exception(
'Symlink %s in releases directory points to non-release path %s - something is wrong. You'
'll have to fix that manually.' % (releaseLinkPath, releasePath))
# Determine if this is SVN or git
scmgit = False
scmsvn = False
if os.path.exists(releasePath + '/git-checkout.log'):
scmgit = True
else:
if os.path.exists(releasePath + '/svn-checkout.log'):
scmsvn = True
else:
raise Exception('Could not determine SVN or GIT')
# get timestamp name
timestamp = getTimestamp(releasePath)
snapPath = basePath + '/snap/' + timestamp
# remove existing best link, decide whether to fix it at the end
# logic here is:
# - if we are told not to modify best, don't.
# - otherwise, if best link doesn't exists
# or if it exists and points to this build
# or if it points nowhere
# then fix it to point to the snapshot after we've moved it.
updateBest = False
bestLinkPath = basePath + '/best'
if options.best:
if (not os.path.exists(bestLinkPath)
or os.path.realpath(bestLinkPath) == releasePath
or (not os.path.exists(os.path.realpath(bestLinkPath)))):
log.info('removing old best link')
ssh('rm -f ' + bestLinkPath, options)
updateBest = True
log.info('Removing release links')
cmd = 'rm /ext/build/%s/%s/releases/%s' % (project, baseVersion,
fullVersion)
ssh(cmd, options)
cmd = 'rm /ext/build/%s/releases/%s' % (project, fullVersion)
ssh(cmd, options)
# get tag path
if scmsvn:
tagUrl = getFileContents(log, releasePath + '/TAG')
log.info("tag URL is '%s'" % (tagUrl))
# delete tag
log.info('deleting SVN tag')
msg = 'Removing tag from reversed release of project %s, version %s' % (
project, version)
escapedMessage = msg.replace("'", "\\'")
shell('svn rm %s -m \'%s\'' % (tagUrl, escapedMessage), options)
# write UNTAG file, remove TAG file
log.info('removing TAG file, writing to UNTAG file')
cmd = "rm %s/TAG ; echo '%s' >> %s/UNTAG" % (releasePath, tagUrl,
releasePath)
ssh(cmd, options)
if scmgit:
cmd = 'ssh git@git deltag ' + project + ' ' + version
try:
shell(cmd, options)
except ShellException, e:
raise Exception(
'Something went wrong with the deltag command. Check the log to see what the error was'
)
def release(project, version, build, options):
"""Release a project/version"""
log = logging.getLogger("release")
log.info('Beginning release')
client = Client()
# Check to make sure version will be legal as a tag
reg = re.compile('[-a-zA-Z0-9_.]+')
for x in range(len(version)):
if reg.match(version[x]) == None:
raise Exception(
"The version specified has characters that are not legal for setting a tag."
)
basePath = '/ext/build/' + project + '/' + version
# resolve path to snapshot
if (build == 'best'):
log.info("Resolving best link:")
else:
log.info("Resolving link to snapshot '%s':" % (build))
if build:
# try snapshot first
snapPath = basePath + '/snap/' + build
snapSource = 'SNAP'
if not os.path.exists(snapPath):
# try tinderbuilds
snapPath = basePath + '/tinderbuilds/' + build
snapSource = 'TINDERBUILD'
if not os.path.exists(snapPath):
# try base
snapPath = basePath + '/' + build
snapSource = 'BASE'
if not os.path.exists(snapPath):
raise Exception("Could not find specified build %s for %s %s" %
(build, project, version))
else:
snapPath = basePath + '/best'
snapSource = 'BEST'
if not os.path.exists(snapPath):
raise Exception(
'Could not find best link at %s - either create a best link or retry with "--build".'
% (snapPath))
if options.paranoid and snapSource == 'TINDERBUILD':
raise Exception(
'Specified build found at %s. Direct release of tinderbuild is not recommended. Rerun with "-sv" to snapshot your build first, or use "--relaxed" to disable this check.'
% snapPath)
# get canonical path
path = os.path.realpath(snapPath)
# parse timestamp
timestamp = getTimestamp(path)
# determine tag
project_xml = path + '/src/project.xml'
if not os.path.exists(project_xml):
project_xml = path + '/src/' + project + '_project/project.xml'
if not os.path.exists(project_xml):
raise Exception('Could not find project.xml at: ' + project_xml)
proj = Project(project_xml)
tag = version
patch = None
if proj.patch:
patch = proj.patch
if options.override_patch:
patch = options.override_patch
if patch:
# It would be reasonable / intuitive for a user to include a leading p in override_patch.
# Remove any leading ps or spaces to normalize expectations.
patch = patch.strip('p \t')
tag += 'p' + patch
if options.override_patch:
log.warn(
'using tag %s, including override patch number instead of value from project.xml (%s)'
% (tag, proj.patch))
# Determine if this is SVN or git
scmgit = False
scmsvn = False
if os.path.exists(path + '/git-checkout.log'):
scmgit = True
else:
if os.path.exists(path + '/svn-checkout.log'):
scmsvn = True
else:
raise Exception('Could not determine SVN or GIT')
# determine checkout revision and path
if scmsvn:
info = client.info(path + "/src")
rev = info.revision
svnpath = info.url.replace("svn://", "svn+ssh://")
if options.paranoid:
if svnpath.find(project) == -1:
raise Exception(
'The project you'
're trying to release doesn'
't contain its project name %s in its SVN path %s. Something is horribly wrong. (Use "--relaxed" to disable this check.)'
% (project, svnPath))
# Determine tag path
tagDir = 'svn+ssh://svn/wm/project/' + project + '/tags'
tagPath = tagDir + '/' + tag
# For git, make sure the repo is writable by the current user
if scmgit:
cmd = 'ssh git@git'
results = shell(cmd, options)
repo_match = False
repo_writable = False
for x in range(len(results)):
if results[x].lstrip('@_RW \t\r\n').rstrip('\r\n') == project:
repo_match = True
if results[x][9] == 'W':
repo_writable = True
else:
raise Exception(
'You dont have right to make changes to the repo for this project, therefore you cannot set the tag.'
)
# check that tag doesn't already exist
if not options.notag:
if scmsvn:
try:
client.ls(tagPath)
# tag exists already! Oops!
raise Exception(
'Tag path %s already exists! Halting. (You can use --notag to disable tag creation and this check.)'
% (tagPath))
except ClientError:
# this is expected, as we haven't created it yet.
pass
log.info("Releasing:")
log.info(" build: %s", path)
if scmsvn:
log.info(" svn: -r %d %s", rev.number, svnpath)
if scmgit:
log.info(" git")
log.info(" tag: %s", tag)
# Make release dirs
log.info("Ensuring that release directories exist")
releasesDir = basePath + '/releases'
allReleasesDir = '/ext/build/' + project + '/releases'
ssh('mkdir -p ' + releasesDir, options)
ssh('mkdir -p ' + allReleasesDir, options)
# Copy to releases
log.info("Moving snap to releases")
cmd = 'test -d ' + path + ' && mv ' + path + ' ' + releasesDir
ssh(cmd, options)
# Create soft links
log.info("Create soft links")
cmd = 'cd ' + releasesDir + '; test -e ' + tag + ' || ln -s ' + timestamp + ' ' + tag
ssh(cmd, options)
cmd = 'cd ' + allReleasesDir + '; test -e ' + tag + ' || ln -s ../' + version + '/releases/' + tag + ' ' + tag
ssh(cmd, options)
# recursively find dependencies and record
# this will overwrite any previous instances (e.g. from a snapshot)
findDepends(log, releasesDir + "/" + timestamp, options)
# Edit save file
log.info("Appending to SAVE file")
msg = '%s %s RELEASE' % (project, version)
writeSave(log, releasesDir + '/' + tag, msg, options)
# Create project tags directory
if scmsvn:
try:
client.ls(tagDir)
log.info("Project tags directory already exists")
except ClientError:
# does not exist, so create it
log.info("Creating project tags directory")
if not options.dryrun:
client.mkdir(
tagDir,
'release-project: make tag directory for ' + project)
if options.notag:
log.notice('skippping tag creation, as --notag is set')
# may want to write tag file, if svn path has a tag in it already.
if svnpath.find('/tags/') != -1:
log.notice(
'src svnpath is a tag already, writing that to TAG path')
cmd = 'cd ' + releasesDir + '/' + tag + '; echo ' + svnpath + ' >> TAG'
ssh(cmd, options)
else:
# Make tag
log.info("Tagging project")
log.info(" > svn copy -r%d %s %s", rev.number, svnpath, tagPath)
message = 'release-project: saving release tag'
def get_log_message():
return True, message
client.callback_get_log_message = get_log_message
if not options.dryrun:
client.copy(svnpath, tagPath, rev)
client.callback_get_log_message = None
# Log to TAG file
log.info("Creating TAG file")
cmd = 'cd ' + releasesDir + '/' + tag + '; echo ' + tagPath + ' >> TAG'
ssh(cmd, options)
#Create tag for git
if scmgit:
cmd = 'cd ' + releasesDir + '/' + tag + '/src && git show --format=oneline --summary | cut -d \' \' -f 1'
githash = shell(cmd, options)
cmd = 'ssh git@git addtag ' + project + ' ' + version + ' ' + githash[0]
try:
shell(cmd, options)
except ShellException, e:
raise Exception(
'Something went wrong with the addtag command. Check the log to see what the error was'
)
cmd = 'cd ' + releasesDir + '/' + tag + '/src && git fetch'
ssh(cmd, options)
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" %
(svn_name, V8_SVN_URL))
else:
print(
"INFO: we will try to checkout and build a private v8 build from <%s>."
% V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r:
print("%s Google V8 code (r%d) from SVN to %s" %
("Update" if update_code else "Checkout", r[-1].number,
V8_HOME))
with open(v8_svn_rev_file, 'w') as f:
f.write(str(r[-1].number))
return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print(
"INFO: we will try to use the system 'svn' command to checkout/update V8 code"
)
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
if exec_cmd(' '.join(args), "checkout or update Google V8 code from SVN"):
if not V8_SVN_REVISION:
ok, out, err = exec_cmd(
' '.join(["svn", "info", V8_HOME]),
"save the current V8 SVN revision to REVISION file",
output=True)
if ok:
with open(v8_svn_rev_file, 'w') as f:
f.write(
re.search(r'Revision:\s(?P<rev>\d+)', out,
re.MULTILINE).groupdict()['rev'])
else:
print("ERROR: fail to fetch SVN info, %s", err)
def _get_svn_info():
try:
from pysvn import Client, ClientError
try:
info = Client().info(".")
return info["url"].split("/")[-1], info["revision"].number
except ClientError:
# Not an svn working dir
#sys.stderr.write("""Hmm, doesn't look like an SVN repo""")
pass
except ImportError:
#sys.stderr.write(" * please consider installing pysvn\n")
#sys.stderr.write("""*** please install pysvn:
#- debian: sudo apt-get install python-svn
#- windows: http://pysvn.tigris.org/project_downloads.html
#***""")
pass
branch_name = None
revision = None
try:
platform = plat_id()
#
# svn executable is found, either linux or win with svn
#
if re.match('^linux.*', platform) or which('svn'):
svn_proc = subprocess.Popen(['svn','info'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
svn_stdout = svn_proc.communicate()[0]
if svn_proc.returncode == 0:
for l in re.split('(?:\r\n|\n)',svn_stdout):
m = re.match('([^:]+): (.*)',l)
if m:
if m.group(1) == 'URL':
url = m.group(2)
purl = urlparse(url)
(head,tail) = os.path.split(purl.path)
branch_name = tail
elif m.group(1) == 'Revision':
revision = m.group(2)
else:
#print 'nomatch', l
pass
#
# Try tortoise
#
elif re.match('^win.*', platform):
svn_proc = subprocess.Popen([SUBWCREV,'.'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
svn_stdout = svn_proc.communicate()[0]
if svn_proc.returncode == 0:
for l in re.split('(?:\r\n|\n)',svn_stdout):
m = re.match('Updated to revision (\d+)',l)
if m:
revision = m.group(1)
break
(head, tail) = os.path.split(os.getcwd())
branch_name = tail
except OSError, e:
#errmsg = sys.argv[0] + ': couldn\'t find branch name\n'
#sys.stderr.write(errmsg)
#sys.stderr.write(str(e))
pass
def __init__(self, repo):
self.repo = repo
self.client = Client()
def __init__(self, address):
self.conn = pymongo.Connection(host=address)
self.db = self.conn.ttengine
self.svnClient = Client()
def main(options):
"""docstring for main"""
client = Client()
client.callback_ssl_server_trust_prompt = ssl_trust_prompt
if (options.verbose > 1):
client.callback_notify = notify
retcode = 0
# check out branch from subversion
if (test_skip(options.branch_flags, update)):
getSourceTree(options.workingDir + os.sep + "branch",
options.branchUrl, client, options)
elif (options.verbose):
print "Skipping branch update"
# compile branch
if (test_skip(options.branch_flags, build)):
print "Compiling branch"
retcode = doCompile(options.workingDir + os.sep + "branch" + os.sep +
"VS2005")
elif (options.verbose):
print "Skipping branch compile"
# If compile successful, Run autotest
if (retcode != 0):
print "Failed to compile branch"
return retcode
if (test_skip(options.branch_flags, autotest)):
print "Running branch autotest"
retcode = doAutotest(options.workingDir + os.sep + "branch")
elif (options.verbose):
print "Skipping branch autotest"
if (retcode != 0):
print "Failed branch autotest"
return retcode
if (test_skip(options.trunk_flags, update)):
getSourceTree(options.workingDir + os.sep + "trunk", options.trunkUrl,
client, options)
elif (options.verbose):
print "Skipping trunk update"
# Determine the revision number of the head of the branch
if (options.endrev == 'HEAD'):
if (options.verbose):
print "Getting last revision number of branch"
messages = client.log(branchUrl,
revision_end=Revision(
pysvn.opt_revision_kind.number, startrev))
endrev = messages[0].revision.number
options.endrev = str(endrev)
if (options.verbose):
print "Revisions to merge", options.startrev + ":" + options.endrev
# Merge changes from branch to trunk
if (not options.skip_merge):
if (options.verbose):
print "Merging branch changes to trunk working directory"
(retcode, conflicts) = doMerge(options.workingDir + os.sep + "trunk",
endrev, options)
elif (options.verbose):
print "Skipping merge"
# How do we tell if there were merge errors?!?
if (retcode != 0):
print "Merge to working directory failed with conflicts."
printList(conflicts)
return retcode
# Compile trunk
if (test_skip(options.trunk_flags, build)):
print "Compiling trunk"
retcode = doCompile(options.workingDir + os.sep + "trunk" + os.sep +
"VS2005")
elif (options.verbose):
print "Skipping trunk compile"
if (retcode != 0):
print "Failed to compile merged trunk"
return retcode
# If compile successful, run autotest
if (test_skip(options.trunk_flags, autotest)):
print "Running trunk autotest"
retcode = doAutotest(options.workingDir + os.sep + "trunk")
elif (options.verbose):
print "Skipping trunk autotest"
if (retcode != 0):
print "Failed in merged autotest"
return retcode
# TODO: If successful, commit.
if (not options.skip_commit):
pass
# Write out new lastrev.py file
fp = open("lastrev.py", 'w')
fp.write("startrev = " + str(endrev) + "\n")
fp.write("prevrev = " + str(startrev))
fp.close()
