def main():
global port
global root_path
global username
global password
global client
if len(sys.argv) <= 2:
print "You need to specify the port to serve on and the URL under"
print "which files will be looked up, respectively."
sys.exit()
client = Client()
client.callback_get_login = get_login
port = int(sys.argv[1])
root_path = sys.argv[2]
if root_path[-1] == "/":
root_path = root_path[:-1]
if len(sys.argv) <= 4:
username = None
password = None
else:
username = sys.argv[3]
password = sys.argv[4]
print "Port: " + str(port)
print "Root path: " + root_path
print "Username: "******"Password: "******"*" * len(password)) if
isinstance(password, basestring) else password)
print
print "svnweb has successfully started up."
HTTPServer(("", port), HTTPHandler).serve_forever()
def create_repo(self, repo):
client = Client()
client.callback_get_login = self.get_login(repo)
client.callback_ssl_server_trust_prompt = self.get_trust(repo)
return client
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" % (svn_name, V8_SVN_URL))
else:
print("INFO: we will try to checkout and build a private v8 build from <%s>." % V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number, V8_SVN_REVISION) if V8_SVN_REVISION else Revision(opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r:
print("%s Google V8 code (r%d) from SVN to %s" % ("Update" if update_code else "Checkout", r[-1].number, V8_HOME))
with open(v8_svn_rev_file, 'w') as f:
f.write(str(r[-1].number))
return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print("INFO: we will try to use the system 'svn' command to checkout/update V8 code")
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
if exec_cmd(' '.join(args), "checkout or update Google V8 code from SVN"):
if not V8_SVN_REVISION:
ok, out, err = exec_cmd(' '.join(["svn", "info", V8_HOME]),
"save the current V8 SVN revision to REVISION file", output=True)
if ok:
with open(v8_svn_rev_file, 'w') as f:
f.write(re.search(r'Revision:\s(?P<rev>\d+)', out, re.MULTILINE).groupdict()['rev'])
else:
print("ERROR: fail to fetch SVN info, %s", err)
def __init__(self, name, svn_root, svn_username, svn_password,
config_file):
self.name = name
self.svn_root = svn_root
self.svn_username = svn_username
self.svn_password = svn_password
self.client = Client()
self.client.callback_get_login = self._credentials
self.parser = ConfigParser()
self.config_file = config_file
class VersionningComponent(Component):
API_VERSION = 1
NAME = 'versionning'
VERSION = '1.0'
REQUIRES = ()
def __init__(self):
Component.__init__(self)
self.repositories = {}
self.client = Client()
def init(self, core):
vardir = core.config.get('CORE', 'vardir')
self.repositories_base = join(vardir, "repositories")
self.checkout_base = join(vardir, "versionned")
for directory in (self.repositories_base, self.checkout_base):
if not exists(directory):
mkdir(directory)
def getRepository(self, name):
if name not in self.repositories:
self.createOrLoadRepository(name)
return self.repositories[name]
def createOrLoadRepository(self, name):
repository_directory = join(self.repositories_base, name)
checkout_directory = join(self.checkout_base, name)
must_create = not exists(repository_directory)
assert must_create != exists(checkout_directory)
if must_create:
self.createRepository(repository_directory, checkout_directory)
repository = Repository(self, checkout_directory, client=self.client)
if must_create:
repository.update()
self.repositories[name] = repository
def createRepository(self, repository_directory, checkout_directory):
process = createProcess(self, ["svnadmin", "create", repository_directory])
result = waitProcess(self, process, 20)
if result != 0:
raise VersionningError("error creating repository %s" % repository_directory)
self.client.checkout("file://%s" % repository_directory, checkout_directory)
def service_listRepositories(self, context):
for key, value in self.repositories.iteritems():
yield key, value.checkout_directory
def service_commit_all(self, context, message):
for repository in self.repositories.values():
repository.commit(message, delete_missing=True)
def testPythonSvn():
log = logging.getLogger("testPythonSvn")
log.notice(
'If this script immediately exists with "Segmentation fault" or "Illegal instruction", see http://wiki/wiki/python-svn'
)
c = Client()
try:
c.mkdir('svn+ssh://svn/wm/sandbox/bug25523', 'testing mkdir')
except ClientError:
# already exists
pass
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" %
(svn_name, V8_SVN_URL))
else:
print(
"INFO: we will try to checkout and build a private v8 build from <%s>."
% V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r: return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print(
"INFO: we will try to use the system 'svn' command to checkout/update V8 code"
)
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
cmdline = ' '.join(args)
exec_cmd(cmdline, "checkout or update Google V8 code from SVN")
def get_revision_number(basedir=None):
''' p SVN的路径。
-> 修订次数。
从SVN中获取修订次数。
'''
client = SvnClient()
ent = client.info(basedir) if basedir else client.info(getcwd())
rev = ent.revision
if rev:
if rev.kind == svn_revision_kind.number:
return rev.number
return 0
def svn_export(url="svn://localhost/", subpath=""):
"""
Ritorna un buffer di memoria contenente
il file richiesto.
Attualmente non viene fatta distinzione tra
file binary e file di testo.
"""
client = Client()
data = StringIO()
data.write(client.cat(url + subpath))
data.seek(0)
return data
def makemovie(url, size):
client = Client()
images = []
r1 = ""
log = client.log(url)
versions = sorted([l['revision'].number for l in log])
for rev in versions:
(r2, diff) = getrev(client, url, rev)
if r2 and not diff:
#really wish I had multiple dispatch here
images.append(gen_one(r2, size))
elif r2 and diff:
images.extend(gen_diff(r1, r2, diff, size))
r1 = r2
raw_input('<enter> to continue')
def checkout_v8(self):
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r: return
print "ERROR: Failed to export from V8 svn repository"
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print "INFO: we will try to use the system 'svn' command to checkout/update V8 code"
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
try:
proc = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr)
proc.communicate()
if proc.returncode != 0:
print "WARN: fail to checkout or update Google v8 code from SVN, error code: ", proc.returncode
except Exception, e:
print "ERROR: fail to invoke 'svn' command, please install it first: %s" % e
sys.exit(-1)
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" % (svn_name, V8_SVN_URL))
else:
print("INFO: we will try to checkout and build a private v8 build from <%s>." % V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number, V8_SVN_REVISION) if V8_SVN_REVISION else Revision(opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r:
print("%s Google V8 code (r%d) from SVN to %s" % ("Update" if update_code else "Checkout", r[-1].number, V8_HOME))
return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print("INFO: we will try to use the system 'svn' command to checkout/update V8 code")
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
cmdline = ' '.join(args)
exec_cmd(cmdline, "checkout or update Google V8 code from SVN")
def __init__(self, name, svn_root, svn_username, svn_password, config_file):
self.name = name
self.svn_root = svn_root
self.svn_username = svn_username
self.svn_password = svn_password
self.client = Client()
self.client.callback_get_login = self._credentials
self.parser = ConfigParser()
self.config_file = config_file
def checkout_v8(self):
update_code = os.path.isdir(V8_HOME) and os.path.exists(os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number, V8_SVN_REVISION) if V8_SVN_REVISION else Revision(opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r: return
print "ERROR: Failed to export from V8 svn repository"
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print "INFO: we will try to use the system 'svn' command to checkout/update V8 code"
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
try:
proc = subprocess.Popen(args, stdout=sys.stdout, stderr=sys.stderr)
proc.communicate()
if proc.returncode != 0:
print "WARN: fail to checkout or update Google v8 code from SVN, error code: ", proc.returncode
except Exception, e:
print "ERROR: fail to invoke 'svn' command, please install it first: %s" % e
sys.exit(-1)
def svn_list(url="svn://localhost/", subpath=""):
"""
Ritorna una lista di dizionari contenente
il contenuto della directory corrente
"""
client = Client()
items = []
for i in client.list(url + subpath, depth=depth.immediates):
d = {}
for k, v in i[0].items():
if k == "kind" and v == node_kind.dir:
d[k] = "dir"
elif k == "kind" and v == node_kind.file:
d[k] = "file"
else:
d[k] = v
items.append(d)
return items
class SVNInterface: #XXX: for Testing pysvn, add new features as desired
"""
Provides an interface to SVN repositories
"""
def __init__(self, repo):
self.repo = repo
self.client = Client()
def lastRev(self):
ret = self.lastLog()
if ret == 'Error connecting to SVN repo':
return -1
else:
return ret[0]['revision']
def lastLog(self, num=1, files=False):
if num == 1:
end = Revision(opt_revision_kind.head)
else:
end = Revision(opt_revision_kind.number, int(self.lastRev()) - (num - 1))
try:
log = self.client.log(self.repo, revision_end=end, discover_changed_paths=files)
except ClientError, message:
msg = 'Error connecting to SVN repo: %s' % (message, )
print msg
return 'Error connecting to SVN repo'
for entry in log:
entry['date'] = asctime(localtime(entry['date'])) # convert EPOCH seconds to human readable
entry['revision'] = str(entry['revision'].number) # str -> converts int -> string
entry['author'] = str(entry['author']) # str -> converts unicode -> ascii
entry['message'] = str(entry['message']) # str -> converts unicode -> ascii
if files:
files = []
for file in entry['changed_paths']:
filename = str(file['path'])
mode = str(file['action'])
files.append('%s (%s)' % (filename, mode))
entry['files'] = files
return log
class SvnRepoMonitor():
def __init__(self, name, svn_root, svn_username, svn_password, config_file):
self.name = name
self.svn_root = svn_root
self.svn_username = svn_username
self.svn_password = svn_password
self.client = Client()
self.client.callback_get_login = self._credentials
self.parser = ConfigParser()
self.config_file = config_file
def _notify(self, author, date, revision, message, paths):
"""Display the changed paths using libnotify"""
title_string = 'New commit #%s in repository %s' % (revision, self.name)
message_string = "<p>[%s] %s</p> <p><i>%s</i>" % \
(date.strftime("%d.%m %H:%M"), author, message)
message_string += "<ul>"
for p in paths[:5]:
if len(p) > 50:
p = "...%s" % p[-50:]
message_string += "<li>%s</li>" % p
if len(paths) > 6:
message_string += "<li>...</li>"
message_string += "</ul></p>"
logging.debug("Open pynotify.Notification: %s | %s" % (title_string, message_string))
pynotify.Notification(title_string, message_string, "view-refresh").show()
def _credentials(self, realm, username, may_save):
"""Return the default login credentials"""
return True, self.svn_username, self.svn_password, False
def discover_changes(self):
"""
Find out the changes occured since the last time this method is ran
"""
logging.debug("Discover Changes in %s" % self.name)
self.parser.read(self.config_file)
if self.parser.has_option(self.name, "last_revision"):
last_revision = self.parser.get(self.name, "last_revision")
else:
last_revision = 0
log = self.client.log(
self.svn_root,
discover_changed_paths=True,
revision_end=Revision(opt_revision_kind.number, last_revision)
)
log = log[:-1] # Ignore last revision
if len(log) > 0:
logging.info("%s new commits in repository %s" % (len(log), self.name))
# update last revision in config file
last_revision = log[0].revision.number
self.parser.set(self.name, "last_revision", last_revision)
self.parser.write(open(self.config_file, 'w'))
log.reverse()
if len(log) > 5: # Show only most recent commits
pynotify.Notification("Even more commits in repository %s" % self.name,
"There are %s more new commits in the repository" % (len(log) - 5),
"view-refresh").show()
log = log[-5:]
for entry in log:
author = entry.get('author')
rev = entry.revision.number
message = entry.message
date = datetime.datetime.fromtimestamp(entry.date)
paths = []
for change in entry.changed_paths:
paths.append(change.path)
self._notify(author, date, rev, message, paths)
else:
logging.debug("No new commits in repository %s" % self.name)
class SvnRepoMonitor():
def __init__(self, name, svn_root, svn_username, svn_password,
config_file):
self.name = name
self.svn_root = svn_root
self.svn_username = svn_username
self.svn_password = svn_password
self.client = Client()
self.client.callback_get_login = self._credentials
self.parser = ConfigParser()
self.config_file = config_file
def _notify(self, author, date, revision, message, paths):
"""Display the changed paths using libnotify"""
title_string = 'New commit #%s in repository %s' % (revision,
self.name)
message_string = "<p>[%s] %s</p> <p><i>%s</i>" % \
(date.strftime("%d.%m %H:%M"), author, message)
message_string += "<ul>"
for p in paths[:5]:
if len(p) > 50:
p = "...%s" % p[-50:]
message_string += "<li>%s</li>" % p
if len(paths) > 6:
message_string += "<li>...</li>"
message_string += "</ul></p>"
logging.debug("Open pynotify.Notification: %s | %s" %
(title_string, message_string))
pynotify.Notification(title_string, message_string,
"view-refresh").show()
def _credentials(self, realm, username, may_save):
"""Return the default login credentials"""
return True, self.svn_username, self.svn_password, False
def discover_changes(self):
"""
Find out the changes occured since the last time this method is ran
"""
logging.debug("Discover Changes in %s" % self.name)
self.parser.read(self.config_file)
if self.parser.has_option(self.name, "last_revision"):
last_revision = self.parser.get(self.name, "last_revision")
else:
last_revision = 0
log = self.client.log(self.svn_root,
discover_changed_paths=True,
revision_end=Revision(opt_revision_kind.number,
last_revision))
log = log[:-1] # Ignore last revision
if len(log) > 0:
logging.info("%s new commits in repository %s" %
(len(log), self.name))
# update last revision in config file
last_revision = log[0].revision.number
self.parser.set(self.name, "last_revision", last_revision)
self.parser.write(open(self.config_file, 'w'))
log.reverse()
if len(log) > 5: # Show only most recent commits
pynotify.Notification(
"Even more commits in repository %s" % self.name,
"There are %s more new commits in the repository" %
(len(log) - 5), "view-refresh").show()
log = log[-5:]
for entry in log:
author = entry.get('author')
rev = entry.revision.number
message = entry.message
date = datetime.datetime.fromtimestamp(entry.date)
paths = []
for change in entry.changed_paths:
paths.append(change.path)
self._notify(author, date, rev, message, paths)
else:
logging.debug("No new commits in repository %s" % self.name)
class Compile(object):
def __init__(self, address):
self.conn = pymongo.Connection(host=address)
self.db = self.conn.ttengine
self.svnClient = Client()
def update(self, machine, branches):
for path in branches:
self.svnClient.update(path)
# 更新编译
def updateCompile(self, recordId, machine, userId, branches):
#self.update(machine, branches)
# 先默认是在本机编译
userInfo = self.db.user.find_one({"_id": ObjectId(userId)})
if userInfo is None:
data = {
"end_time": datetime.now(),
"status": 2,
"remark": "用户信息为空"
}
self.updateRecord(recordId, data)
return
if len(branches) == 0:
data = {
"end_time": datetime.now(),
"status": 2,
"remark": "未选择代码分支"
}
self.updateRecord(recordId, data)
return
rawDir = os.curdir
userName = userInfo["username"]
for path in branches:
# 运行编译脚本
print "change dir", path
os.chdir(path)
print "make clean", path
p = Popen(["make", "clean", "user=%s" % userName],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
p.wait()
print "make clean completed", path
if recordId is not None:
print "make -j8 all", path, recordId
p = Popen(["make", "-j8", "all",
"user=%s" % userName],
stdin=PIPE,
stdout=PIPE,
stderr=PIPE)
outStr, errorStr = p.communicate()
if len(errorStr) > 0:
data = {}
data["status"] = 2
data["remark"] = errorStr
self.updateRecord(recordId, data)
print "compile completed: %s" % path
os.chdir(rawDir)
data = {}
data["end_time"] = datetime.now()
data["status"] = 3
data["remark"] = "编译完成"
self.updateRecord(recordId, data)
def updateRecord(self, recordId, datas):
pass
#if recordId is not None:
# self.db.compiling_record.update({"_id": ObjectId(recordId)}, {"$set": datas})
def getCustomer(self, customerId):
customer = self.db.customer.find_one({"_id": ObjectId(customerId)})
return customer
def getUser(self, userId):
user = self.db.user.find_one({"_id": ObjectId(userId)})
return user
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" %
(svn_name, V8_SVN_URL))
else:
print(
"INFO: we will try to checkout and build a private v8 build from <%s>."
% V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(
os.path.join(V8_HOME, 'include', 'v8.h'))
try:
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number,
V8_SVN_REVISION) if V8_SVN_REVISION else Revision(
opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r:
print("%s Google V8 code (r%d) from SVN to %s" %
("Update" if update_code else "Checkout", r[-1].number,
V8_HOME))
with open(v8_svn_rev_file, 'w') as f:
f.write(str(r[-1].number))
return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print(
"INFO: we will try to use the system 'svn' command to checkout/update V8 code"
)
if update_code:
args = ["svn", "up", V8_HOME]
else:
os.makedirs(V8_HOME)
args = ["svn", "co", V8_SVN_URL, V8_HOME]
if V8_SVN_REVISION:
args += ['-r', str(V8_SVN_REVISION)]
if exec_cmd(' '.join(args), "checkout or update Google V8 code from SVN"):
if not V8_SVN_REVISION:
ok, out, err = exec_cmd(
' '.join(["svn", "info", V8_HOME]),
"save the current V8 SVN revision to REVISION file",
output=True)
if ok:
with open(v8_svn_rev_file, 'w') as f:
f.write(
re.search(r'Revision:\s(?P<rev>\d+)', out,
re.MULTILINE).groupdict()['rev'])
else:
print("ERROR: fail to fetch SVN info, %s", err)
def __init__(self, address):
self.conn = pymongo.Connection(host=address)
self.db = self.conn.ttengine
self.svnClient = Client()
def __init__(self, address):
self.conn = pymongo.Connection(host=address)
self.db = self.conn.ttengine
self.svnClient = Client()
def get_revisions(since_revision):
client = Client()
end_rev = Revision(opt_revision_kind.number,
since_revision)
return client.log(app.config["SVN_URL"], Revision(opt_revision_kind.head),
end_rev)[:-1]
def unrelease(project, version, options):
"""Undoes the project release for the specified project/version"""
log = logging.getLogger("unrelease")
client = Client()
# calculate version with and without patch number
matchPatch = re.compile(r'([1-9][0-9.]+)p([1-9][0-9.]*)')
m = matchPatch.match(version)
if m:
baseVersion = m.group(1)
fullVersion = version
patch = m.group(2)
else:
baseVersion = version
fullVersion = version
patch = None
basePath = '/ext/build/%s/%s' % (project, baseVersion)
releaseLinkPath = basePath + '/releases/' + fullVersion
if not os.path.exists(releaseLinkPath):
raise Exception('Could not find link to release %s for %s at %s' %
(version, project, releaseLinkPath))
# get canonical path
releasePath = os.path.realpath(releaseLinkPath)
# sanity check
if releasePath.find('releases') == -1:
raise Exception(
'Symlink %s in releases directory points to non-release path %s - something is wrong. You'
'll have to fix that manually.' % (releaseLinkPath, releasePath))
# Determine if this is SVN or git
scmgit = False
scmsvn = False
if os.path.exists(releasePath + '/git-checkout.log'):
scmgit = True
else:
if os.path.exists(releasePath + '/svn-checkout.log'):
scmsvn = True
else:
raise Exception('Could not determine SVN or GIT')
# get timestamp name
timestamp = getTimestamp(releasePath)
snapPath = basePath + '/snap/' + timestamp
# remove existing best link, decide whether to fix it at the end
# logic here is:
# - if we are told not to modify best, don't.
# - otherwise, if best link doesn't exists
# or if it exists and points to this build
# or if it points nowhere
# then fix it to point to the snapshot after we've moved it.
updateBest = False
bestLinkPath = basePath + '/best'
if options.best:
if (not os.path.exists(bestLinkPath)
or os.path.realpath(bestLinkPath) == releasePath
or (not os.path.exists(os.path.realpath(bestLinkPath)))):
log.info('removing old best link')
ssh('rm -f ' + bestLinkPath, options)
updateBest = True
log.info('Removing release links')
cmd = 'rm /ext/build/%s/%s/releases/%s' % (project, baseVersion,
fullVersion)
ssh(cmd, options)
cmd = 'rm /ext/build/%s/releases/%s' % (project, fullVersion)
ssh(cmd, options)
# get tag path
if scmsvn:
tagUrl = getFileContents(log, releasePath + '/TAG')
log.info("tag URL is '%s'" % (tagUrl))
# delete tag
log.info('deleting SVN tag')
msg = 'Removing tag from reversed release of project %s, version %s' % (
project, version)
escapedMessage = msg.replace("'", "\\'")
shell('svn rm %s -m \'%s\'' % (tagUrl, escapedMessage), options)
# write UNTAG file, remove TAG file
log.info('removing TAG file, writing to UNTAG file')
cmd = "rm %s/TAG ; echo '%s' >> %s/UNTAG" % (releasePath, tagUrl,
releasePath)
ssh(cmd, options)
if scmgit:
cmd = 'ssh git@git deltag ' + project + ' ' + version
try:
shell(cmd, options)
except ShellException, e:
raise Exception(
'Something went wrong with the deltag command. Check the log to see what the error was'
)
def __init__(self, repo):
self.repo = repo
self.client = Client()
pass
open(logParser, 'a').close()
logging.basicConfig(level=eval(Conf.log_debug_level), filename=logParser, format='%(asctime)s :: %(levelname)s :: %(message)s', datefmt='%m/%d/%Y %H:%M:%S')
logging.info('Init Parser with launchWAMPSub'.format())
PASSWORDS = {
unicode(Conf.websocket_user): unicode(Conf.websocket_pass)
}
USER = unicode(Conf.websocket_user)
urlCrossbar = Conf.websocket_router
portCrossbar = Conf.websocket_port
realmCrossbar = unicode(Conf.websocket_realm)
uriTopic = unicode(Conf.websocket_uriparser)
SVNclient = Client()
SVNclient.callback_get_login = get_login
SVNclient.callback_get_log_message = get_log_message
rpcApi = xmlrpclib.ServerProxy('http://127.0.0.1:{}'.format(Conf.master_rpc_wpub_port), allow_none=True)
launchWAMPSub(rpcApi)
except Exception, e:
logging.error('Failed to init Parser with WAMP : {}'.format(e))
sys.exit()
def checkout_v8():
if svn_name:
print("INFO: we will try to update v8 to %s at <%s>" % (svn_name, V8_SVN_URL))
else:
print("INFO: we will try to checkout and build a private v8 build from <%s>." % V8_SVN_URL)
print("=" * 20)
print("INFO: Checking out or Updating Google V8 code from SVN...\n")
update_code = os.path.isdir(V8_HOME) and os.path.exists(os.path.join(V8_HOME, 'include', 'v8.h'))
try:
print('skipping pysvn install')
raise ImportError('Skip attempt to use python-svn/pysvn')
from pysvn import Client, Revision, opt_revision_kind
svnClient = Client()
rev = Revision(opt_revision_kind.number, V8_SVN_REVISION) if V8_SVN_REVISION else Revision(opt_revision_kind.head)
if update_code:
r = svnClient.update(V8_HOME, revision=rev)
else:
r = svnClient.checkout(V8_SVN_URL, V8_HOME, revision=rev)
if r:
print("%s Google V8 code (r%d) from SVN to %s" % ("Update" if update_code else "Checkout", r[-1].number, V8_HOME))
with open(v8_svn_rev_file, 'w') as f:
f.write(str(r[-1].number))
return
print("ERROR: Failed to export from V8 svn repository")
except ImportError:
#print "WARN: could not import pysvn. Ensure you have the pysvn package installed."
#print " on debian/ubuntu, this is called 'python-svn'; on Fedora/RHEL, this is called 'pysvn'."
print("INFO: we will try to use the system 'svn' command to checkout/update V8 code")
# print('PYv8Home %s' % PYV8_HOME)
# print('V8_HOME: exiting\n>%s<' % V8_HOME)
# import sys
# sys.exit(42)
if update_code and False:
args = ["svn", "up", V8_HOME]
# svn: svn up(date) V8_HOME
# git: git pull (repository path to update)
git_args = ["git", "--work-tree="+V8_HOME,"--git-dir"+V8_HOME+"/.git","pull"]
else:
os.makedirs(V8_HOME)
# make the folder
args = ["svn", "co", V8_SVN_URL, V8_HOME]
# svn: svn checkout (repo to clone) (location to put it)
# git: git clone (repo to clone) (location to put it)
git_args = ["git", "clone", V8_GIT_URL, V8_HOME]
if V8_SVN_REVISION:
# if there is a specific version to pull
args += ['-r', str(V8_SVN_REVISION)]
# svn: -r specifies the "commit"/branch id/tag whatever to use as version
# git: can clone just a specific branch. I'd rather clone it all and checkout
# a specific branch, then pull its changes just in case
if exec_cmd(' '.join(git_args), "git clone a new repository for Google v8 code"):
print('I think it cloned')
'''
class Compile(object):
def __init__(self, address):
self.conn = pymongo.Connection(host=address)
self.db = self.conn.ttengine
self.svnClient = Client()
def update(self, machine, branches):
for path in branches:
self.svnClient.update(path)
# 更新编译
def updateCompile(self, recordId, machine, userId, branches):
#self.update(machine, branches)
# 先默认是在本机编译
userInfo = self.db.user.find_one({"_id": ObjectId(userId)})
if userInfo is None:
data = {
"end_time" :datetime.now(),
"status" : 2,
"remark" : "用户信息为空"
}
self.updateRecord(recordId, data)
return
if len(branches) == 0:
data = {
"end_time" :datetime.now(),
"status" : 2,
"remark" : "未选择代码分支"
}
self.updateRecord(recordId, data)
return
rawDir = os.curdir
userName = userInfo["username"]
for path in branches:
# 运行编译脚本
print "change dir", path
os.chdir(path)
print "make clean", path
p = Popen(["make", "clean", "user=%s" % userName], stdin=PIPE, stdout=PIPE, stderr=PIPE)
p.wait()
print "make clean completed", path
if recordId is not None:
print "make -j8 all", path, recordId
p = Popen(["make", "-j8", "all", "user=%s" % userName], stdin=PIPE, stdout=PIPE, stderr=PIPE)
outStr, errorStr = p.communicate()
if len(errorStr) > 0:
data = {}
data["status"] = 2
data["remark"] = errorStr
self.updateRecord(recordId, data)
print "compile completed: %s" % path
os.chdir(rawDir)
data = {}
data["end_time"] = datetime.now()
data["status"] = 3
data["remark"] = "编译完成"
self.updateRecord(recordId, data)
def updateRecord(self, recordId, datas):
pass
#if recordId is not None:
# self.db.compiling_record.update({"_id": ObjectId(recordId)}, {"$set": datas})
def getCustomer(self, customerId):
customer = self.db.customer.find_one({"_id":ObjectId(customerId)})
return customer
def getUser(self, userId):
user = self.db.user.find_one({"_id":ObjectId(userId)})
return user
def release(project, version, build, options):
"""Release a project/version"""
log = logging.getLogger("release")
log.info('Beginning release')
client = Client()
# Check to make sure version will be legal as a tag
reg = re.compile('[-a-zA-Z0-9_.]+')
for x in range(len(version)):
if reg.match(version[x]) == None:
raise Exception(
"The version specified has characters that are not legal for setting a tag."
)
basePath = '/ext/build/' + project + '/' + version
# resolve path to snapshot
if (build == 'best'):
log.info("Resolving best link:")
else:
log.info("Resolving link to snapshot '%s':" % (build))
if build:
# try snapshot first
snapPath = basePath + '/snap/' + build
snapSource = 'SNAP'
if not os.path.exists(snapPath):
# try tinderbuilds
snapPath = basePath + '/tinderbuilds/' + build
snapSource = 'TINDERBUILD'
if not os.path.exists(snapPath):
# try base
snapPath = basePath + '/' + build
snapSource = 'BASE'
if not os.path.exists(snapPath):
raise Exception("Could not find specified build %s for %s %s" %
(build, project, version))
else:
snapPath = basePath + '/best'
snapSource = 'BEST'
if not os.path.exists(snapPath):
raise Exception(
'Could not find best link at %s - either create a best link or retry with "--build".'
% (snapPath))
if options.paranoid and snapSource == 'TINDERBUILD':
raise Exception(
'Specified build found at %s. Direct release of tinderbuild is not recommended. Rerun with "-sv" to snapshot your build first, or use "--relaxed" to disable this check.'
% snapPath)
# get canonical path
path = os.path.realpath(snapPath)
# parse timestamp
timestamp = getTimestamp(path)
# determine tag
project_xml = path + '/src/project.xml'
if not os.path.exists(project_xml):
project_xml = path + '/src/' + project + '_project/project.xml'
if not os.path.exists(project_xml):
raise Exception('Could not find project.xml at: ' + project_xml)
proj = Project(project_xml)
tag = version
patch = None
if proj.patch:
patch = proj.patch
if options.override_patch:
patch = options.override_patch
if patch:
# It would be reasonable / intuitive for a user to include a leading p in override_patch.
# Remove any leading ps or spaces to normalize expectations.
patch = patch.strip('p \t')
tag += 'p' + patch
if options.override_patch:
log.warn(
'using tag %s, including override patch number instead of value from project.xml (%s)'
% (tag, proj.patch))
# Determine if this is SVN or git
scmgit = False
scmsvn = False
if os.path.exists(path + '/git-checkout.log'):
scmgit = True
else:
if os.path.exists(path + '/svn-checkout.log'):
scmsvn = True
else:
raise Exception('Could not determine SVN or GIT')
# determine checkout revision and path
if scmsvn:
info = client.info(path + "/src")
rev = info.revision
svnpath = info.url.replace("svn://", "svn+ssh://")
if options.paranoid:
if svnpath.find(project) == -1:
raise Exception(
'The project you'
're trying to release doesn'
't contain its project name %s in its SVN path %s. Something is horribly wrong. (Use "--relaxed" to disable this check.)'
% (project, svnPath))
# Determine tag path
tagDir = 'svn+ssh://svn/wm/project/' + project + '/tags'
tagPath = tagDir + '/' + tag
# For git, make sure the repo is writable by the current user
if scmgit:
cmd = 'ssh git@git'
results = shell(cmd, options)
repo_match = False
repo_writable = False
for x in range(len(results)):
if results[x].lstrip('@_RW \t\r\n').rstrip('\r\n') == project:
repo_match = True
if results[x][9] == 'W':
repo_writable = True
else:
raise Exception(
'You dont have right to make changes to the repo for this project, therefore you cannot set the tag.'
)
# check that tag doesn't already exist
if not options.notag:
if scmsvn:
try:
client.ls(tagPath)
# tag exists already! Oops!
raise Exception(
'Tag path %s already exists! Halting. (You can use --notag to disable tag creation and this check.)'
% (tagPath))
except ClientError:
# this is expected, as we haven't created it yet.
pass
log.info("Releasing:")
log.info(" build: %s", path)
if scmsvn:
log.info(" svn: -r %d %s", rev.number, svnpath)
if scmgit:
log.info(" git")
log.info(" tag: %s", tag)
# Make release dirs
log.info("Ensuring that release directories exist")
releasesDir = basePath + '/releases'
allReleasesDir = '/ext/build/' + project + '/releases'
ssh('mkdir -p ' + releasesDir, options)
ssh('mkdir -p ' + allReleasesDir, options)
# Copy to releases
log.info("Moving snap to releases")
cmd = 'test -d ' + path + ' && mv ' + path + ' ' + releasesDir
ssh(cmd, options)
# Create soft links
log.info("Create soft links")
cmd = 'cd ' + releasesDir + '; test -e ' + tag + ' || ln -s ' + timestamp + ' ' + tag
ssh(cmd, options)
cmd = 'cd ' + allReleasesDir + '; test -e ' + tag + ' || ln -s ../' + version + '/releases/' + tag + ' ' + tag
ssh(cmd, options)
# recursively find dependencies and record
# this will overwrite any previous instances (e.g. from a snapshot)
findDepends(log, releasesDir + "/" + timestamp, options)
# Edit save file
log.info("Appending to SAVE file")
msg = '%s %s RELEASE' % (project, version)
writeSave(log, releasesDir + '/' + tag, msg, options)
# Create project tags directory
if scmsvn:
try:
client.ls(tagDir)
log.info("Project tags directory already exists")
except ClientError:
# does not exist, so create it
log.info("Creating project tags directory")
if not options.dryrun:
client.mkdir(
tagDir,
'release-project: make tag directory for ' + project)
if options.notag:
log.notice('skippping tag creation, as --notag is set')
# may want to write tag file, if svn path has a tag in it already.
if svnpath.find('/tags/') != -1:
log.notice(
'src svnpath is a tag already, writing that to TAG path')
cmd = 'cd ' + releasesDir + '/' + tag + '; echo ' + svnpath + ' >> TAG'
ssh(cmd, options)
else:
# Make tag
log.info("Tagging project")
log.info(" > svn copy -r%d %s %s", rev.number, svnpath, tagPath)
message = 'release-project: saving release tag'
def get_log_message():
return True, message
client.callback_get_log_message = get_log_message
if not options.dryrun:
client.copy(svnpath, tagPath, rev)
client.callback_get_log_message = None
# Log to TAG file
log.info("Creating TAG file")
cmd = 'cd ' + releasesDir + '/' + tag + '; echo ' + tagPath + ' >> TAG'
ssh(cmd, options)
#Create tag for git
if scmgit:
cmd = 'cd ' + releasesDir + '/' + tag + '/src && git show --format=oneline --summary | cut -d \' \' -f 1'
githash = shell(cmd, options)
cmd = 'ssh git@git addtag ' + project + ' ' + version + ' ' + githash[0]
try:
shell(cmd, options)
except ShellException, e:
raise Exception(
'Something went wrong with the addtag command. Check the log to see what the error was'
)
cmd = 'cd ' + releasesDir + '/' + tag + '/src && git fetch'
ssh(cmd, options)
def __init__(self):
Component.__init__(self)
self.repositories = {}
self.client = Client()
def _get_svn_info():
try:
from pysvn import Client, ClientError
try:
info = Client().info(".")
return info["url"].split("/")[-1], info["revision"].number
except ClientError:
# Not an svn working dir
#sys.stderr.write("""Hmm, doesn't look like an SVN repo""")
pass
except ImportError:
#sys.stderr.write(" * please consider installing pysvn\n")
#sys.stderr.write("""*** please install pysvn:
#- debian: sudo apt-get install python-svn
#- windows: http://pysvn.tigris.org/project_downloads.html
#***""")
pass
branch_name = None
revision = None
try:
platform = plat_id()
#
# svn executable is found, either linux or win with svn
#
if re.match('^linux.*', platform) or which('svn'):
svn_proc = subprocess.Popen(['svn','info'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
svn_stdout = svn_proc.communicate()[0]
if svn_proc.returncode == 0:
for l in re.split('(?:\r\n|\n)',svn_stdout):
m = re.match('([^:]+): (.*)',l)
if m:
if m.group(1) == 'URL':
url = m.group(2)
purl = urlparse(url)
(head,tail) = os.path.split(purl.path)
branch_name = tail
elif m.group(1) == 'Revision':
revision = m.group(2)
else:
#print 'nomatch', l
pass
#
# Try tortoise
#
elif re.match('^win.*', platform):
svn_proc = subprocess.Popen([SUBWCREV,'.'], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
svn_stdout = svn_proc.communicate()[0]
if svn_proc.returncode == 0:
for l in re.split('(?:\r\n|\n)',svn_stdout):
m = re.match('Updated to revision (\d+)',l)
if m:
revision = m.group(1)
break
(head, tail) = os.path.split(os.getcwd())
branch_name = tail
except OSError, e:
#errmsg = sys.argv[0] + ': couldn\'t find branch name\n'
#sys.stderr.write(errmsg)
#sys.stderr.write(str(e))
pass
def main(options):
"""docstring for main"""
client = Client()
client.callback_ssl_server_trust_prompt = ssl_trust_prompt
if (options.verbose > 1):
client.callback_notify = notify
retcode = 0
# check out branch from subversion
if (test_skip(options.branch_flags, update)):
getSourceTree(options.workingDir + os.sep + "branch",
options.branchUrl, client, options)
elif (options.verbose):
print "Skipping branch update"
# compile branch
if (test_skip(options.branch_flags, build)):
print "Compiling branch"
retcode = doCompile(options.workingDir + os.sep + "branch" + os.sep +
"VS2005")
elif (options.verbose):
print "Skipping branch compile"
# If compile successful, Run autotest
if (retcode != 0):
print "Failed to compile branch"
return retcode
if (test_skip(options.branch_flags, autotest)):
print "Running branch autotest"
retcode = doAutotest(options.workingDir + os.sep + "branch")
elif (options.verbose):
print "Skipping branch autotest"
if (retcode != 0):
print "Failed branch autotest"
return retcode
if (test_skip(options.trunk_flags, update)):
getSourceTree(options.workingDir + os.sep + "trunk", options.trunkUrl,
client, options)
elif (options.verbose):
print "Skipping trunk update"
# Determine the revision number of the head of the branch
if (options.endrev == 'HEAD'):
if (options.verbose):
print "Getting last revision number of branch"
messages = client.log(branchUrl,
revision_end=Revision(
pysvn.opt_revision_kind.number, startrev))
endrev = messages[0].revision.number
options.endrev = str(endrev)
if (options.verbose):
print "Revisions to merge", options.startrev + ":" + options.endrev
# Merge changes from branch to trunk
if (not options.skip_merge):
if (options.verbose):
print "Merging branch changes to trunk working directory"
(retcode, conflicts) = doMerge(options.workingDir + os.sep + "trunk",
endrev, options)
elif (options.verbose):
print "Skipping merge"
# How do we tell if there were merge errors?!?
if (retcode != 0):
print "Merge to working directory failed with conflicts."
printList(conflicts)
return retcode
# Compile trunk
if (test_skip(options.trunk_flags, build)):
print "Compiling trunk"
retcode = doCompile(options.workingDir + os.sep + "trunk" + os.sep +
"VS2005")
elif (options.verbose):
print "Skipping trunk compile"
if (retcode != 0):
print "Failed to compile merged trunk"
return retcode
# If compile successful, run autotest
if (test_skip(options.trunk_flags, autotest)):
print "Running trunk autotest"
retcode = doAutotest(options.workingDir + os.sep + "trunk")
elif (options.verbose):
print "Skipping trunk autotest"
if (retcode != 0):
print "Failed in merged autotest"
return retcode
# TODO: If successful, commit.
if (not options.skip_commit):
pass
# Write out new lastrev.py file
fp = open("lastrev.py", 'w')
fp.write("startrev = " + str(endrev) + "\n")
fp.write("prevrev = " + str(startrev))
fp.close()
