def main(options, _args):
runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True)
convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'),
os.path.join(vmWareDir, 'user_data')), log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def main(options, _args):
runSystemEx(
'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx(
'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"'
% (options('general.version'), options('general.version')),
log=True)
runSystemEx('scp %s:/export/%s .' %
(options('general.remote_name'), options('general.image')),
log=True)
runSystemEx('cp %s /usr/local/projects/clovr/images' %
options('general.image'),
log=True)
runSystemEx('cp %s VMware_conversion/shared/convert_img.img' %
options('general.image'),
log=True)
convertChannel = threads.runThreadWithChannel(
convertImage).channel.sendWithChannel(options)
waitForPasswordChange()
bundleChannel = threads.runThreadWithChannel(
bundleAMI).channel.sendWithChannel(options)
try:
convertChannel.receive()
vmWareDir = 'clovr-vmware.%s' % options('general.version')
runSystemEx('mkdir -p ' + vmWareDir, log=True)
runSystemEx(
'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(
vmWareDir,
'clovr.9-04.x86-64.%s.vmdk' % options('general.version')))
runSystemEx('mkdir -p %s %s' % (os.path.join(
vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')),
log=True)
runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir,
log=True)
fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w')
clovrConf = config.configFromMap(
dict(version=options('general.version')))
for line in open('/usr/local/projects/clovr/start_clovr.vmx'):
fout.write(config.replaceStr(line, clovrConf))
except Exception, err:
errorPrint('Converting image failed. Error message:')
errorPrint(str(err))
def attemptDownload(options, url):
cmd = ["wget", "-nv", "-P", options("general.base_dir")]
if options("general.continue_download"):
cmd.append("-c")
cmd.append(url)
pr = commands.ProgramRunner(" ".join(cmd), stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True)
downloaderChan = threads.runThreadWithChannel(runDownloader).channel.sendWithChannel(pr)
##
# Wait 20 seconds or until files start appearing, if nothing appears
# then we will continue on to monitorDownload and it will fail out anyways
count = 20
while count > 0 and not getDownloadFilenames(options("general.base_dir"), url):
count -= 1
time.sleep(1)
logging.debugPrint(lambda: "Downloading with a minimum acceptable rate of %d" % options("general.min_rate"))
return monitorDownload(pr, downloaderChan, options("general.base_dir"), url, options("general.min_rate"))
def attemptDownload(options, url):
cmd = ['wget', '-nv', '-P', options('general.base_dir')]
if options('general.continue_download'):
cmd.append('-c')
cmd.append(url)
pr = commands.ProgramRunner(' '.join(cmd),
stdoutf=sys.stdout.write,
stderrf=sys.stderr.write,
log=True)
downloaderChan = threads.runThreadWithChannel(runDownloader).channel.sendWithChannel(pr)
##
# Wait 20 seconds or until files start appearing, if nothing appears
# then we will continue on to monitorDownload and it will fail out anyways
count = 20
while count > 0 and not getDownloadFilenames(options('general.base_dir'), url):
count -= 1
time.sleep(1)
logging.debugPrint(lambda : 'Downloading with a minimum acceptable rate of %d' % options('general.min_rate'))
return monitorDownload(pr, downloaderChan, options('general.base_dir'), url, options('general.min_rate'))
def main(options, args):
logging.DEBUG = options("general.debug")
queue = Queue.Queue()
##
# Track the downloaded URL names for joining later if specified
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
md5, url = line.split(" ", 1)
url = url.strip()
urls.append(url)
queue.put((url, md5))
else:
for url in args:
urls.append(url)
queue.put((url, None))
if options("general.join_name") and options("general.join_md5"):
md5 = calculateMD5([os.path.join(options("general.base_dir"), options("general.join_name"))])
##
# If they match, then no need to download and exit cleanly
if md5 == options("general.join_md5"):
return
retChans = [
threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue))
for _ in range(options("general.max_threads"))
]
successUrls = []
failedUrls = []
for c in retChans:
ret = c.receive()
while ret is not None:
url, succ = ret
if succ:
successUrls.append(url)
else:
failedUrls.append(url)
ret = c.receive()
if failedUrls:
for url in failedUrls:
logging.errorPrint(url)
##
# If any URLs failed, exit with fail
sys.exit(1)
else:
if options("general.join_name"):
logging.debugPrint(lambda: "Joining files into: " + options("general.join_name"))
files = []
for url in urls:
files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url)))
fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb")
for f in files:
logging.debugPrint(lambda: "Reading: " + f)
fin = open(f, "rb")
d = fin.read(1000000)
while d:
fout.write(d)
d = fin.read(1000000)
fin.close()
fout.close()
logging.debugPrint(lambda: "Deleting downloaded files after join")
for f in files:
logging.debugPrint(lambda: "Deleting: " + f)
deleteFile(f)
def main(options, args):
logging.DEBUG = options('general.debug')
queue = Queue.Queue()
##
# Track the downloaded URL names for joining later if specified
urls = []
if not args:
for line in [l for l in sys.stdin if l.strip()]:
md5, url = line.split(' ', 1)
url = url.strip()
urls.append(url)
queue.put((url, md5))
else:
for url in args:
urls.append(url)
queue.put((url, None))
if options('general.join_name') and options('general.join_md5'):
md5 = calculateMD5([os.path.join(options('general.base_dir'), options('general.join_name'))])
##
# If they match, then no need to download and exit cleanly
if md5 == options('general.join_md5'):
return
retChans = [threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options('general.max_threads'))]
successUrls = []
failedUrls = []
for c in retChans:
ret = c.receive()
while ret is not None:
url, succ = ret
if succ:
successUrls.append(url)
else:
failedUrls.append(url)
ret = c.receive()
if failedUrls:
for url in failedUrls:
logging.errorPrint(url)
##
# If any URLs failed, exit with fail
sys.exit(1)
else:
if options('general.join_name'):
logging.debugPrint(lambda : 'Joining files into: ' + options('general.join_name'))
files = []
for url in urls:
files.extend(sorted(getDownloadFilenames(options('general.base_dir'), url)))
fout = open(os.path.join(options('general.base_dir'), options('general.join_name')), 'wb')
for f in files:
logging.debugPrint(lambda : 'Reading: ' + f)
fin = open(f, 'rb')
d = fin.read(1000000)
while d:
fout.write(d)
d = fin.read(1000000)
fin.close()
fout.close()
logging.debugPrint(lambda : 'Deleting downloaded files after join')
for f in files:
logging.debugPrint(lambda : 'Deleting: ' + f)
deleteFile(f)
