def main(options, _args): runSystemEx('svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel(convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel(bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx('mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join(vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join(vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap(dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def main(options, _args): runSystemEx( 'svn copy https://clovr.svn.sourceforge.net/svnroot/clovr/trunk https://clovr.svn.sourceforge.net/svnroot/clovr/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx( 'svn copy https://vappio.svn.sourceforge.net/svnroot/vappio/trunk https://vappio.svn.sourceforge.net/svnroot/vappio/tags/%s -m "Cutting release %s"' % (options('general.version'), options('general.version')), log=True) runSystemEx('scp %s:/export/%s .' % (options('general.remote_name'), options('general.image')), log=True) runSystemEx('cp %s /usr/local/projects/clovr/images' % options('general.image'), log=True) runSystemEx('cp %s VMware_conversion/shared/convert_img.img' % options('general.image'), log=True) convertChannel = threads.runThreadWithChannel( convertImage).channel.sendWithChannel(options) waitForPasswordChange() bundleChannel = threads.runThreadWithChannel( bundleAMI).channel.sendWithChannel(options) try: convertChannel.receive() vmWareDir = 'clovr-vmware.%s' % options('general.version') runSystemEx('mkdir -p ' + vmWareDir, log=True) runSystemEx( 'mv VMware_conversion/shared/converted_img.vmdk %s' % os.path.join( vmWareDir, 'clovr.9-04.x86-64.%s.vmdk' % options('general.version'))) runSystemEx('mkdir -p %s %s' % (os.path.join( vmWareDir, 'keys'), os.path.join(vmWareDir, 'user_data')), log=True) runSystemEx('cp -rv /usr/local/projects/clovr/shared ' + vmWareDir, log=True) fout = open(os.path.join(vmWareDir, 'start_clovr.vmx'), 'w') clovrConf = config.configFromMap( dict(version=options('general.version'))) for line in open('/usr/local/projects/clovr/start_clovr.vmx'): fout.write(config.replaceStr(line, clovrConf)) except Exception, err: errorPrint('Converting image failed. Error message:') errorPrint(str(err))
def attemptDownload(options, url): cmd = ["wget", "-nv", "-P", options("general.base_dir")] if options("general.continue_download"): cmd.append("-c") cmd.append(url) pr = commands.ProgramRunner(" ".join(cmd), stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) downloaderChan = threads.runThreadWithChannel(runDownloader).channel.sendWithChannel(pr) ## # Wait 20 seconds or until files start appearing, if nothing appears # then we will continue on to monitorDownload and it will fail out anyways count = 20 while count > 0 and not getDownloadFilenames(options("general.base_dir"), url): count -= 1 time.sleep(1) logging.debugPrint(lambda: "Downloading with a minimum acceptable rate of %d" % options("general.min_rate")) return monitorDownload(pr, downloaderChan, options("general.base_dir"), url, options("general.min_rate"))
def attemptDownload(options, url): cmd = ['wget', '-nv', '-P', options('general.base_dir')] if options('general.continue_download'): cmd.append('-c') cmd.append(url) pr = commands.ProgramRunner(' '.join(cmd), stdoutf=sys.stdout.write, stderrf=sys.stderr.write, log=True) downloaderChan = threads.runThreadWithChannel(runDownloader).channel.sendWithChannel(pr) ## # Wait 20 seconds or until files start appearing, if nothing appears # then we will continue on to monitorDownload and it will fail out anyways count = 20 while count > 0 and not getDownloadFilenames(options('general.base_dir'), url): count -= 1 time.sleep(1) logging.debugPrint(lambda : 'Downloading with a minimum acceptable rate of %d' % options('general.min_rate')) return monitorDownload(pr, downloaderChan, options('general.base_dir'), url, options('general.min_rate'))
def main(options, args): logging.DEBUG = options("general.debug") queue = Queue.Queue() ## # Track the downloaded URL names for joining later if specified urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: md5, url = line.split(" ", 1) url = url.strip() urls.append(url) queue.put((url, md5)) else: for url in args: urls.append(url) queue.put((url, None)) if options("general.join_name") and options("general.join_md5"): md5 = calculateMD5([os.path.join(options("general.base_dir"), options("general.join_name"))]) ## # If they match, then no need to download and exit cleanly if md5 == options("general.join_md5"): return retChans = [ threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options("general.max_threads")) ] successUrls = [] failedUrls = [] for c in retChans: ret = c.receive() while ret is not None: url, succ = ret if succ: successUrls.append(url) else: failedUrls.append(url) ret = c.receive() if failedUrls: for url in failedUrls: logging.errorPrint(url) ## # If any URLs failed, exit with fail sys.exit(1) else: if options("general.join_name"): logging.debugPrint(lambda: "Joining files into: " + options("general.join_name")) files = [] for url in urls: files.extend(sorted(getDownloadFilenames(options("general.base_dir"), url))) fout = open(os.path.join(options("general.base_dir"), options("general.join_name")), "wb") for f in files: logging.debugPrint(lambda: "Reading: " + f) fin = open(f, "rb") d = fin.read(1000000) while d: fout.write(d) d = fin.read(1000000) fin.close() fout.close() logging.debugPrint(lambda: "Deleting downloaded files after join") for f in files: logging.debugPrint(lambda: "Deleting: " + f) deleteFile(f)
def main(options, args): logging.DEBUG = options('general.debug') queue = Queue.Queue() ## # Track the downloaded URL names for joining later if specified urls = [] if not args: for line in [l for l in sys.stdin if l.strip()]: md5, url = line.split(' ', 1) url = url.strip() urls.append(url) queue.put((url, md5)) else: for url in args: urls.append(url) queue.put((url, None)) if options('general.join_name') and options('general.join_md5'): md5 = calculateMD5([os.path.join(options('general.base_dir'), options('general.join_name'))]) ## # If they match, then no need to download and exit cleanly if md5 == options('general.join_md5'): return retChans = [threads.runThreadWithChannel(downloadUrls).channel.sendWithChannel((options, queue)) for _ in range(options('general.max_threads'))] successUrls = [] failedUrls = [] for c in retChans: ret = c.receive() while ret is not None: url, succ = ret if succ: successUrls.append(url) else: failedUrls.append(url) ret = c.receive() if failedUrls: for url in failedUrls: logging.errorPrint(url) ## # If any URLs failed, exit with fail sys.exit(1) else: if options('general.join_name'): logging.debugPrint(lambda : 'Joining files into: ' + options('general.join_name')) files = [] for url in urls: files.extend(sorted(getDownloadFilenames(options('general.base_dir'), url))) fout = open(os.path.join(options('general.base_dir'), options('general.join_name')), 'wb') for f in files: logging.debugPrint(lambda : 'Reading: ' + f) fin = open(f, 'rb') d = fin.read(1000000) while d: fout.write(d) d = fin.read(1000000) fin.close() fout.close() logging.debugPrint(lambda : 'Deleting downloaded files after join') for f in files: logging.debugPrint(lambda : 'Deleting: ' + f) deleteFile(f)