def _cmd_create(args): data = { 'id': args.ID, 'playlist': 'lobby', 'template': 'main' } pooldir = os.path.join('lobbyscreens', args.ID) os_utils.ensureDirExists(pooldir, noisy=True) os_utils.ensureDirExists(os.path.join(pooldir, 'files'), noisy=True) written = [] with open(os.path.join(pooldir, '__POOL__.yml'), 'w') as f: yaml.dump(data, f, default_flow_style=False) log.info('Wrote %s.', f.name) written += [f.name] with open('.gitignore', 'w') as f: f.write('/parsed.yml\n') written += [f.name] with os_utils.Chdir(pooldir): if not os.path.isdir('.git'): os_utils.cmd(['git', 'init'], echo=True, show_output=True, critical=True) os_utils.cmd(['git', 'lfs', 'install'], echo=True, show_output=True, critical=True) os_utils.cmd(['git', 'lfs', 'track', '*.png', '*.gif', '*.jpg', '*.webm', '*.webp'], echo=True, show_output=True, critical=True) os_utils.cmd(['git', 'add', '.gitattributes']+written, echo=True, show_output=True, critical=True)
def shouldBuild(self): if args.rebuild_all: return True if self.name in args.get_snapshot: log.info("Getting snapshot of %s (%s pre-build)...", script_dir, self.name) for root, _, files in os.walk(script_dir): for filename in files: self.preExisting.append(os.path.relpath(os.path.abspath(os.path.join(root, filename)), script_dir)) if self.name in args.rebuild: return True if not os.path.isdir(self.build_dir): return True if not os.path.isfile(self.builder_meta_file): return True manifest = {} with open(self.builder_meta_file, "r") as f: data = yaml.load(f) manifest = data.get("manifest", {}) for expectedFile in self.expected: if not os.path.isfile(expectedFile): return True relfilepath = os.path.relpath(expectedFile, script_dir) if relfilepath not in manifest: return True if os.stat(expectedFile).m_time != manifest[relfilepath]: return True return False
def parseWord(self, line): """ walkers: noun "w oo" 'k @ z' present: verb 'p r e' "z @ n t" monument: noun "mo" 'n y u' 'm @ n t' """ global REGEX_SEARCH_STRINGS lineChunks = line.split(' ') self.name = lineChunks[0].strip(':') self.type = lineChunks[1].strip() pronunciation = ' '.join(lineChunks[2:]) for match in REGEX_SEARCH_STRINGS.finditer(pronunciation): stressLevel = 0 if match.group(1) == '"': stressLevel = 1 phonemes = [] for phoneme in match.group(2).split(' '): if phoneme not in self.VALID_PHONEMES: log.error( 'INVALID PHONEME "{0}" IN LEX ENTRY "{1}"'.format( phoneme, self.name)) sys.exit(1) if self.phoneset in self.PHONE_CONVERSIONS: phoneset = self.PHONE_CONVERSIONS[self.phoneset] if phoneme in phoneset: phoneme = phoneset[phoneme] phonemes += [phoneme] self.syllables += [(phonemes, stressLevel)] log.info('Parsed {0} as {1}.'.format(pronunciation, repr(self.syllables)))
def build(self): os_utils.ensureDirExists(os.path.dirname(self.target)) # BUGFIX: Coffeescript sometimes doesn't want to overwrite shit. - N3X if os.path.isfile(self.target): os.remove(self.target) coffeefile = self.files[0] if len(self.files) > 1: coffeefile = self.getCoffeeFile() if os.path.isfile(coffeefile): os.remove(coffeefile) with codecs.open(coffeefile, 'w', encoding='utf-8-sig') as outf: tq = tqdm(self.files, desc='Concatenating...', leave=False) for infilename in tq: #outf.write('\n`// FILE: {}`\n'.format(infilename)) with codecs.open(infilename, 'r', encoding='utf-8-sig') as inf: for line in inf: outf.write(line.rstrip() + "\n") #outf.write('\n`//# sourceURL={}\n`\n'.format(infilename)) tq.close() coffeefile_basename, _ = os.path.splitext(os.path.basename(coffeefile)) os_utils.cmd([self.coffee_executable] + self.coffee_opts + ['-o', os.path.dirname(self.target), coffeefile], critical=True, echo=self.should_echo_commands(), show_output=True) coffee_output_file = os.path.join(os.path.dirname(self.target), coffeefile_basename+'.js') if coffee_output_file != self.target: log.info('Renaming %s to %s...', coffee_output_file, self.target) os.rename(coffee_output_file, self.target)
def _cmd_collect(args=None): allpools = {} for pooldirname in os.listdir('lobbyscreens'): pooldir = os.path.join('lobbyscreens', pooldirname) data = None datafile = os.path.join(pooldir, '__POOL__.yml') if os.path.isfile(datafile): with open(datafile, 'r') as f: data = yaml.safe_load(f) if data is None: continue pool = Pool() pool.ID = pooldirname pool.deserialize(data) poolfilesdir = os.path.join(pooldir, 'files') for imagebasename in os.listdir(poolfilesdir): basename, ext = os.path.splitext(imagebasename) #print(basename, ext) if ext not in ('.jpg', '.png', '.gif', '.svg', '.webm', '.webp', '.mp4', '.ogv'): #print(' SKIPPED') continue anim = Animation() anim.ID = basename data = None filedatapath = os.path.join(poolfilesdir, basename+'.yml') if os.path.isfile(filedatapath): with open(filedatapath, 'r') as f: data = yaml.safe_load(f) filedatapath = os.path.join(poolfilesdir, basename+'.toml') if os.path.isfile(filedatapath): with open(filedatapath, 'r') as f: data = toml.load(f) filedatapath = os.path.join(poolfilesdir, basename+'.json') if os.path.isfile(filedatapath): with open(filedatapath, 'r') as f: data = json.load(f) if data is not None: anim.deserialize(data) anim.url = imagebasename fullpath = os.path.join(poolfilesdir, imagebasename) destfile = os.path.join('htdocs', 'img', 'lobby', pool.ID, anim.url) os_utils.ensureDirExists(os.path.dirname(destfile), noisy=False) os_utils.single_copy(fullpath, destfile, as_file=True, noisy=False) pool.add(anim) with open(os.path.join(pooldir, 'parsed.yml'), 'w') as f: yaml.dump(pool.serialize(suppress_id=True), f, default_flow_style=False) log.info('Found pool %r: %d animations', pool.ID, len(pool.animations)) allpools[pool.ID] = pool.serialize() os_utils.ensureDirExists('data') with open('data/lobby.json', 'w') as f: json.dump(allpools, f, indent=2)
def is_stale(self): self._updateCacheInfo() if not os.path.isfile(self.target): return True etag = '' if os.path.isfile(self.etagfile): with open(self.etagfile, 'r') as f: etag = f.read() with log.info('Checking for changes to %s...', self.url): res = requests.head(self.url, allow_redirects=True, headers={'If-None-Match': etag}) if res.status_code == 304: log.info('304 - Not Modified') return False if etag == res.headers.get('ETag'): return False res.raise_for_status() with log.info('Response headers:'): for k, v in res.headers.items(): log.info('%s: %s', k, v) log.info('HTTP %d', res.status_code) with open(self.etagfile, 'w') as f: f.write(res.headers.get('ETag')) return True
def mkproject(bm: BuildMaestro, project: str, depends: List[str] = []): with log.info('Configuring %s...', project): proj_dir = os.path.join('src', project) projin = bm.add( ReplaceTextTarget(os.path.join(proj_dir, f'{project}.csproj'), os.path.join(proj_dir, f'{project}.csproj.in'), replacements={ re.escape('$(ONIPath)'): CONFIG.get('paths.oni').replace( '\\', '\\\\') })) dll = os.path.join('src', project, 'bin', project + '.dll') csfiles = [ f for f in os_utils.get_file_list(os.path.join(proj_dir, 'Source'), prefix=os.path.join( proj_dir, 'Source')) if f.endswith('.cs') ] csfiles.sort() for csfile in csfiles: log.info(csfile) csp = bm.add( MSBuildTarget(dll, os.path.join(proj_dir, f'{project}.sln'), files=csfiles, dependencies=[projin.target] + depends)) csp.msb.properties['ONIPath'] = CONFIG.get('paths.oni') #csp.msb.configuration = 'Debug' deploydir = os.path.join(LOCALMODS, project) bm.add(CopyFileTarget(deploydir, dll, dependencies=[csp.target])) os_utils.ensureDirExists(deploydir, noisy=True) for basefilename in os.listdir(os.path.join('Mods', project)): filename = os.path.join('Mods', project, basefilename) _, ext = os.path.splitext(basefilename) if ext in ('.json', '.txt'): cf = bm.add( CopyFileTarget(os.path.join(deploydir, basefilename), filename, dependencies=[csp.target], verbose=True)) log.info('Found config: %s', basefilename) return csp
def updateManifest(self): currentFiles = [] newfiles = [] if self.name in args.get_snapshot: log.info("Getting snapshot of %s (%s post-build)...", script_dir, self.name) for root, _, files in os.walk(script_dir): for filename in files: currentFiles.append(os.path.relpath(os.path.abspath(os.path.join(root, filename)), script_dir)) log.info("Comparing...") newfiles = [fn.replace("\\", "/") for fn in currentFiles if fn not in self.preExisting] newmanifest = {} with log.info("Checking for %d expected files...", len(self.expected)): for expectedFile in self.expected: if not os.path.isfile(expectedFile): log.error("MISSING %s", expectedFile) return False relfilepath = os.path.relpath(expectedFile, self.build_dir) newmanifest[relfilepath] = os.stat(expectedFile).m_time log.info("All check out!") with open(self.builder_meta_file, "w") as f: yaml.dump( {"configuration": self.configuration, "manifest": newmanifest, "newfiles": newfiles}, f, default_flow_style=False, ) return True
def main(): argp = argparse.ArgumentParser() argp.add_argument('--go', action='store_true') args = argp.parse_args() files_to_proc = [] for root, _, files in os.walk(IN_DIR): for bfn in files: fullpath = os.path.abspath(os.path.join(root, bfn)) if bfn.endswith('.bak'): log.info('rm %s', fullpath) os.remove(fullpath) if bfn.endswith('.php'): files_to_proc += [fullpath] for filename in tqdm.tqdm(files_to_proc, desc='Moving files...', unit='file'): namespace = None outpath = None with open(filename, 'r') as f: for line in f: m = re.match(REG_NS, line) if m is not None: namespace = m.group(1) break if namespace is None: continue nschunks = namespace.split('\\') if nschunks[0] == '': nschunks = nschunks[1:] nschunks = nschunks[1:] nschunks += [os.path.basename(filename).replace('.class', '').replace('.interface','')] outpath = os.path.abspath(os.path.join(OUT_DIR, *nschunks)) if outpath == filename: continue cmd = [os_utils.which('git'), 'mv', os.path.relpath(filename), os.path.relpath(outpath)] if args.go: os_utils.ensureDirExists(os.path.dirname(outpath), noisy=True) os_utils.cmd([os_utils.which('git'), 'add', os.path.relpath(filename)], echo=True, show_output=True) os_utils.cmd(cmd, echo=True, critical=True) else: log.info(' '.join(cmd)) os_utils.del_empty_dirs(IN_DIR, quiet=False)
def dlPackagesIn(pkgdefs, superrepo="build"): os_utils.ensureDirExists("download") for destination, retrievalData in pkgdefs.items(): rebuild = args.rebuild_all or destination in args.rebuild destination = os.path.join(superrepo, destination) dlType = retrievalData["type"] if dlType == "git": remote = retrievalData.get("remote", "origin") branch = retrievalData.get("branch", "master") commit = retrievalData.get("commit") submodules = retrievalData.get("submodules", False) submodules_remote = retrievalData.get("submodules_remote", False) tag = retrievalData.get("tag") if "uri" not in retrievalData: log.critical("uri not in def for %s", destination) git = GitRepository(destination, retrievalData["uri"], quiet=True, noisy_clone=True) with log.info("Checking for updates to %s...", destination): if rebuild or not os.path.isdir(destination): if rebuild or git.CheckForUpdates(remote, branch, tag=tag, commit=commit): log.info("Updates detecting, pulling...") git.Pull(remote, branch, tag=tag, commit=commit, cleanup=True) if submodules: if rebuild: with os_utils.Chdir(destination): os_utils.cmd( ["git", "submodule", "foreach", "--recursive", "git clean -dfx"], echo=True, show_output=True, critical=True, ) git.UpdateSubmodules(submodules_remote) elif dlType == "hg": remote = retrievalData.get("remote", "default") branch = retrievalData.get("branch", "master") commit = retrievalData.get("commit", retrievalData.get("tag")) if "uri" not in retrievalData: log.critical("uri not in def for %s", destination) hg = HgRepository(destination, retrievalData["uri"], quiet=True, noisy_clone=True) with log.info("Checking for updates to %s...", destination): if rebuild or not os.path.isdir(destination): if rebuild or hg.CheckForUpdates(remote, branch): log.info("Updates detecting, pulling...") hg.Pull(remote, branch, commit, cleanup=True) elif dlType == "http": url = retrievalData["url"] ext = retrievalData.get("ext", url[url.rfind(".") :]) filename = os.path.join( script_dir, "download", retrievalData.get("filename", hashlib.md5(url).hexdigest() + ext) ) if not os.path.isfile(filename): with log.info("Downloading %s...", url): http.DownloadFile(url, filename) if (rebuild or not os.path.isdir(destination)) and not retrievalData.get("download-only", False): if rebuild: os_utils.safe_rmtree(destination) os_utils.ensureDirExists(destination) with os_utils.Chdir(destination): os_utils.decompressFile(filename)
def build(self): linebuf = '' nlines = 0 lastbytecount = 0 lastcheck = 0 longest_line = 0 os_utils.ensureDirExists(os.path.dirname(self.target)) nbytes = os.path.getsize(self.subject) with codecs.open(self.subject, 'r', encoding=self.read_encoding) as inf: with codecs.open(self.target + '.out', 'w', encoding=self.write_encoding) as outf: progBar = tqdm.tqdm( total=nbytes, unit='B', leave=False) if self.display_progress else None outf.write(self.text) while True: block = inf.read(4096) block = block.replace('\r\n', '\n') block = block.replace('\r', '\n') if not block: # EOF outf.write(linebuf) nlines += 1 charsInLine = len(linebuf) if charsInLine > longest_line: longest_line = charsInLine break for c in block: nbytes += 1 if self.display_progress: # if nbytes % 10 == 1: cms = utils.current_milli_time() if cms - lastcheck >= 250: progBar.set_postfix({ 'linebuf': len(linebuf), 'nlines': nlines }) progBar.update(nbytes - lastbytecount) lastcheck = cms lastbytecount = nbytes linebuf += c if c in '\r\n': outf.write(linebuf) nlines += 1 charsInLine = len(linebuf) if charsInLine > longest_line: longest_line = charsInLine linebuf = '' if self.display_progress: progBar.close() with log.info('Completed.'): log.info('Lines.......: %d', nlines) log.info('Chars.......: %d', nbytes) log.info('Longest line: %d chars', longest_line) shutil.move(self.target + '.out', self.target)
def __init__(self, sources, destination, rsync_executable=None, progress=False, delete=False, opts=['-Rruhavp'], chmod=0o755, chown=None, chgrp=None, show_output=False, dependencies=[], provides=[], name='rsync', keyfile=None): self.rsync_executable = rsync_executable or os_utils.which('rsync') self.opts = opts self.progress = progress self.chmod = chmod self.chown = chown self.chgrp = chgrp self.show_output = show_output self.progress = progress self.sources = sources self.delete = delete self.destination = destination self.keyfile = keyfile files = [] for source in sources: with log.info('Scanning %s...', source): if os.path.isdir(source): files += os_utils.get_file_list(source) if os.path.isfile(source): files += [source] super().__init__(target=str( Path('.build', hashlib.sha256(self.name).hexdigest() + '.target')), files=files, dependencies=dependencies, provides=provides, name=name)
def __init__(self, sources, destination, rsync_executable=None, progress=False, delete=False, opts=['-Rruavp'], chmod=0o755, chown=None, chgrp=None, show_output=False, dependencies=[], provides=[], name='rsync', keyfile=None): self.rsync_executable = rsync_executable or os_utils.which('rsync') self.opts = opts self.progress = progress self.chmod = chmod self.chown = chown self.chgrp = chgrp self.show_output = show_output self.progress = progress self.sources = sources self.delete = delete self.destination = destination self.keyfile = keyfile files = [] for source in sources: with log.info('Scanning %s...', source): if os.path.isdir(source): files += os_utils.get_file_list(source) if os.path.isfile(source): files += [source] super().__init__(target=self.genVirtualTarget( name.replace('\\', '_').replace('/', '_')), files=files, dependencies=dependencies, provides=provides, name=name)
''' import os,sys, tempfile script_dir=os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(script_dir,'buildtools')) from buildtools import log, Config, Properties, replace_vars from buildtools.wrapper import Ant DefaultConfig={ 'ant': { 'properties':{ 'files':['build.properties'], 'keys': {}, 'secret-keys':[], 'outfile': '/tmp/jenkins-%%JOB_NAME%%.properties' } } } config=Config('build-eclipse.yml',DefaultConfig) outfile=replace_vars(config.get('outfile','/tmp/jenkins-%%JOB_NAME%%.properties'),os.environ) with log.info('Building %s...',outfile): properties=Properties() properties.properties=config.get('ant.properties.keys',{}) for filename in config.get('ant.properties.files',[]): filename=replace_vars(filename,os.environ) properties.Load(filename,expand_vars=os.environ) properties.Save(outfile)
def build(self): if self.archive.endswith('.zip'): with log.info('Extracting %r as ZIP archive...', self.archive): with zipfile.ZipFile(self.archive) as z: z.extractall(path=self.target_dir)
ENV.set("QMAKESPEC", config.get("qt-makespec", "win32-msvc2013")) #: x64 or x86 short_arch = "x64" if config["architecture"] == "x86_64" else "x86" #: 64 or 32 nbits = "64" if config["architecture"] == "x86_64" else "32" superrepo = os.path.join("build", "modorganizer_super") if not os.path.isdir(superrepo): os_utils.ensureDirExists(superrepo) with os_utils.Chdir(superrepo): os_utils.cmd([EXECUTABLES["git"], "init"], show_output=True, critical=True) ymlvars = {"nbits": nbits, "script_dir": script_dir} prerequisites = YAMLConfig("prerequisites.yml", variables=ymlvars).cfg with log.info("Downloading prerequisites..."): dlPackagesIn(prerequisites) # Copied from Unimake. projs = [ ("modorganizer-archive", "archive", "master", ["7zip", "Qt5"]), ("modorganizer-uibase", "uibase", "new_vfs_library", ["Qt5", "boost"]), ("modorganizer-lootcli", "lootcli", "master", ["LootApi", "boost"]), ("modorganizer-esptk", "esptk", "master", ["boost"]), ("modorganizer-bsatk", "bsatk", "master", ["zlib"]), ("modorganizer-nxmhandler", "nxmhandler", "master", ["Qt5"]), ("modorganizer-helper", "helper", "master", ["Qt5"]), ( "modorganizer-game_gamebryo", "game_gamebryo", "new_vfs_library",
def build(self): definition = {} with open(self.filename, 'r') as r: definition=yaml.load(r)['enum'] if 'auto-value' in definition: autoval = definition['auto-value'] i=autoval.get('start',0) for k in definition['values'].keys(): if definition[k].get('auto', True): definition[k]['value']=1 >> i if definition.get('flags', False) else i i += 1 flags = False if 'flags' in definition and definition['flags']: flags=True definition['tests']=definition.get('tests',{}) definition['tests']['unique']=definition['tests'].get('unique',True) definition['tests']['single-bit-only']=definition['tests'].get('single-bit-only',True) default = definition.get('default', 0) for k,vpak in definition['values'].items(): val = self._get_value_for(vpak) if self._get_for(vpak, 'default', False): if flags: default |= val else: default = val if flags or 'tests' in definition: with log.info('Testing %s....', definition['name']): tests = definition.get('tests',{}) if 'increment' in tests: incrdef = tests['increment'] start = incrdef.get('start',0) stop = incrdef.get('stop', len(definition['values'])) vals = [] for k,vpak in definition['values'].items(): vals += [self._get_value_for(vpak)] for i in range(start,stop): if i not in vals: log.error('increment: Missing value %d!', i) if 'unique' in tests and tests['unique']: vals={} for k,vpak in definition['values'].items(): val = self._get_value_for(vpak) if val in vals: log.error('unique: Entry %s is not using a unique value!', k) vals[val]=True if flags: if 'single-bit-only' in tests and tests['single-bit-only']: for k,vpak in definition['values'].items(): val = self._get_value_for(vpak) c = 0 while val > 0: c = val & 1 val >>= 1 if c > 1: log.error('single-bit-only: Entry %s has too many bits!', k) break definition['default'] = default os_utils.ensureDirExists(os.path.dirname(self.target), noisy=True) with open(self.target, 'w') as w: self.writer.write(w, definition)
import os script_dir = os.path.dirname(__file__) from buildtools import log from buildtools.buildsystem.visualstudio import VisualStudio2015Solution, ProjectType target = os.path.join(script_dir, '..', 'NMM', 'NexusClient.sln') csprojfile = os.path.join('..', 'NexusClientCli', 'NexusClientCLI', 'NexusClientCLI.csproj') sln = VisualStudio2015Solution() sln.LoadFromFile(target) changed = False if 'NexusClientCli' not in sln.projectsByName: proj = sln.AddProject('NexusClientCli', ProjectType.CSHARP_PROJECT, csprojfile) log.info('Added %s (%s) to %s', proj.name, proj.guid, target) changed = True else: proj = sln.projectsByName['NexusClientCli'] log.info('Project %s (%s) already exists in %s', proj.name, proj.guid, target) if proj.projectfile != csprojfile: oprojfile = proj.projectfile proj.projectfile = csprojfile log.info('Fixed CSPROJ location: %s -> %s', oprojfile, csprojfile) changed = True if changed: sln.SaveToFile(target + '.patched')
def build(self): gitmodules = {} with open(self.gitmodulesfile, 'r') as tomlf: smid = None for line in tomlf: line = line.strip() m = REG_SUBMODULE_SECTION.match(line) if m is not None: smid = m.group(1).strip() gitmodules[smid] = {} if '=' in line: k, v = line.split('=', 2) gitmodules[smid][k.strip()] = v.strip() gitconfig = {} with open(self.gitconfigfile, 'r') as tomlf: smid = None for line in tomlf: line = line.strip() #print(line) m = REG_SUBMODULE_SECTION.match(line) if m is not None: smid = m.group(1).strip() gitconfig[smid] = {} if smid is not None and '=' in line: #print(line) k, v = line.split('=', 2) gitconfig[smid][k.strip()] = v.strip() ''' with open(self.gitmodulesfile + '.yml', 'w') as f: yaml.dump(gitmodules, f, default_flow_style=False) with open('.gitconfig.yml', 'w') as f: yaml.dump(gitconfig, f, default_flow_style=False) ''' for repoID, repoconf in gitconfig.items(): if repoID not in gitmodules.keys(): with log.warn('Submodule %s is present in .git/config but not .gitmodules!', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) log.info('path = %s', pathspec) for repoID, repoconf in gitmodules.items(): if repoID not in gitconfig.keys(): with log.warn('Submodule %s is present in .gitmodules but not .git/config!', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) opts = [] if branch != 'HEAD': opts += ['-b', branch] log.info('path = %s', pathspec) if os.path.isdir(path): log.warn('Removing existing %s directory.', path) shutil.rmtree(path) cmd = ['git', 'submodule', 'add']+opts+['-f', '--name', repoID, '--', repoconf.get('url'), pathspec] os_utils.cmd(cmd, critical=True, echo=self.should_echo_commands(), show_output=True) #log.error('Would exec: %s', ' '.join(cmd)) for repoID, repoconf in gitmodules.items(): with log.info('Checking %s...', repoID): pathspec = repoconf.get('path', repoID) path = os.path.abspath(pathspec) tag = repoconf.get('tag', None) branch = repoconf.get('branch', 'HEAD' if tag is None else None) if os.path.isdir(path): desired_commit = '' cmdline = ['git', 'ls-tree', Git.GetBranch(), pathspec] stdout, stderr = os_utils.cmd_output(cmdline, echo=self.should_echo_commands(), critical=True) skip_this = False for line in (stdout+stderr).decode('utf-8').splitlines(): if line.startswith('error:') or line.startswith('fatal:'): log.critical(line) raise error.SubprocessThrewError(cmdline, line) line,repoID = line.strip().split('\t') _, _, desired_commit = line.split(' ') if not skip_this: with os_utils.Chdir(path, quiet=not self.should_echo_commands()): cur_commit = Git.GetCommit(short=False, quiet=not self.should_echo_commands()) #log.info(desired_commit) #log.info(cur_commit) if cur_commit == desired_commit: log.info('Commits are synced, skipping.') continue repo = GitRepository(path, origin_uri=repoconf['url'], submodule=True) if repo.CheckForUpdates(branch=branch, quiet=False): if os.path.isdir(path): os_utils.cmd(['git', 'submodule', 'sync', '--', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True) os_utils.cmd(['git', 'submodule', 'update', '--init', '--recursive', pathspec], critical=True, echo=self.should_echo_commands(), show_output=True)
import os, sys, tempfile script_dir = os.path.dirname(os.path.realpath(__file__)) sys.path.append(os.path.join(script_dir, 'buildtools')) from buildtools import log, Config, Properties, replace_vars from buildtools.wrapper import Ant DefaultConfig = { 'ant': { 'properties': { 'files': ['build.properties'], 'keys': {}, 'secret-keys': [], 'outfile': '/tmp/jenkins-%%JOB_NAME%%.properties' } } } config = Config('build-eclipse.yml', DefaultConfig) outfile = replace_vars( config.get('outfile', '/tmp/jenkins-%%JOB_NAME%%.properties'), os.environ) with log.info('Building %s...', outfile): properties = Properties() properties.properties = config.get('ant.properties.keys', {}) for filename in config.get('ant.properties.files', []): filename = replace_vars(filename, os.environ) properties.Load(filename, expand_vars=os.environ) properties.Save(outfile)
def main(): argp = argparse.ArgumentParser(description='Generation script for ss13-vox.') #argp.add_argument('--codebase', choices=['vg', 'tg'], default='vg', help='Which codebase to generate for. (Affects output code and paths.)') argp.add_argument('--threads', '-j', type=int, default=multiprocessing.cpu_count(), help='How many threads to use in ffmpeg.') #argp.add_argument('phrasefiles', nargs='+', type=str, help='A list of phrase files.') args = argp.parse_args() if not os.path.isdir('tmp'): os.makedirs('tmp') DIST_DIR = 'dist' PREEX_SOUND = 'sound/vox/{ID}.wav' NUVOX_SOUND = 'sound/vox_{SEX}/{ID}.wav' voices = [] vox_sounds_path = '' templatefile = '' config = BaseConfig() config.cfg = YAMLConfig('config.yml') pathcfg = BaseConfig() pathcfg.cfg = YAMLConfig('paths.yml').cfg[config.get('codebase', 'vg')] PREEX_SOUND = pathcfg.get('sound.old-vox', PREEX_SOUND) NUVOX_SOUND = pathcfg.get('sound.new-vox', NUVOX_SOUND) voice_assignments = {} all_voices = [] default_voice: Voice = VoiceRegistry.Get(USSLTFemale.ID) sfx_voice: SFXVoice = SFXVoice() configured_voices: Dict[str, dict] = {} for sexID, voiceid in config.get('voices', {'fem': USSLTFemale.ID}).items(): voice = VoiceRegistry.Get(voiceid) assert sexID != '' voice.assigned_sex = sexID if sexID in ('fem', 'mas'): sex = EVoiceSex(sexID) assert voice.SEX == sex voices += [voice] elif sexID == 'default': default_voice = voice voice_assignments[voice.assigned_sex] = [] all_voices += [voice] configured_voices[sexID] = voice.serialize() voice_assignments[sfx_voice.assigned_sex] = [] all_voices += [sfx_voice] configured_voices[sfx_voice.assigned_sex] = sfx_voice.serialize() vox_sounds_path = os.path.join(DIST_DIR, pathcfg.get('vox_sounds.path')) templatefile = pathcfg.get('vox_sounds.template') vox_data_path = os.path.join(DIST_DIR, pathcfg.get('vox_data')) DATA_DIR = os.path.join(DIST_DIR, 'data') os_utils.ensureDirExists(DATA_DIR) with log.info('Parsing lexicon...'): lexicon = ParseLexiconText('lexicon.txt') phrases=[] phrasesByID = {} broked = False for filename in config.get('phrasefiles', ['announcements.txt', 'voxwords.txt']): for p in ParsePhraseListFrom(filename): if p.id in phrasesByID: duplicated = phrasesByID[p.id] log.critical('Duplicate phrase with ID %s in file %s on line %d! First instance in file %s on line %d.', p.id, p.deffile, p.defline, duplicated.deffile, duplicated.defline) broked = True continue phrases += [p] phrasesByID[p.id] = p if broked: sys.exit(1) soundsToKeep = set() for sound in OTHERSOUNDS: soundsToKeep.add(os.path.join(DIST_DIR, sound + '.ogg')) phrases.sort(key=lambda x: x.id) overrides = config.get('overrides', {}) for phrase in phrases: if phrase.id in overrides: phrase.fromOverrides(overrides.get(phrase.id)) phrase_voices = list(voices) # If it has a path, it's being manually specified. if '/' in phrase.id: phrase.filename = phrase.id + '.ogg' phrase_voices = [default_voice] soundsToKeep.add(os.path.abspath(os.path.join(DIST_DIR, phrase.filename))) else: phrase.filename = ''+NUVOX_SOUND if phrase.hasFlag(EPhraseFlags.OLD_VOX): phrase_voices = [default_voice] phrase.filename = PREEX_SOUND.format(ID=phrase.id) for voice in ['fem', 'mas']: phrase.files[voice] = FileData() phrase.files[voice].filename = phrase.filename phrase.files[voice].checksum = '' phrase.files[voice].duration = phrase.override_duration or -1 phrase.files[voice].size = phrase.override_size or -1 #voice_assignments[voice].append(phrase) soundsToKeep.add(os.path.abspath(os.path.join(DIST_DIR, phrase.filename))) continue if phrase.hasFlag(EPhraseFlags.SFX): phrase_voices = [sfx_voice] if not phrase.hasFlag(EPhraseFlags.OLD_VOX): log.info('%s - %r', phrase.id, [x.assigned_sex for x in phrase_voices]) for v in phrase_voices: voice_assignments[v.assigned_sex].append(phrase) #phrase.files[v.assigned_sex] = fd #sys.exit(1) for voice in all_voices: print(voice.ID, voice.assigned_sex) DumpLexiconScript(voice.FESTIVAL_VOICE_ID, lexicon.values(), 'tmp/VOXdict.lisp') for phrase in voice_assignments[voice.assigned_sex]: GenerateForWord(phrase, voice, soundsToKeep, args) sexes=set() for vk, fd in phrase.files.items(): soundsToKeep.add(os.path.abspath(os.path.join(DIST_DIR, fd.filename))) jenv = jinja2.Environment(loader=jinja2.FileSystemLoader(['./templates'])) jenv.add_extension('jinja2.ext.do') # {% do ... %} templ = jenv.get_template(templatefile) with log.info('Writing sound list to %s...', vox_sounds_path): os_utils.ensureDirExists(os.path.dirname(vox_sounds_path)) assetcache={} sound2id={} with open(vox_sounds_path, 'w') as f: sexes = { 'fem': [], 'mas': [], 'default': [], #'sfx': [], } for p in phrases: for k in p.files.keys(): assetcache[p.getAssetKey(k)] = p.files[k].filename sound2id[p.files[k].filename] = p.getAssetKey(k) if p.hasFlag(EPhraseFlags.NOT_VOX): continue for k in p.files.keys(): if p.hasFlag(EPhraseFlags.SFX): for sid in ('fem', 'mas'): if p not in sexes[sid]: sexes[sid].append(p) else: sexes[k].append(p) f.write(templ.render( InitClass=InitClass, SEXES=sexes, ASSETCACHE=assetcache, SOUND2ID=sound2id, PHRASES=[p for p in phrases if not p.hasFlag(EPhraseFlags.NOT_VOX)])) soundsToKeep.add(os.path.abspath(vox_sounds_path)) os_utils.ensureDirExists(DATA_DIR) with open(os.path.join(DATA_DIR, 'vox_data.json'), 'w') as f: data = { 'version': 2, 'compiled': time.time(), 'voices': configured_voices, 'words': collections.OrderedDict({w.id: w.serialize() for w in phrases if '/' not in w.id}), } json.dump(data, f, indent=2) soundsToKeep.add(os.path.abspath(os.path.join(DATA_DIR, 'vox_data.json'))) with open('tmp/written.txt', 'w') as f: for filename in sorted(soundsToKeep): f.write(f'{filename}\n') for root, _, files in os.walk(DIST_DIR, topdown=False): for name in files: filename = os.path.abspath(os.path.join(root, name)) if filename not in soundsToKeep: log.warning('Removing {0} (no longer defined)'.format(filename)) os.remove(filename)
def prepare_nmm(context): sln = VisualStudio2015Solution() sln.LoadFromFile(os.path.join(build_path, "Nexus-Mod-Manager", 'NexusClient.sln')) ncc_csproj = os.path.join(build_path, 'NexusClientCLI', 'NexusClientCLI', 'NexusClientCLI.csproj') if not os.path.isfile(ncc_csproj): log.critical('NOT FOUND: %s', ncc_csproj) else: log.info('FOUND: %s', ncc_csproj) changed = False projfile = VS2015Project() projfile.LoadFromFile(ncc_csproj) projguid = projfile.PropertyGroups[0].element.find('ProjectGuid').text log.info('ProjectGuid = %s', projguid) if "NexusClientCli" not in sln.projectsByName: newproj = sln.AddProject('NexusClientCli', ProjectType.CSHARP_PROJECT, ncc_csproj, guid=projguid) log.info('Adding project %s (%s) to NexusClient.sln', newproj.name, newproj.guid) changed = True else: newproj = sln.projectsByName['NexusClientCli'] log.info('Project %s (%s) already exists in NexusClient.sln', newproj.name, newproj.guid) if newproj.projectfile != ncc_csproj: log.info('Changing projectfile: %s -> %s', newproj.projectfile, ncc_csproj) newproj.projectfile = ncc_csproj changed = True if changed: log.info('Writing NexusClientCli.sln') sln.SaveToFile(os.path.relpath(os.path.join(build_path, "Nexus-Mod-Manager", 'NexusClientCli.sln'))) # So we dont get conflicts when pulling return True
import os script_dir = os.path.dirname(__file__) from buildtools import log from buildtools.buildsystem.visualstudio import VisualStudio2015Solution, ProjectType target=os.path.join(script_dir, '..', 'NMM','NexusClient.sln') csprojfile = os.path.join('..','NexusClientCli','NexusClientCLI','NexusClientCLI.csproj') sln = VisualStudio2015Solution() sln.LoadFromFile(target) changed=False if 'NexusClientCli' not in sln.projectsByName: proj = sln.AddProject('NexusClientCli',ProjectType.CSHARP_PROJECT,csprojfile) log.info('Added %s (%s) to %s', proj.name, proj.guid, target) changed=True else: proj = sln.projectsByName['NexusClientCli'] log.info('Project %s (%s) already exists in %s', proj.name, proj.guid, target) if proj.projectfile != csprojfile: oprojfile = proj.projectfile proj.projectfile = csprojfile log.info('Fixed CSPROJ location: %s -> %s',oprojfile,csprojfile) changed=True if changed: sln.SaveToFile(target+'.patched')
def GenerateForWord(phrase: Phrase, voice: Voice, writtenfiles: set, args: Optional[argparse.Namespace] = None): global PHRASELENGTHS, OLD_SFX, KNOWN_PHONEMES, OTHERSOUNDS my_phonemes = {} if phrase.hasFlag(EPhraseFlags.OLD_VOX): log.info('Skipping %s.ogg (Marked as OLD_VOX)', phrase.id) return if phrase.hasFlag(EPhraseFlags.NOT_VOX): OTHERSOUNDS += [phrase.id] if phrase.parsed_phrase is not None: for _word in phrase.parsed_phrase: _word = _word.lower() if _word in KNOWN_PHONEMES: my_phonemes[_word] = KNOWN_PHONEMES[_word].toLisp().replace('\n', '') filename = phrase.filename.format(ID=phrase.id, SEX=voice.assigned_sex) sox_args = voice.genSoxArgs(args) md5 = json.dumps(phrase.serialize()) md5 += '\n'.join(my_phonemes.values()) md5 += ''.join(sox_args) + PRE_SOX_ARGS + ''.join(RECOMPRESS_ARGS) md5 += voice.ID md5 += filename #filename = os.path.join('sound', 'vox_fem', phrase.id + '.ogg') #if '/' in phrase.id: # filename = os.path.join(phrase.id + '.ogg') oggfile = os.path.abspath(os.path.join('dist', filename)) cachebase = os.path.abspath(os.path.join('cache', phrase.id.replace(os.sep, '_').replace('.', ''))) checkfile = cachebase + voice.ID + '.dat' cachefile = cachebase + voice.ID + '.json' fdata = FileData() fdata.voice = voice.ID fdata.filename = os.path.relpath(oggfile, 'dist') def commitWritten(): nonlocal phrase, voice, oggfile, writtenfiles, fdata if voice.ID == SFXVoice.ID: # Both masculine and feminine voicepacks link to SFX. for sex in ['fem', 'mas']: phrase.files[sex] = fdata else: phrase.files[voice.assigned_sex] = fdata writtenfiles.add(os.path.abspath(oggfile)) parent = os.path.dirname(oggfile) if not os.path.isdir(parent): os.makedirs(parent) parent = os.path.dirname(cachefile) if not os.path.isdir(parent): os.makedirs(parent) if os.path.isfile(oggfile) and os.path.isfile(cachefile): old_md5 = '' if os.path.isfile(checkfile): with open(checkfile, 'r') as md5f: old_md5 = md5f.read() if old_md5 == md5: cachedata = {} with open(cachefile, 'r') as cachef: cachedata = json.load(cachef) fdata.deserialize(cachedata) log.info('Skipping {0} for {1} (exists)'.format(filename, voice.ID)) commitWritten() return log.info('Generating {0} for {1} ({2!r})'.format(filename, voice.ID, phrase.phrase)) text2wave = None if phrase.hasFlag(EPhraseFlags.SFX): text2wave = 'ffmpeg -i '+phrase.phrase+' tmp/VOX-word.wav' else: with open('tmp/VOX-word.txt', 'w') as wf: wf.write(phrase.phrase) text2wave = 'text2wave tmp/VOX-word.txt -o tmp/VOX-word.wav' if os.path.isfile('tmp/VOXdict.lisp'): text2wave = 'text2wave -eval tmp/VOXdict.lisp tmp/VOX-word.txt -o tmp/VOX-word.wav' with open(checkfile, 'w') as wf: wf.write(md5) for fn in ('tmp/VOX-word.wav', 'tmp/VOX-soxpre-word.wav', 'tmp/VOX-sox-word.wav', 'tmp/VOX-encoded.ogg'): if os.path.isfile(fn): os.remove(fn) cmds = [] cmds += [(text2wave.split(' '), 'tmp/VOX-word.wav')] if not phrase.hasFlag(EPhraseFlags.NO_PROCESS) or not phrase.hasFlag(EPhraseFlags.NO_TRIM): cmds += [(['sox', 'tmp/VOX-word.wav', 'tmp/VOX-soxpre-word.wav'] + PRE_SOX_ARGS.split(' '), 'tmp/VOX-soxpre-word.wav')] if not phrase.hasFlag(EPhraseFlags.NO_PROCESS): cmds += [(['sox', cmds[-1][1], 'tmp/VOX-sox-word.wav'] + sox_args, 'tmp/VOX-sox-word.wav')] cmds += [(['oggenc', cmds[-1][1], '-o', 'tmp/VOX-encoded.ogg'], 'tmp/VOX-encoded.ogg')] cmds += [(['ffmpeg', '-i', 'tmp/VOX-encoded.ogg']+RECOMPRESS_ARGS+['-threads',args.threads]+[oggfile], oggfile)] for command_spec in cmds: (command, cfn) = command_spec with os_utils.TimeExecution(command[0]): os_utils.cmd(command, echo=True, critical=True, show_output=command[0] in ('text2wave',)) command = ['ffprobe', '-v', 'quiet', '-print_format', 'json', '-show_format', '-show_streams', oggfile] with os_utils.TimeExecution(command[0]): captured = os_utils.cmd_out(command, echo=False, critical=True) fdata.fromJSON(json.loads(captured)) fdata.checksum = md5sum(oggfile) for command_spec in cmds: (command, cfn) = command_spec if not os.path.isfile(fn): log.error("File '{0}' doesn't exist, command '{1}' probably failed!".format(cfn, command)) sys.exit(1) with open(cachefile, 'w') as f: json.dump(fdata.serialize(), f) commitWritten()