def make(self): if self.config == "android": target_cpu = "arm" elif self.config == "32bit": target_cpu = "x86" elif self.config == "64bit": target_cpu = "x64" else: raise Exception("Unknown config in V8Builder.make!") objdir = os.path.realpath(self.objdir()) if not os.path.isdir(objdir): out_dir = os.path.join(self.folder, 'v8', 'out') if not os.path.isdir(out_dir): os.mkdir(out_dir) os.mkdir(objdir) with utils.chdir(os.path.join(self.folder, 'v8')): config = [ 'is_debug = false', 'target_cpu = "{}"'.format(target_cpu) ] if self.config == "arm": config += [ 'symbol_level = 1', 'v8_android_log_stdout = true', 'target_os = "android"' ] args = 'gn gen ' + objdir + ' --args=\'' + " ".join(config) + '\'' Run(args, self.env.get(), shell=True) Run(["ninja", "-C", objdir, "d8"], self.env.get())
def Update(rev = None): assert rev == None env = os.environ.copy() with FolderChanger('..'): Run(['gclient', 'sync'], {"PATH": "depot_tools/:"+env["PATH"]}) output = Run(['git', 'pull', 'origin', 'master']) return re.search("Already up-to-date", output) == None
def patch(self): with utils.chdir(self.folder): # Hack 1: Remove reporting errors for warnings that currently are present. Run([ "sed", "-i.bac", "s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/", "Source/JavaScriptCore/Configurations/Base.xcconfig" ]) Run([ "sed", "-i.bac", "s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/", "Source/bmalloc/Configurations/Base.xcconfig" ]) Run([ "sed", "-i.bac", "s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/", "Source/WTF/Configurations/Base.xcconfig" ]) Run([ "sed", "-i.bac", "s/std::numeric_limits<unsigned char>::max()/255/", "Source/bmalloc/bmalloc/SmallLine.h" ]) #Run(["sed","-i.bac","s/std::numeric_limits<unsigned char>::max()/255/","Source/bmalloc/bmalloc/SmallRun.h"]) # Hack 2: This check fails currently. Disable checking to still have a build. os.remove("Tools/Scripts/check-for-weak-vtables-and-externals")
def build(self): env = os.environ.copy() if self.cxx is not None: env['CXX'] = self.cxx if self.cc is not None: env['CC'] = self.cc if self.cpp is not None: env['CPP'] = self.cpp if self.link is not None: env['LINK'] = self.link if self.cxx_host is not None: env['CXX_host'] = self.cxx_host if self.cc_host is not None: env['CC_host'] = self.cc_host if self.cpp_host is not None: env['CPP_host'] = self.cpp_host if self.link_host is not None: env['LINK_host'] = self.link_host env["GYP_DEFINES"] = "clang=1" if self.cpu == 'x64': Run(['make', 'x64.release', '-j3'], env) elif self.cpu == 'arm': if self.hardfp: Run([ 'make', 'arm.release', 'hardfp=on', 'i18nsupport=off', '-j3' ], env) else: Run(['make', 'arm.release', 'i18nsupport=off', '-j3'], env) elif self.cpu == 'x86': Run(['make', 'ia32.release', '-j3'], env)
def clone(self): os.mkdir(self.folder) with chdir(self.folder): # get depot_tools Run(['git', 'clone', 'https://chromium.googlesource.com/chromium/tools/depot_tools.git']) # get actual v8 source Run(['fetch', 'v8'], env=self.make_env())
def __init__(self): super(NativeCompiler, self).__init__() self.cc = utils.config.get('native', 'cc') self.cxx = utils.config.get('native', 'cxx') self.args = utils.config.get('native', 'options').split(' ') self.mode = utils.config.get('native', 'mode') output = Run([self.cxx, '--version']) self.signature = output.splitlines()[0].strip()
def update(self, rev = None): assert rev == None with FolderChanger(self.path()): Run(['git', 'pull', 'origin', 'master']) env = os.environ.copy() with FolderChanger(self.path()): Run(['gclient', 'sync'], {"PATH": os.path.abspath("../depot_tools/")+":"+env["PATH"]})
def clone(self): os.mkdir(self.folder) with FolderChanger(self.folder): # get depot_tools Run(['git', 'clone', 'https://chromium.googlesource.com/chromium/tools/depot_tools.git']) # get actual v8 source env = os.environ.copy() Run(['fetch', 'v8'], {"PATH": "depot_tools/:"+env["PATH"]})
def __init__(self): super(NativeCompiler, self).__init__() self.cc = utils.config.get("native", "cc") self.cxx = utils.config.get("native", "cxx") self.args = utils.config.get("native", "options").split(" ") self.mode = utils.config.get("native", "mode") output = Run([self.cxx, "--version"]) self.signature = output.splitlines()[0].strip()
def Update(rev=None): output = Run(['hg', 'pull', '-u']) succeeded = re.search("no changes found", output) == None if not rev: return succeeded output = Run(['hg', 'update', '-r', rev]) if re.search("unknown revision", output) != None: raise Exception('unknown revision: ' + output) return True
def update(self, rev = None): assert rev == None with chdir(self.path()): Run(['git', 'pull', 'origin', 'master']) env = os.environ.copy() with chdir(self.path()): Run(['gclient', 'sync'], self.make_env())
def update(self, rev = None): assert rev != None output = Run(['hg', 'up', '--check', '--cwd', self.folder]) output = Run(['hg', 'pull', '--cwd', self.folder]) output = Run(['hg', 'pull', '-r', rev, '--cwd', self.folder, "https://hg.mozilla.org/try"]) output = Run(['hg', 'update', '-r', rev, '--cwd', self.folder]) if re.search("unknown revision", output) != None: raise Exception('unknown revision: ' + output) return
def Update(rev=None): output = Run(['svn', 'update']) succeeded = re.search("At revision", output) == None if not rev: return succeeded output = Run(['svn', 'update', '-r', rev]) if re.search("No such revision", output) != None: raise Exception('unknown revision: ' + output) return True
def _update(self, rev): # sourcePath = os.path.join(self.repoPath, self.source) with utils.FolderChanger(self.sourcePath): env = os.environ.copy() # reset to master branch Run(['git', 'reset', '--hard'], env) Run(['git', 'checkout', 'master'], env) Run(['git', 'fetch'], env) Run(['git', 'reset', '--hard', rev]) Run(['gclient', 'sync', '-D', '-j8', '-f'], env)
def update(self, rev = None): with chdir(self.path()): if not rev: output = Run(['svn', 'update']) return output = Run(['svn', 'update', '-r', rev]) if re.search("No such revision", output) != None: raise Exception('unknown revision: ' + output) return
def update(self, rev = None): output = Run(['hg', 'pull', '--cwd', self.folder]) if not rev: output = Run(['hg', 'update', '--cwd', self.folder]) return else: output = Run(['hg', 'update', '-r', rev, '--cwd', self.folder]) if re.search("unknown revision", output) != None: raise Exception('unknown revision: ' + output) return
def make(self): try: self.patch() with utils.FolderChanger(os.path.join(self.folder, 'Tools', 'Scripts')): args = ['/usr/bin/perl', 'build-jsc'] if self.config == '32bit': args += ['--32-bit'] Run(args, self.env.get()) finally: self.clean() Run(["install_name_tool", "-change", "/System/Library/Frameworks/JavaScriptCore.framework/Versions/A/JavaScriptCore", self.objdir()+"/JavaScriptCore.framework/JavaScriptCore", self.objdir() + "/jsc"])
def build(self): Run(['make', 'dependencies']) if self.cpu == 'x64': Run(['make', 'x64.release']) elif self.cpu == 'arm': if self.hardfp: Run(['make', 'arm.release', 'hardfp=on']) else: Run(['make', 'arm.release']) elif self.cpu == 'x86': Run(['make', 'ia32.release'])
def build(self): env = os.environ.copy() # env["NO_AUTH_BOTO_CONFIG"] = "/repos/boto.cfg" # sourcePath = os.path.join(self.repoPath, self.source) # add build command code here with utils.FolderChanger(self.sourcePath): try: in_argns_name = self.cpu + ".gn" in_argns = os.path.join(self.dirname, 'gn_file', in_argns_name) out_argns = os.path.join(self.repoPath, self.source, 'out', self.cpu, 'args.gn') if not os.path.isdir( os.path.join(self.repoPath, self.source, 'out', self.cpu)): os.mkdir( os.path.join(self.repoPath, self.source, 'out', self.cpu)) Run(['copy', in_argns, out_argns], env) Run([ 'gn', 'gen', os.path.join(self.sourcePath, 'out', self.cpu) ], env) Run([ 'ninja', '-C', os.path.join(self.sourcePath, 'out', self.cpu), 'chrome', 'mini_installer', '-j40' ], env) except subprocess.CalledProcessError as e: print("Dirty build failed!") try: in_argns_name = self.cpu + ".gn" in_argns = os.path.join(self.dirname, 'gn_file', in_argns_name) out_argns = os.path.join(self.repoPath, self.source, 'out', self.cpu, 'args.gn') Run([ 'rmdir', '/s', '/q', os.path.join(self.repoPath, self.source, 'out', self.cpu) ]) Run([ 'mkdir', os.path.join(self.repoPath, self.source, 'out', self.cpu) ]) Run(['copy', in_argns, out_argns], env) Run([ 'gn', 'gen', os.path.join(self.sourcePath, 'out', self.cpu) ], env) Run([ 'ninja', '-C', os.path.join(self.sourcePath, 'out', self.cpu), 'chrome', 'mini_installer', '-j40' ], env) except subprocess.CalledProcessError as e: print("Clean build failed!") raise e
def reconf(self): # Step 1. autoconf. with utils.FolderChanger(os.path.join(self.folder, 'js', 'src')): if platform.system() == "Darwin": utils.Shell("autoconf213") elif platform.system() == "Linux": utils.Shell("autoconf2.13") elif platform.system() == "Windows": utils.Shell("autoconf-2.13") # Step 2. configure if os.path.exists(os.path.join(self.folder, 'js', 'src', 'Opt')): shutil.rmtree(os.path.join(self.folder, 'js', 'src', 'Opt')) os.mkdir(os.path.join(self.folder, 'js', 'src', 'Opt')) with utils.FolderChanger(os.path.join(self.folder, 'js', 'src', 'Opt')): args = ['--enable-optimize', '--disable-debug'] if platform.architecture()[0] == "64bit" and self.config == "32bit": if platform.system() == "Darwin": args.append("--target=i686-apple-darwin10.0.0") elif platform.system() == "Linux": args.append("--target=i686-pc-linux-gnu") else: assert False Run(['../configure'] + args, self.env.get()) return True
def clean(self): with utils.chdir(self.folder): Run([ "svn", "revert", "Tools/Scripts/check-for-weak-vtables-and-externals" ]) Run([ "svn", "revert", "Source/JavaScriptCore/Configurations/Base.xcconfig" ]) Run([ "svn", "revert", "Source/bmalloc/Configurations/Base.xcconfig" ]) Run(["svn", "revert", "Source/WTF/Configurations/Base.xcconfig"]) Run(["svn", "revert", "Source/bmalloc/bmalloc/SmallLine.h"])
def _move_zip(self, rev=None, remote=True): name = "chromium_" name += now() if rev and type(rev) == str: name += '_rev_' name += rev name += '.7z' print(name) src = self.libpaths() # if not os.path.isdir(self.out_path): # os.mkdir(self.out_path) # dest = os.path.join(self.out_path, name) result = {'status': 1, 'msg': None} # try: # utils.move(src=src, dest=dest) # except Exception as e: # result['msg'] = e # result['status'] = -5 if os.path.exists(src): if remote: try: Run(['scp', src, self.remote_out_path + name]) print('scp to remote apache2 web succeed!') result['msg'] = self.remote_url + name except Exception as e: result['status'] = -6 result['msg'] = e else: print("move to apache2 web succeed!") result['msg'] = self.url + name else: result['status'] = -5 result['msg'] = "Cannot find moved 7zip file in web folder!" return result
def identify(self): with chdir(self.path()): output = Run(['svn', 'info']) m = re.search("Revision: ([0-9]+)", output) if m == None: raise Exception('unknown output from svn: ' + output) return m.group(1)
def identify(self): with chdir(self.path()): output = Run(['git', 'log', '-1']) m = re.match("commit ([0-9a-z]+)\s*", output) if m == None: raise Exception('unknown output from git: ' + output) return m.group(1)
def build(self): with utils.FolderChanger(os.path.join('Tools', 'Scripts')): if self.cpu == 'x86': args = ['/usr/bin/perl', 'build-jsc', '--32-bit'] else: args = ['/usr/bin/perl', 'build-jsc'] args.extend(self.extra) Run(args)
def run(cfg, train_loader, tr_comp, saver, trainer, valid_dict): # TODO resume # trainer = Engine(...) # trainer.load_state_dict(state_dict) # trainer.run(data) # checkpoint handler = ModelCheckpoint(saver.model_dir, 'train', n_saved=3, create_dir=True) checkpoint_params = tr_comp.state_dict() trainer.add_event_handler(Events.EPOCH_COMPLETED, handler, checkpoint_params) timer = Timer(average=True) timer.attach(trainer, start=Events.EPOCH_STARTED, resume=Events.ITERATION_STARTED, pause=Events.ITERATION_COMPLETED, step=Events.ITERATION_COMPLETED) # average metric to attach on trainer names = ["Acc", "Loss"] names.extend(tr_comp.loss_function_map.keys()) for n in names: RunningAverage(output_transform=Run(n)).attach(trainer, n) @trainer.on(Events.EPOCH_COMPLETED) def adjust_learning_rate(engine): tr_comp.scheduler.step() @trainer.on(Events.ITERATION_COMPLETED(every=cfg.TRAIN.LOG_ITER_PERIOD)) def log_training_loss(engine): message = f"Epoch[{engine.state.epoch}], " + \ f"Iteration[{engine.state.iteration}/{len(train_loader)}], " + \ f"Base Lr: {tr_comp.scheduler.get_last_lr()[0]:.2e}, " for loss_name in engine.state.metrics.keys(): message += f"{loss_name}: {engine.state.metrics[loss_name]:.4f}, " if tr_comp.xent and tr_comp.xent.learning_weight: message += f"xentWeight: {tr_comp.xent.uncertainty.mean().item():.4f}, " logger.info(message) # adding handlers using `trainer.on` decorator API @trainer.on(Events.EPOCH_COMPLETED) def print_times(engine): logger.info('Epoch {} done. Time per batch: {:.3f}[s] Speed: {:.1f}[samples/s]' .format(engine.state.epoch, timer.value() * timer.step_count, train_loader.batch_size / timer.value())) logger.info('-' * 80) timer.reset() @trainer.on(Events.EPOCH_COMPLETED(every=cfg.EVAL.EPOCH_PERIOD)) def log_validation_results(engine): logger.info(f"Valid - Epoch: {engine.state.epoch}") eval_multi_dataset(cfg, valid_dict, tr_comp) trainer.run(train_loader, max_epochs=cfg.TRAIN.MAX_EPOCHS)
def make(self): try: self.patch() with utils.FolderChanger(os.path.join(self.folder, 'Tools', 'Scripts')): args = ['/usr/bin/perl', 'build-jsc'] if self.config == '32bit': args += ['--32-bit'] Run(args, self.env.get()) finally: self.clean()
def make(self): args = ['make', '-j6', '-C', os.path.join(self.folder, 'v8')] if self.config == '32bit': args += ['ia32.release'] elif self.config == '64bit': args += ['x64.release'] else: assert True Run(args, self.env.get())
def make(self): args = ['make', '-j6'] if self.config == '32bit': args += ['ia32.release'] elif self.config == '64bit': args += ['x64.release'] else: assert True with utils.FolderChanger(os.path.join(self.folder, 'v8')): Run(args, self.env.get())
def sameRepo(self): with FolderChanger(self.path()): try: output = Run(['svn', 'info']) except: return False print self.repo print output if "URL: " + self.repo in output: return True exit() return False
def sameRepo(self): if not os.path.exists(self.path()): return False with chdir(self.path()): try: output = Run(['svn', 'info']) except: return False print self.repo print output if "URL: "+self.repo in output: return True exit() return False