def make(self):
if self.config == "android":
target_cpu = "arm"
elif self.config == "32bit":
target_cpu = "x86"
elif self.config == "64bit":
target_cpu = "x64"
else:
raise Exception("Unknown config in V8Builder.make!")
objdir = os.path.realpath(self.objdir())
if not os.path.isdir(objdir):
out_dir = os.path.join(self.folder, 'v8', 'out')
if not os.path.isdir(out_dir):
os.mkdir(out_dir)
os.mkdir(objdir)
with utils.chdir(os.path.join(self.folder, 'v8')):
config = [
'is_debug = false', 'target_cpu = "{}"'.format(target_cpu)
]
if self.config == "arm":
config += [
'symbol_level = 1', 'v8_android_log_stdout = true',
'target_os = "android"'
]
args = 'gn gen ' + objdir + ' --args=\'' + " ".join(config) + '\''
Run(args, self.env.get(), shell=True)
Run(["ninja", "-C", objdir, "d8"], self.env.get())
def Update(rev = None):
assert rev == None
env = os.environ.copy()
with FolderChanger('..'):
Run(['gclient', 'sync'], {"PATH": "depot_tools/:"+env["PATH"]})
output = Run(['git', 'pull', 'origin', 'master'])
return re.search("Already up-to-date", output) == None
def patch(self):
with utils.chdir(self.folder):
# Hack 1: Remove reporting errors for warnings that currently are present.
Run([
"sed", "-i.bac",
"s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/",
"Source/JavaScriptCore/Configurations/Base.xcconfig"
])
Run([
"sed", "-i.bac",
"s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/",
"Source/bmalloc/Configurations/Base.xcconfig"
])
Run([
"sed", "-i.bac",
"s/GCC_TREAT_WARNINGS_AS_ERRORS = YES;/GCC_TREAT_WARNINGS_AS_ERRORS=NO;/",
"Source/WTF/Configurations/Base.xcconfig"
])
Run([
"sed", "-i.bac",
"s/std::numeric_limits<unsigned char>::max()/255/",
"Source/bmalloc/bmalloc/SmallLine.h"
])
#Run(["sed","-i.bac","s/std::numeric_limits<unsigned char>::max()/255/","Source/bmalloc/bmalloc/SmallRun.h"])
# Hack 2: This check fails currently. Disable checking to still have a build.
os.remove("Tools/Scripts/check-for-weak-vtables-and-externals")
def build(self):
env = os.environ.copy()
if self.cxx is not None:
env['CXX'] = self.cxx
if self.cc is not None:
env['CC'] = self.cc
if self.cpp is not None:
env['CPP'] = self.cpp
if self.link is not None:
env['LINK'] = self.link
if self.cxx_host is not None:
env['CXX_host'] = self.cxx_host
if self.cc_host is not None:
env['CC_host'] = self.cc_host
if self.cpp_host is not None:
env['CPP_host'] = self.cpp_host
if self.link_host is not None:
env['LINK_host'] = self.link_host
env["GYP_DEFINES"] = "clang=1"
if self.cpu == 'x64':
Run(['make', 'x64.release', '-j3'], env)
elif self.cpu == 'arm':
if self.hardfp:
Run([
'make', 'arm.release', 'hardfp=on', 'i18nsupport=off',
'-j3'
], env)
else:
Run(['make', 'arm.release', 'i18nsupport=off', '-j3'], env)
elif self.cpu == 'x86':
Run(['make', 'ia32.release', '-j3'], env)
def clone(self):
os.mkdir(self.folder)
with chdir(self.folder):
# get depot_tools
Run(['git', 'clone', 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'])
# get actual v8 source
Run(['fetch', 'v8'], env=self.make_env())
def __init__(self):
super(NativeCompiler, self).__init__()
self.cc = utils.config.get('native', 'cc')
self.cxx = utils.config.get('native', 'cxx')
self.args = utils.config.get('native', 'options').split(' ')
self.mode = utils.config.get('native', 'mode')
output = Run([self.cxx, '--version'])
self.signature = output.splitlines()[0].strip()
def __init__(self):
super(NativeCompiler, self).__init__()
self.cc = utils.config.get('native', 'cc')
self.cxx = utils.config.get('native', 'cxx')
self.args = utils.config.get('native', 'options').split(' ')
self.mode = utils.config.get('native', 'mode')
output = Run([self.cxx, '--version'])
self.signature = output.splitlines()[0].strip()
def update(self, rev = None):
assert rev == None
with FolderChanger(self.path()):
Run(['git', 'pull', 'origin', 'master'])
env = os.environ.copy()
with FolderChanger(self.path()):
Run(['gclient', 'sync'], {"PATH": os.path.abspath("../depot_tools/")+":"+env["PATH"]})
def clone(self):
os.mkdir(self.folder)
with FolderChanger(self.folder):
# get depot_tools
Run(['git', 'clone', 'https://chromium.googlesource.com/chromium/tools/depot_tools.git'])
# get actual v8 source
env = os.environ.copy()
Run(['fetch', 'v8'], {"PATH": "depot_tools/:"+env["PATH"]})
def __init__(self):
super(NativeCompiler, self).__init__()
self.cc = utils.config.get("native", "cc")
self.cxx = utils.config.get("native", "cxx")
self.args = utils.config.get("native", "options").split(" ")
self.mode = utils.config.get("native", "mode")
output = Run([self.cxx, "--version"])
self.signature = output.splitlines()[0].strip()
def Update(rev=None):
output = Run(['hg', 'pull', '-u'])
succeeded = re.search("no changes found", output) == None
if not rev:
return succeeded
output = Run(['hg', 'update', '-r', rev])
if re.search("unknown revision", output) != None:
raise Exception('unknown revision: ' + output)
return True
def update(self, rev = None):
assert rev == None
with chdir(self.path()):
Run(['git', 'pull', 'origin', 'master'])
env = os.environ.copy()
with chdir(self.path()):
Run(['gclient', 'sync'], self.make_env())
def update(self, rev = None):
assert rev != None
output = Run(['hg', 'up', '--check', '--cwd', self.folder])
output = Run(['hg', 'pull', '--cwd', self.folder])
output = Run(['hg', 'pull', '-r', rev, '--cwd', self.folder, "https://hg.mozilla.org/try"])
output = Run(['hg', 'update', '-r', rev, '--cwd', self.folder])
if re.search("unknown revision", output) != None:
raise Exception('unknown revision: ' + output)
return
def Update(rev=None):
output = Run(['svn', 'update'])
succeeded = re.search("At revision", output) == None
if not rev:
return succeeded
output = Run(['svn', 'update', '-r', rev])
if re.search("No such revision", output) != None:
raise Exception('unknown revision: ' + output)
return True
def _update(self, rev):
# sourcePath = os.path.join(self.repoPath, self.source)
with utils.FolderChanger(self.sourcePath):
env = os.environ.copy()
# reset to master branch
Run(['git', 'reset', '--hard'], env)
Run(['git', 'checkout', 'master'], env)
Run(['git', 'fetch'], env)
Run(['git', 'reset', '--hard', rev])
Run(['gclient', 'sync', '-D', '-j8', '-f'], env)
def update(self, rev = None):
with chdir(self.path()):
if not rev:
output = Run(['svn', 'update'])
return
output = Run(['svn', 'update', '-r', rev])
if re.search("No such revision", output) != None:
raise Exception('unknown revision: ' + output)
return
def update(self, rev = None):
output = Run(['hg', 'pull', '--cwd', self.folder])
if not rev:
output = Run(['hg', 'update', '--cwd', self.folder])
return
else:
output = Run(['hg', 'update', '-r', rev, '--cwd', self.folder])
if re.search("unknown revision", output) != None:
raise Exception('unknown revision: ' + output)
return
def make(self):
try:
self.patch()
with utils.FolderChanger(os.path.join(self.folder, 'Tools', 'Scripts')):
args = ['/usr/bin/perl', 'build-jsc']
if self.config == '32bit':
args += ['--32-bit']
Run(args, self.env.get())
finally:
self.clean()
Run(["install_name_tool", "-change", "/System/Library/Frameworks/JavaScriptCore.framework/Versions/A/JavaScriptCore", self.objdir()+"/JavaScriptCore.framework/JavaScriptCore", self.objdir() + "/jsc"])
def build(self):
Run(['make', 'dependencies'])
if self.cpu == 'x64':
Run(['make', 'x64.release'])
elif self.cpu == 'arm':
if self.hardfp:
Run(['make', 'arm.release', 'hardfp=on'])
else:
Run(['make', 'arm.release'])
elif self.cpu == 'x86':
Run(['make', 'ia32.release'])
def build(self):
env = os.environ.copy()
# env["NO_AUTH_BOTO_CONFIG"] = "/repos/boto.cfg"
# sourcePath = os.path.join(self.repoPath, self.source)
# add build command code here
with utils.FolderChanger(self.sourcePath):
try:
in_argns_name = self.cpu + ".gn"
in_argns = os.path.join(self.dirname, 'gn_file', in_argns_name)
out_argns = os.path.join(self.repoPath, self.source, 'out',
self.cpu, 'args.gn')
if not os.path.isdir(
os.path.join(self.repoPath, self.source, 'out',
self.cpu)):
os.mkdir(
os.path.join(self.repoPath, self.source, 'out',
self.cpu))
Run(['copy', in_argns, out_argns], env)
Run([
'gn', 'gen',
os.path.join(self.sourcePath, 'out', self.cpu)
], env)
Run([
'ninja', '-C',
os.path.join(self.sourcePath, 'out', self.cpu), 'chrome',
'mini_installer', '-j40'
], env)
except subprocess.CalledProcessError as e:
print("Dirty build failed!")
try:
in_argns_name = self.cpu + ".gn"
in_argns = os.path.join(self.dirname, 'gn_file',
in_argns_name)
out_argns = os.path.join(self.repoPath, self.source, 'out',
self.cpu, 'args.gn')
Run([
'rmdir', '/s', '/q',
os.path.join(self.repoPath, self.source, 'out',
self.cpu)
])
Run([
'mkdir',
os.path.join(self.repoPath, self.source, 'out',
self.cpu)
])
Run(['copy', in_argns, out_argns], env)
Run([
'gn', 'gen',
os.path.join(self.sourcePath, 'out', self.cpu)
], env)
Run([
'ninja', '-C',
os.path.join(self.sourcePath, 'out', self.cpu),
'chrome', 'mini_installer', '-j40'
], env)
except subprocess.CalledProcessError as e:
print("Clean build failed!")
raise e
def reconf(self):
# Step 1. autoconf.
with utils.FolderChanger(os.path.join(self.folder, 'js', 'src')):
if platform.system() == "Darwin":
utils.Shell("autoconf213")
elif platform.system() == "Linux":
utils.Shell("autoconf2.13")
elif platform.system() == "Windows":
utils.Shell("autoconf-2.13")
# Step 2. configure
if os.path.exists(os.path.join(self.folder, 'js', 'src', 'Opt')):
shutil.rmtree(os.path.join(self.folder, 'js', 'src', 'Opt'))
os.mkdir(os.path.join(self.folder, 'js', 'src', 'Opt'))
with utils.FolderChanger(os.path.join(self.folder, 'js', 'src', 'Opt')):
args = ['--enable-optimize', '--disable-debug']
if platform.architecture()[0] == "64bit" and self.config == "32bit":
if platform.system() == "Darwin":
args.append("--target=i686-apple-darwin10.0.0")
elif platform.system() == "Linux":
args.append("--target=i686-pc-linux-gnu")
else:
assert False
Run(['../configure'] + args, self.env.get())
return True
def clean(self):
with utils.chdir(self.folder):
Run([
"svn", "revert",
"Tools/Scripts/check-for-weak-vtables-and-externals"
])
Run([
"svn", "revert",
"Source/JavaScriptCore/Configurations/Base.xcconfig"
])
Run([
"svn", "revert", "Source/bmalloc/Configurations/Base.xcconfig"
])
Run(["svn", "revert", "Source/WTF/Configurations/Base.xcconfig"])
Run(["svn", "revert", "Source/bmalloc/bmalloc/SmallLine.h"])
def _move_zip(self, rev=None, remote=True):
name = "chromium_"
name += now()
if rev and type(rev) == str:
name += '_rev_'
name += rev
name += '.7z'
print(name)
src = self.libpaths()
# if not os.path.isdir(self.out_path):
# os.mkdir(self.out_path)
# dest = os.path.join(self.out_path, name)
result = {'status': 1, 'msg': None}
# try:
# utils.move(src=src, dest=dest)
# except Exception as e:
# result['msg'] = e
# result['status'] = -5
if os.path.exists(src):
if remote:
try:
Run(['scp', src, self.remote_out_path + name])
print('scp to remote apache2 web succeed!')
result['msg'] = self.remote_url + name
except Exception as e:
result['status'] = -6
result['msg'] = e
else:
print("move to apache2 web succeed!")
result['msg'] = self.url + name
else:
result['status'] = -5
result['msg'] = "Cannot find moved 7zip file in web folder!"
return result
def identify(self):
with chdir(self.path()):
output = Run(['svn', 'info'])
m = re.search("Revision: ([0-9]+)", output)
if m == None:
raise Exception('unknown output from svn: ' + output)
return m.group(1)
def identify(self):
with chdir(self.path()):
output = Run(['git', 'log', '-1'])
m = re.match("commit ([0-9a-z]+)\s*", output)
if m == None:
raise Exception('unknown output from git: ' + output)
return m.group(1)
def build(self):
with utils.FolderChanger(os.path.join('Tools', 'Scripts')):
if self.cpu == 'x86':
args = ['/usr/bin/perl', 'build-jsc', '--32-bit']
else:
args = ['/usr/bin/perl', 'build-jsc']
args.extend(self.extra)
Run(args)
def run(cfg, train_loader, tr_comp, saver, trainer, valid_dict):
# TODO resume
# trainer = Engine(...)
# trainer.load_state_dict(state_dict)
# trainer.run(data)
# checkpoint
handler = ModelCheckpoint(saver.model_dir, 'train', n_saved=3, create_dir=True)
checkpoint_params = tr_comp.state_dict()
trainer.add_event_handler(Events.EPOCH_COMPLETED,
handler,
checkpoint_params)
timer = Timer(average=True)
timer.attach(trainer,
start=Events.EPOCH_STARTED,
resume=Events.ITERATION_STARTED,
pause=Events.ITERATION_COMPLETED,
step=Events.ITERATION_COMPLETED)
# average metric to attach on trainer
names = ["Acc", "Loss"]
names.extend(tr_comp.loss_function_map.keys())
for n in names:
RunningAverage(output_transform=Run(n)).attach(trainer, n)
@trainer.on(Events.EPOCH_COMPLETED)
def adjust_learning_rate(engine):
tr_comp.scheduler.step()
@trainer.on(Events.ITERATION_COMPLETED(every=cfg.TRAIN.LOG_ITER_PERIOD))
def log_training_loss(engine):
message = f"Epoch[{engine.state.epoch}], " + \
f"Iteration[{engine.state.iteration}/{len(train_loader)}], " + \
f"Base Lr: {tr_comp.scheduler.get_last_lr()[0]:.2e}, "
for loss_name in engine.state.metrics.keys():
message += f"{loss_name}: {engine.state.metrics[loss_name]:.4f}, "
if tr_comp.xent and tr_comp.xent.learning_weight:
message += f"xentWeight: {tr_comp.xent.uncertainty.mean().item():.4f}, "
logger.info(message)
# adding handlers using `trainer.on` decorator API
@trainer.on(Events.EPOCH_COMPLETED)
def print_times(engine):
logger.info('Epoch {} done. Time per batch: {:.3f}[s] Speed: {:.1f}[samples/s]'
.format(engine.state.epoch, timer.value() * timer.step_count,
train_loader.batch_size / timer.value()))
logger.info('-' * 80)
timer.reset()
@trainer.on(Events.EPOCH_COMPLETED(every=cfg.EVAL.EPOCH_PERIOD))
def log_validation_results(engine):
logger.info(f"Valid - Epoch: {engine.state.epoch}")
eval_multi_dataset(cfg, valid_dict, tr_comp)
trainer.run(train_loader, max_epochs=cfg.TRAIN.MAX_EPOCHS)
def make(self):
try:
self.patch()
with utils.FolderChanger(os.path.join(self.folder, 'Tools', 'Scripts')):
args = ['/usr/bin/perl', 'build-jsc']
if self.config == '32bit':
args += ['--32-bit']
Run(args, self.env.get())
finally:
self.clean()
def make(self):
args = ['make', '-j6', '-C', os.path.join(self.folder, 'v8')]
if self.config == '32bit':
args += ['ia32.release']
elif self.config == '64bit':
args += ['x64.release']
else:
assert True
Run(args, self.env.get())
def make(self):
args = ['make', '-j6']
if self.config == '32bit':
args += ['ia32.release']
elif self.config == '64bit':
args += ['x64.release']
else:
assert True
with utils.FolderChanger(os.path.join(self.folder, 'v8')):
Run(args, self.env.get())
def sameRepo(self):
with FolderChanger(self.path()):
try:
output = Run(['svn', 'info'])
except:
return False
print self.repo
print output
if "URL: " + self.repo in output:
return True
exit()
return False
def sameRepo(self):
if not os.path.exists(self.path()):
return False
with chdir(self.path()):
try:
output = Run(['svn', 'info'])
except:
return False
print self.repo
print output
if "URL: "+self.repo in output:
return True
exit()
return False