def setup(config):
workdir = os.path.abspath(os.getcwd())
configdir = os.path.join(os.path.dirname(__file__), 'config')
INSTRUMENTATION_PATH = os.path.abspath('llvm/lib/Transforms/Instrumentation')
if not os.path.exists('llvm'):
util.llvm.download(llvmver, os.getcwd())
# Apply patches common to both baseline and LowFat
bug81066_patch = os.path.join(os.path.dirname(__file__), 'bug81066.patch')
os.chdir('llvm/projects/compiler-rt')
util.patch(os.path.abspath(bug81066_patch), '-p0')
os.chdir(workdir)
if not os.path.exists('llvm-4.0.0') and not os.path.exists('llvm-lowfat'):
os.mkdir('llvm-4.0.0')
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-4.0.0'),
install_dir=workdir)
if config != configs[0]:
os.environ['CC'] = os.path.abspath('llvm-4.0.0/bin/clang')
os.environ['CXX'] = os.path.abspath('llvm-4.0.0/bin/clang++')
if not os.path.exists('llvm-lowfat'):
print 'patching llvm/clang/compiler-rt version ' + llvmver + '...'
llvm_patch = os.path.join(os.path.dirname(__file__), 'llvm.patch')
clang_patch = os.path.join(os.path.dirname(__file__), 'clang.patch')
compilerrt_patch = os.path.join(os.path.dirname(__file__), 'compiler-rt.patch')
legacy_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'legacy_compiler-rt.patch')
modern_compilerrt_patch = os.path.join(os.path.dirname(__file__), 'modern_compiler-rt.patch')
os.chdir('llvm')
util.patch(os.path.abspath(llvm_patch))
os.chdir(workdir)
os.chdir('llvm/tools/clang')
util.patch(os.path.abspath(clang_patch))
os.chdir(workdir)
os.chdir('llvm/projects/compiler-rt')
util.patch(os.path.abspath(compilerrt_patch))
cpuinfo = util.stdout(['cat', '/proc/cpuinfo']).strip().split('\n')
legacy = False
if not [s for s in cpuinfo if ' bmi1' in s]:
legacy = True
if not [s for s in cpuinfo if ' bmi2' in s]:
legacy = True
if not [s for s in cpuinfo if ' abm' in s]:
legacy = True
if legacy:
util.patch(os.path.abspath(legacy_compilerrt_patch))
else:
util.patch(os.path.abspath(modern_compilerrt_patch))
os.chdir(workdir)
print 'building the LowFat config builder...'
CONFIG = 'sizes.cfg 32'
os.chdir(os.path.join(os.path.dirname(__file__), 'config'))
util.make()
util.run('./lowfat-config ' + CONFIG)
util.make('lowfat-check-config')
util.run('./lowfat-check-config')
os.chdir(workdir)
print 'copying the LowFat config files...'
RUNTIME_PATH = os.path.abspath('llvm/projects/compiler-rt/lib/lowfat')
shutil.copy(os.path.join(configdir, 'lowfat_config.h'), RUNTIME_PATH)
shutil.copy(os.path.join(configdir, 'lowfat_config.c'), RUNTIME_PATH)
CLANGLIB_PATH = os.path.abspath('llvm/tools/clang/lib/Basic')
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.c'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.inc'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat_config.h'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat_config.h'),
os.path.join(CLANGLIB_PATH, 'lowfat_config.h'))
shutil.copy(os.path.join(RUNTIME_PATH, 'lowfat.h'),
os.path.join(INSTRUMENTATION_PATH, 'lowfat.h'))
os.mkdir('llvm-lowfat')
util.llvm.compile(src_dir=os.path.abspath('llvm'),
build_dir=os.path.abspath('llvm-lowfat'),
install_dir=workdir)
os.chdir('llvm-lowfat')
if not os.path.exists('lib/LowFat'):
if not os.path.exists('lib'):
os.mkdir('lib')
os.mkdir('lib/LowFat')
shutil.copy(os.path.join(configdir, 'lowfat.ld'),
os.path.join('lib/LowFat', 'lowfat.ld'))
if not os.path.exists('bin/lowfat-ptr-info'):
os.chdir(configdir)
util.make('lowfat-ptr-info')
shutil.copy('lowfat-ptr-info', os.path.join(workdir, 'llvm-lowfat', 'bin'))
os.chdir(workdir)
if not os.path.exists('LowFat.so'):
clangxx = os.path.join('llvm-4.0.0', 'bin', 'clang++')
llvm_config = os.path.join('llvm-4.0.0', 'bin', 'llvm-config')
cxxflags = util.stdout([llvm_config, '--cxxflags']).rstrip('\n')
includedir = util.stdout([llvm_config, '--includedir']).rstrip('\n')
cmd = ' '.join([clangxx, '-DLOWFAT_PLUGIN',
os.path.join(INSTRUMENTATION_PATH, 'LowFat.cpp'),
'-c -Wall -O2',
'-I' + INSTRUMENTATION_PATH,
'-o LowFat.o',
cxxflags, includedir])
util.run(cmd)
ldflags = util.stdout([llvm_config, '--ldflags']).rstrip('\n')
cmd = ' '.join([clangxx, '-shared -rdynamic -o LowFat.so LowFat.o', ldflags])
util.run(cmd)
os.chdir(workdir)
# Create spec config file
print 'creating spec config file...'
spec_cfg = '-'.join([command, config]) + '.cfg'
if os.path.exists(spec_cfg):
os.remove(spec_cfg)
if config == configs[0]:
path = os.path.join('llvm-4.0.0', 'bin')
cflags = []
else:
path = os.path.join('llvm-lowfat', 'bin')
cflags = ['-fsanitize=lowfat']
blacklist = os.path.join(os.path.dirname(__file__), 'blacklist.txt')
cflags.extend(['-mllvm', '-lowfat-no-check-blacklist=%s' % blacklist])
cflags.append('-DPATCH_PERLBENCH_OVERFLOW')
cflags.append('-DPATCH_H264REF_OVERFLOW')
if config == configs[2]:
cflags.append('-g')
cflags.append('-fno-inline-functions')
if config == configs[3]:
cflags.append('-mllvm -lowfat-no-abort')
cc = [os.path.abspath(os.path.join(path, 'clang'))]
cc.extend(cflags)
cxx = [os.path.abspath(os.path.join(path, 'clang++'))]
cxx.extend(cflags)
util.spec.create_config(dir=os.getcwd(), file=spec_cfg,
name='-'.join([command, config]),
cc=' '.join(cc), cxx=' '.join(cxx))
for ep in range(epoch):
mse = 0.0
mean_acc = 0.0
mb = np.random.permutation(ndata)
t1 = time.clock()
for i in range(0, ndata, mbsize):
x_batch = data[mb[i : i + mbsize]]
y_batch = tar[mb[i : i + mbsize]]
if gpu:
x_batch = cuda.to_gpu(x_batch)
y_batch = cuda.to_gpu(y_batch)
optimizer.zero_grads()
loss, acc = forward(x_batch, y_batch, model, actfs, dr)
loss.backward()
optimizer.update()
optimizer.weight_decay(re) # L2 reguralization.
mse += loss.data
mean_acc += acc.data
mse = cuda.to_cpu(mse)
mean_acc = cuda.to_cpu(mean_acc)
mse /= nmb
mean_acc /= nmb
t2 = time.clock()
util.stdout("%4d %s %s ( %f sec)\n" % (ep, str(mse), str(mean_acc), t2 - t1))
sys.stdout.flush()
if gpu:
model.to_cpu()
dataio.savenn(outf, model, actfs)
data = dataio.dataio(args['<file>'], args['--df'], visnum).astype(np.float32)
ndata = data.shape[0]
if not gpu:
trainer = rbm.train_cpu
xp = np
else:
trainer = rbm.train_gpu
xp = cupy
cuda.check_cuda_available()
cuda.get_device(0).use()
rbm.to_gpu()
rbm.init_grads()
np.random.seed(seed)
mbnum = ndata / mbsize
for i in range(epoch):
t1 = time.clock()
e = 0.0
mblst = np.random.permutation(ndata)
for mb in range(0, ndata, mbsize):
e += trainer(data[mblst[mb:mb+mbsize]], lr, mm, re)
e /= mbnum
t2 = time.clock()
util.stdout("%4d th-epoch mse= %9e (%f sec)\n" % (i, e, t2-t1))
sys.stdout.flush()
if gpu:
rbm.to_cpu()
dataio.saveRBM(outf, af, rbm.W.T.copy(), rbm.hbias, rbm.vbias)