def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None:
# transform the enclosed annot_body_code
annotated_stmts = parser.getParser(self.line_no).parse(self.annot_body_code)
if len(annotated_stmts) == 1:
annotated_stmt = annotated_stmts[0]
else:
annotated_stmt = ast.CompStmt(annotated_stmts[0])
stmts[0].stmt = annotated_stmt
# apply transformations
t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen(self.language)
transformed_code = '\n'
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# return the transformed code
return transformed_code
def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None:
# transform the enclosed annot_body_code
annotated_stmts = parser.getParser(self.line_no).parse(
self.annot_body_code)
if len(annotated_stmts) == 1:
annotated_stmt = annotated_stmts[0]
else:
annotated_stmt = ast.CompStmt(annotated_stmts[0])
stmts[0].stmt = annotated_stmt
# apply transformations
t = transformation.Transformation(self.perf_params, self.verbose,
self.language, self.tinfo)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen(self.language)
transformed_code = '\n'
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# return the transformed code
return transformed_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code,
lang):
'''To transform the annotated code region'''
# extract leader annotation information
leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info
# parse the module body
p = parser.getParser(line_no)
stmts = p.parse(module_body)
# apply transformation procedure on each statement
transformed_stmts = transformator.transform(stmts)
# unparse
transformed_code = ''
extra_indent = ' '
include_orig_loop = False
if include_orig_loop:
transformed_code += '\n' + indent + '#if ORIG_LOOP' + '\n'
transformed_code += annot_body_code.replace(
'\n', '\n' + extra_indent)
transformed_code += '\n' + indent + '#else ' + '\n\n'
for s in transformed_stmts:
transformed_code += s.unparseToC(indent + extra_indent,
extra_indent)
transformed_code += '\n' + indent + '#endif ' + '\n' + indent
else:
for s in transformed_stmts:
transformed_code += s.unparseToC(indent, extra_indent)
return leader_annot_code + transformed_code + trailer_annot_code
def run(program):
"""Take a gotolang program (as a string) and runs it."""
# Generate the abstact syntax tree
ast = getParser().parse(program, lexer=getLexer())
# Cast to list
if not isinstance(ast, list):
ast = [ast]
# initialise interpriter
cmdrunner = getInterpriter()
statement_pos = 0
# Mainloop
while 0 <= statement_pos < len(ast):
# For error reporing
cmdrunner.st_num = statement_pos
# exex statement and find next one
next_statement_num = cmdrunner.visit(ast[int(statement_pos)])
# last statement num was not a goto
if next_statement_num is None:
statement_pos += 1
# if it was a goto, then goto there
else:
statement_pos = next_statement_num
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang):
'''To transform the annotated code region'''
# extract leader annotation information
leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info
# parse the module body
p = parser.getParser(line_no)
stmts = p.parse(module_body)
# apply transformation procedure on each statement
transformed_stmts = transformator.transform(stmts)
# unparse
transformed_code = ''
extra_indent = ' '
include_orig_loop = False
if include_orig_loop:
transformed_code += '\n' + indent + '#if ORIG_LOOP' + '\n'
transformed_code += annot_body_code.replace('\n', '\n' + extra_indent)
transformed_code += '\n' + indent + '#else ' + '\n\n'
for s in transformed_stmts:
transformed_code += s.unparseToC(indent + extra_indent, extra_indent)
transformed_code += '\n' + indent + '#endif ' + '\n' + indent
else:
for s in transformed_stmts:
transformed_code += s.unparseToC(indent, extra_indent)
return leader_annot_code + transformed_code + trailer_annot_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang):
'''To transform the annotated code region'''
# extract leader annotation information
leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info
# parse the module body
p = parser.getParser(line_no)
vars = p.parse(module_body)
# semantic check
for v in vars:
if lang == src.main.C_CPP:
v.rowMajorCheck()
elif lang == src.main.FORTRAN:
v.columnMajorCheck()
else:
print 'internal error: unknown source language'
sys.exit(1)
# unparse
if lang == src.main.C_CPP:
extra_indent = ' ' * 2
transformed_code = unparser.unparseToC(vars, annot_body_code,
indent, extra_indent)
elif lang == src.main.FORTRAN:
indent = ' ' * 6
extra_indent = ' ' * 2
transformed_code = unparser.unparseToFortran(vars, annot_body_code,
indent, extra_indent)
else:
print 'internal error: unknown source language'
sys.exit(1)
return leader_annot_code + transformed_code + trailer_annot_code
def get_dtd_entities(dtd_path):
if dtd_path in dtd_entities_cache:
return dtd_entities_cache[dtd_path]
dtd_parser = parser.getParser('.dtd')
dtd_parser.readFile(dtd_path)
dtd_entities_cache[dtd_path] = dtd_parser.parse()
return dtd_entities_cache[dtd_path]
def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None:
# transform the enclosed annot_body_code
annotated_stmts = parser.getParser(self.line_no).parse(
self.annot_body_code)
if len(annotated_stmts) == 1:
annotated_stmt = annotated_stmts[0]
else:
annotated_stmt = ast.CompStmt(annotated_stmts[0])
stmts[0].stmt = annotated_stmt
# apply transformations
t = transformation.Transformation(self.perf_params, self.verbose,
self.language, self.tinfo)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen(self.language)
transformed_code = '\n'
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# Example on applying another visitor, e.g., for analysis
#exampleVisitor = astvisitors.ExampleVisitor()
#exampleVisitor.visit(transformed_stmts)
# Count operations visitor
opsVisitor = astvisitors.CountingVisitor()
opsVisitor.visit(transformed_stmts)
debug(str(opsVisitor), level=3)
# CFG
if True:
try:
from orio.module.loop.cfg import CFGGraph
cfg = CFGGraph(transformed_stmts)
except Exception, e:
err('[module.loop.loop] cannot construct CFG: ', e)
def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None:
# transform the enclosed annot_body_code
annotated_stmts = parser.getParser(self.line_no).parse(self.annot_body_code)
if len(annotated_stmts) == 1:
annotated_stmt = annotated_stmts[0]
else:
annotated_stmt = ast.CompStmt(annotated_stmts[0])
stmts[0].stmt = annotated_stmt
# apply transformations
t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen(self.language)
transformed_code = '\n'
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# Example on applying another visitor, e.g., for analysis
#exampleVisitor = astvisitors.ExampleVisitor()
#exampleVisitor.visit(transformed_stmts)
# Count operations visitor
opsVisitor = astvisitors.CountingVisitor()
opsVisitor.visit(transformed_stmts)
info(str(opsVisitor))
# CFG
if True:
try:
from orio.module.loop.cfg import CFGGraph
cfg = CFGGraph(transformed_stmts)
except Exception, e:
err('[module.loop.loop] cannot construct CFG: ',e)
def transform(self):
'''To apply a memory-alignment transformation on the annotated code'''
# parse the annotation orio.module.code to get the variables to be checked
vars = parser.getParser(self.line_no).parse(self.module_body_code)
# perform a semantic check
for v in vars:
v.semantCheck()
# generate the alignment optimization code
indent = ' ' * self.indent_size
transformed_code = codegen.CodeGen(vars, self.annot_body_code, indent, self.language).generate()
# return the transformed code
return transformed_code
def transform(self):
'''To apply a memory-alignment transformation on the annotated code'''
# parse the annotation module code to get the variables to be checked
vars = parser.getParser(self.line_no).parse(self.module_body_code)
# perform a semantic check
for v in vars:
v.semantCheck()
# generate the alignment optimization code
indent = ' ' * self.indent_size
transformed_code = codegen.CodeGen(vars, self.annot_body_code, indent).generate()
# return the transformed code
return transformed_code
def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
# apply transformations
t = transformator.Transformator(self.perf_params, self.verbose)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen()
transformed_code = ''
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# return the transformed code
return transformed_code
def transform(self):
'''To apply loop transformations on the annotated code'''
# parse the code to get the AST
stmts = parser.getParser(self.line_no).parse(self.module_body_code)
# apply transformations
t = transformator.Transformator(self.perf_params, self.verbose)
transformed_stmts = t.transform(stmts)
# generate code for the transformed ASTs
indent = ' ' * self.indent_size
extra_indent = ' '
cgen = codegen.CodeGen()
transformed_code = ''
for s in transformed_stmts:
transformed_code += cgen.generate(s, indent, extra_indent)
# return the transformed code
return transformed_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code,
lang):
'''To transform the annotated code region'''
# extract leader annotation information
leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info
# parse the module body
p = parser.getParser(line_no)
vars = p.parse(module_body)
# semantic check
for v in vars:
if lang == src.main.C_CPP:
v.rowMajorCheck()
elif lang == src.main.FORTRAN:
v.columnMajorCheck()
else:
print 'internal error: unknown source language'
sys.exit(1)
# unparse
if lang == src.main.C_CPP:
extra_indent = ' ' * 2
transformed_code = unparser.unparseToC(vars, annot_body_code,
indent, extra_indent)
elif lang == src.main.FORTRAN:
indent = ' ' * 6
extra_indent = ' ' * 2
transformed_code = unparser.unparseToFortran(
vars, annot_body_code, indent, extra_indent)
else:
print 'internal error: unknown source language'
sys.exit(1)
return leader_annot_code + transformed_code + trailer_annot_code
def testSpaceParser(self):
self.assertEqual(type(parser.getParser('space')), parser.SpaceParser)
def testBaseParser(self):
self.assertEqual(type(parser.getParser('base')), parser.BaseParser)
def main():
global msglogger
# Parse arguments
prsr = parser.getParser()
distiller.knowledge_distillation.add_distillation_args(prsr, ALL_MODEL_NAMES, True)
args = prsr.parse_args()
if not os.path.exists(args.output_dir):
os.makedirs(args.output_dir)
msglogger = apputils.config_pylogger(os.path.join(script_dir, 'logging.conf'), args.name, args.output_dir)
# Log various details about the execution environment. It is sometimes useful
# to refer to past experiment executions and this information may be useful.
apputils.log_execution_env_state(sys.argv, gitroot=module_path)
msglogger.debug("Distiller: %s", distiller.__version__)
start_epoch = 0
best_epochs = [distiller.MutableNamedTuple({'epoch': 0, 'top1': 0, 'sparsity': 0})
for i in range(args.num_best_scores)]
if args.deterministic:
# Experiment reproducibility is sometimes important. Pete Warden expounded about this
# in his blog: https://petewarden.com/2018/03/19/the-machine-learning-reproducibility-crisis/
# In Pytorch, support for deterministic execution is still a bit clunky.
if args.workers > 1:
msglogger.error('ERROR: Setting --deterministic requires setting --workers/-j to 0 or 1')
exit(1)
# Use a well-known seed, for repeatability of experiments
torch.manual_seed(0)
random.seed(0)
np.random.seed(0)
cudnn.deterministic = True
else:
# This issue: https://github.com/pytorch/pytorch/issues/3659
# Implies that cudnn.benchmark should respect cudnn.deterministic, but empirically we see that
# results are not re-produced when benchmark is set. So enabling only if deterministic mode disabled.
cudnn.benchmark = True
if args.cpu or not torch.cuda.is_available():
# Set GPU index to -1 if using CPU
args.device = 'cpu'
args.gpus = -1
else:
args.device = 'cuda'
if args.gpus is not None:
try:
args.gpus = [int(s) for s in args.gpus.split(',')]
except ValueError:
msglogger.error('ERROR: Argument --gpus must be a comma-separated list of integers only')
exit(1)
available_gpus = torch.cuda.device_count()
for dev_id in args.gpus:
if dev_id >= available_gpus:
msglogger.error('ERROR: GPU device ID {0} requested, but only {1} devices available'
.format(dev_id, available_gpus))
exit(1)
# Set default device in case the first one on the list != 0
torch.cuda.set_device(args.gpus[0])
# Infer the dataset from the model name
args.dataset = 'cifar10' if 'cifar' in args.arch else 'imagenet'
args.num_classes = 10 if args.dataset == 'cifar10' else 1000
if args.earlyexit_thresholds:
args.num_exits = len(args.earlyexit_thresholds) + 1
args.loss_exits = [0] * args.num_exits
args.losses_exits = []
args.exiterrors = []
# Create the model
model = create_model(args.pretrained, args.dataset, args.arch,
parallel=not args.load_serialized, device_ids=args.gpus)
compression_scheduler = None
# Create a couple of logging backends. TensorBoardLogger writes log files in a format
# that can be read by Google's Tensor Board. PythonLogger writes to the Python logger.
tflogger = TensorBoardLogger(msglogger.logdir)
pylogger = PythonLogger(msglogger)
# capture thresholds for early-exit training
if args.earlyexit_thresholds:
msglogger.info('=> using early-exit threshold values of %s', args.earlyexit_thresholds)
# We can optionally resume from a checkpoint
if args.resume:
model, compression_scheduler, start_epoch = apputils.load_checkpoint(model, chkpt_file=args.resume)
model.to(args.device)
# Define loss function (criterion) and optimizer
criterion = nn.CrossEntropyLoss().to(args.device)
optimizer = torch.optim.SGD(model.parameters(), lr=args.lr,
momentum=args.momentum,
weight_decay=args.weight_decay)
msglogger.info('Optimizer Type: %s', type(optimizer))
msglogger.info('Optimizer Args: %s', optimizer.defaults)
if args.ADC:
return automated_deep_compression(model, criterion, optimizer, pylogger, args)
# This sample application can be invoked to produce various summary reports.
if args.summary:
return summarize_model(model, args.dataset, which_summary=args.summary)
# Load the datasets: the dataset to load is inferred from the model name passed
# in args.arch. The default dataset is ImageNet, but if args.arch contains the
# substring "_cifar", then cifar10 is used.
train_loader, val_loader, test_loader, _ = apputils.load_data(
args.dataset, os.path.expanduser(args.data), args.batch_size,
args.workers, args.validation_size, args.deterministic)
msglogger.info('Dataset sizes:\n\ttraining=%d\n\tvalidation=%d\n\ttest=%d',
len(train_loader.sampler), len(val_loader.sampler), len(test_loader.sampler))
activations_collectors = create_activation_stats_collectors(model, *args.activation_stats)
if args.sensitivity is not None:
sensitivities = np.arange(args.sensitivity_range[0], args.sensitivity_range[1], args.sensitivity_range[2])
return sensitivity_analysis(model, criterion, test_loader, pylogger, args, sensitivities)
if args.evaluate:
return evaluate_model(model, criterion, test_loader, pylogger, activations_collectors, args, compression_scheduler)
if args.compress:
# The main use-case for this sample application is CNN compression. Compression
# requires a compression schedule configuration file in YAML.
compression_scheduler = distiller.file_config(model, optimizer, args.compress, compression_scheduler)
# Model is re-transferred to GPU in case parameters were added (e.g. PACTQuantizer)
model.to(args.device)
elif compression_scheduler is None:
compression_scheduler = distiller.CompressionScheduler(model)
if args.thinnify:
#zeros_mask_dict = distiller.create_model_masks_dict(model)
assert args.resume is not None, "You must use --resume to provide a checkpoint file to thinnify"
distiller.remove_filters(model, compression_scheduler.zeros_mask_dict, args.arch, args.dataset, optimizer=None)
apputils.save_checkpoint(0, args.arch, model, optimizer=None, scheduler=compression_scheduler,
name="{}_thinned".format(args.resume.replace(".pth.tar", "")), dir=msglogger.logdir)
print("Note: your model may have collapsed to random inference, so you may want to fine-tune")
return
args.kd_policy = None
if args.kd_teacher:
teacher = create_model(args.kd_pretrained, args.dataset, args.kd_teacher, device_ids=args.gpus)
if args.kd_resume:
teacher, _, _ = apputils.load_checkpoint(teacher, chkpt_file=args.kd_resume)
dlw = distiller.DistillationLossWeights(args.kd_distill_wt, args.kd_student_wt, args.kd_teacher_wt)
args.kd_policy = distiller.KnowledgeDistillationPolicy(model, teacher, args.kd_temp, dlw)
compression_scheduler.add_policy(args.kd_policy, starting_epoch=args.kd_start_epoch, ending_epoch=args.epochs,
frequency=1)
msglogger.info('\nStudent-Teacher knowledge distillation enabled:')
msglogger.info('\tTeacher Model: %s', args.kd_teacher)
msglogger.info('\tTemperature: %s', args.kd_temp)
msglogger.info('\tLoss Weights (distillation | student | teacher): %s',
' | '.join(['{:.2f}'.format(val) for val in dlw]))
msglogger.info('\tStarting from Epoch: %s', args.kd_start_epoch)
for epoch in range(start_epoch, start_epoch + args.epochs):
# This is the main training loop.
msglogger.info('\n')
if compression_scheduler:
compression_scheduler.on_epoch_begin(epoch)
# Train for one epoch
with collectors_context(activations_collectors["train"]) as collectors:
train(train_loader, model, criterion, optimizer, epoch, compression_scheduler,
loggers=[tflogger, pylogger], args=args)
distiller.log_weights_sparsity(model, epoch, loggers=[tflogger, pylogger])
distiller.log_activation_statsitics(epoch, "train", loggers=[tflogger],
collector=collectors["sparsity"])
if args.masks_sparsity:
msglogger.info(distiller.masks_sparsity_tbl_summary(model, compression_scheduler))
# evaluate on validation set
with collectors_context(activations_collectors["valid"]) as collectors:
top1, top5, vloss = validate(val_loader, model, criterion, [pylogger], args, epoch)
distiller.log_activation_statsitics(epoch, "valid", loggers=[tflogger],
collector=collectors["sparsity"])
save_collectors_data(collectors, msglogger.logdir)
stats = ('Peformance/Validation/',
OrderedDict([('Loss', vloss),
('Top1', top1),
('Top5', top5)]))
distiller.log_training_progress(stats, None, epoch, steps_completed=0, total_steps=1, log_freq=1,
loggers=[tflogger])
if compression_scheduler:
compression_scheduler.on_epoch_end(epoch, optimizer)
# Update the list of top scores achieved so far, and save the checkpoint
is_best = top1 > best_epochs[-1].top1
if top1 > best_epochs[0].top1:
best_epochs[0].epoch = epoch
best_epochs[0].top1 = top1
# Keep best_epochs sorted such that best_epochs[0] is the lowest top1 in the best_epochs list
best_epochs = sorted(best_epochs, key=lambda score: score.top1)
for score in reversed(best_epochs):
if score.top1 > 0:
msglogger.info('==> Best Top1: %.3f on Epoch: %d', score.top1, score.epoch)
apputils.save_checkpoint(epoch, args.arch, model, optimizer, compression_scheduler,
best_epochs[-1].top1, is_best, args.name, msglogger.logdir)
# Finally run results on the test set
test(test_loader, model, criterion, [pylogger], activations_collectors, args=args)
def testBaseParser(self):
self.assertEqual(type(parser.getParser('base')), parser.BaseParser)
def _cyan(string):
return '\033[96m%s\033[m' % string
if __name__ == '__main__':
root_dir = os.getcwd()
locale_dir = os.path.join(root_dir, 'locale')
base_locale = os.path.join(locale_dir, 'en-US')
locales = [f for f in os.listdir(locale_dir) if f != 'en-US']
locales.sort()
files = [f for f in os.listdir(base_locale)]
files.sort()
for string_file in files:
print _blue(string_file)
parser = getParser(string_file)
parser.readFile(os.path.join(base_locale, string_file))
entities = [e for e in parser]
for locale in locales:
added = 0
same = 0
translated = 0
path = os.path.join(locale_dir, locale, string_file)
if os.path.exists(path):
parser.readFile(path)
locale_entities = {e.key: e for e in parser}
else:
locale_entities = {}
output = codecsopen(path, 'w', 'utf-8')
def testSpaceParser(self):
self.assertEqual(type(parser.getParser('space')), parser.SpaceParser)
def _cyan(string):
return '\033[96m%s\033[m' % string
if __name__ == '__main__':
root_dir = os.getcwd()
locale_dir = os.path.join(root_dir, 'locale')
base_locale = os.path.join(locale_dir, 'en-US')
locales = [f for f in os.listdir(locale_dir) if f != 'en-US']
locales.sort()
files = [f for f in os.listdir(base_locale)]
files.sort()
for string_file in files:
print _blue(string_file)
parser = getParser(string_file)
parser.readFile(os.path.join(base_locale, string_file))
entities = [e for e in parser]
for locale in locales:
added = 0
same = 0
translated = 0
path = os.path.join(locale_dir, locale, string_file)
if os.path.exists(path):
parser.readFile(path)
locale_entities = {e.key: e for e in parser}
else:
locale_entities = {}
output = codecsopen(path, 'w', 'utf-8')
