def transform(self): '''To apply loop transformations on the annotated code''' # parse the code to get the AST stmts = parser.getParser(self.line_no).parse(self.module_body_code) if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None: # transform the enclosed annot_body_code annotated_stmts = parser.getParser(self.line_no).parse(self.annot_body_code) if len(annotated_stmts) == 1: annotated_stmt = annotated_stmts[0] else: annotated_stmt = ast.CompStmt(annotated_stmts[0]) stmts[0].stmt = annotated_stmt # apply transformations t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo) transformed_stmts = t.transform(stmts) # generate code for the transformed ASTs indent = ' ' * self.indent_size extra_indent = ' ' cgen = codegen.CodeGen(self.language) transformed_code = '\n' for s in transformed_stmts: transformed_code += cgen.generate(s, indent, extra_indent) # return the transformed code return transformed_code
def transform(self): '''To apply loop transformations on the annotated code''' # parse the code to get the AST stmts = parser.getParser(self.line_no).parse(self.module_body_code) if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None: # transform the enclosed annot_body_code annotated_stmts = parser.getParser(self.line_no).parse( self.annot_body_code) if len(annotated_stmts) == 1: annotated_stmt = annotated_stmts[0] else: annotated_stmt = ast.CompStmt(annotated_stmts[0]) stmts[0].stmt = annotated_stmt # apply transformations t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo) transformed_stmts = t.transform(stmts) # generate code for the transformed ASTs indent = ' ' * self.indent_size extra_indent = ' ' cgen = codegen.CodeGen(self.language) transformed_code = '\n' for s in transformed_stmts: transformed_code += cgen.generate(s, indent, extra_indent) # return the transformed code return transformed_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang): '''To transform the annotated code region''' # extract leader annotation information leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info # parse the module body p = parser.getParser(line_no) stmts = p.parse(module_body) # apply transformation procedure on each statement transformed_stmts = transformator.transform(stmts) # unparse transformed_code = '' extra_indent = ' ' include_orig_loop = False if include_orig_loop: transformed_code += '\n' + indent + '#if ORIG_LOOP' + '\n' transformed_code += annot_body_code.replace( '\n', '\n' + extra_indent) transformed_code += '\n' + indent + '#else ' + '\n\n' for s in transformed_stmts: transformed_code += s.unparseToC(indent + extra_indent, extra_indent) transformed_code += '\n' + indent + '#endif ' + '\n' + indent else: for s in transformed_stmts: transformed_code += s.unparseToC(indent, extra_indent) return leader_annot_code + transformed_code + trailer_annot_code
def run(program): """Take a gotolang program (as a string) and runs it.""" # Generate the abstact syntax tree ast = getParser().parse(program, lexer=getLexer()) # Cast to list if not isinstance(ast, list): ast = [ast] # initialise interpriter cmdrunner = getInterpriter() statement_pos = 0 # Mainloop while 0 <= statement_pos < len(ast): # For error reporing cmdrunner.st_num = statement_pos # exex statement and find next one next_statement_num = cmdrunner.visit(ast[int(statement_pos)]) # last statement num was not a goto if next_statement_num is None: statement_pos += 1 # if it was a goto, then goto there else: statement_pos = next_statement_num
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang): '''To transform the annotated code region''' # extract leader annotation information leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info # parse the module body p = parser.getParser(line_no) stmts = p.parse(module_body) # apply transformation procedure on each statement transformed_stmts = transformator.transform(stmts) # unparse transformed_code = '' extra_indent = ' ' include_orig_loop = False if include_orig_loop: transformed_code += '\n' + indent + '#if ORIG_LOOP' + '\n' transformed_code += annot_body_code.replace('\n', '\n' + extra_indent) transformed_code += '\n' + indent + '#else ' + '\n\n' for s in transformed_stmts: transformed_code += s.unparseToC(indent + extra_indent, extra_indent) transformed_code += '\n' + indent + '#endif ' + '\n' + indent else: for s in transformed_stmts: transformed_code += s.unparseToC(indent, extra_indent) return leader_annot_code + transformed_code + trailer_annot_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang): '''To transform the annotated code region''' # extract leader annotation information leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info # parse the module body p = parser.getParser(line_no) vars = p.parse(module_body) # semantic check for v in vars: if lang == src.main.C_CPP: v.rowMajorCheck() elif lang == src.main.FORTRAN: v.columnMajorCheck() else: print 'internal error: unknown source language' sys.exit(1) # unparse if lang == src.main.C_CPP: extra_indent = ' ' * 2 transformed_code = unparser.unparseToC(vars, annot_body_code, indent, extra_indent) elif lang == src.main.FORTRAN: indent = ' ' * 6 extra_indent = ' ' * 2 transformed_code = unparser.unparseToFortran(vars, annot_body_code, indent, extra_indent) else: print 'internal error: unknown source language' sys.exit(1) return leader_annot_code + transformed_code + trailer_annot_code
def get_dtd_entities(dtd_path): if dtd_path in dtd_entities_cache: return dtd_entities_cache[dtd_path] dtd_parser = parser.getParser('.dtd') dtd_parser.readFile(dtd_path) dtd_entities_cache[dtd_path] = dtd_parser.parse() return dtd_entities_cache[dtd_path]
def transform(self): '''To apply loop transformations on the annotated code''' # parse the code to get the AST stmts = parser.getParser(self.line_no).parse(self.module_body_code) if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None: # transform the enclosed annot_body_code annotated_stmts = parser.getParser(self.line_no).parse( self.annot_body_code) if len(annotated_stmts) == 1: annotated_stmt = annotated_stmts[0] else: annotated_stmt = ast.CompStmt(annotated_stmts[0]) stmts[0].stmt = annotated_stmt # apply transformations t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo) transformed_stmts = t.transform(stmts) # generate code for the transformed ASTs indent = ' ' * self.indent_size extra_indent = ' ' cgen = codegen.CodeGen(self.language) transformed_code = '\n' for s in transformed_stmts: transformed_code += cgen.generate(s, indent, extra_indent) # Example on applying another visitor, e.g., for analysis #exampleVisitor = astvisitors.ExampleVisitor() #exampleVisitor.visit(transformed_stmts) # Count operations visitor opsVisitor = astvisitors.CountingVisitor() opsVisitor.visit(transformed_stmts) debug(str(opsVisitor), level=3) # CFG if True: try: from orio.module.loop.cfg import CFGGraph cfg = CFGGraph(transformed_stmts) except Exception, e: err('[module.loop.loop] cannot construct CFG: ', e)
def transform(self): '''To apply loop transformations on the annotated code''' # parse the code to get the AST stmts = parser.getParser(self.line_no).parse(self.module_body_code) if isinstance(stmts[0], ast.TransformStmt) and stmts[0].stmt is None: # transform the enclosed annot_body_code annotated_stmts = parser.getParser(self.line_no).parse(self.annot_body_code) if len(annotated_stmts) == 1: annotated_stmt = annotated_stmts[0] else: annotated_stmt = ast.CompStmt(annotated_stmts[0]) stmts[0].stmt = annotated_stmt # apply transformations t = transformation.Transformation(self.perf_params, self.verbose, self.language, self.tinfo) transformed_stmts = t.transform(stmts) # generate code for the transformed ASTs indent = ' ' * self.indent_size extra_indent = ' ' cgen = codegen.CodeGen(self.language) transformed_code = '\n' for s in transformed_stmts: transformed_code += cgen.generate(s, indent, extra_indent) # Example on applying another visitor, e.g., for analysis #exampleVisitor = astvisitors.ExampleVisitor() #exampleVisitor.visit(transformed_stmts) # Count operations visitor opsVisitor = astvisitors.CountingVisitor() opsVisitor.visit(transformed_stmts) info(str(opsVisitor)) # CFG if True: try: from orio.module.loop.cfg import CFGGraph cfg = CFGGraph(transformed_stmts) except Exception, e: err('[module.loop.loop] cannot construct CFG: ',e)
def transform(self): '''To apply a memory-alignment transformation on the annotated code''' # parse the annotation orio.module.code to get the variables to be checked vars = parser.getParser(self.line_no).parse(self.module_body_code) # perform a semantic check for v in vars: v.semantCheck() # generate the alignment optimization code indent = ' ' * self.indent_size transformed_code = codegen.CodeGen(vars, self.annot_body_code, indent, self.language).generate() # return the transformed code return transformed_code
def transform(self): '''To apply a memory-alignment transformation on the annotated code''' # parse the annotation module code to get the variables to be checked vars = parser.getParser(self.line_no).parse(self.module_body_code) # perform a semantic check for v in vars: v.semantCheck() # generate the alignment optimization code indent = ' ' * self.indent_size transformed_code = codegen.CodeGen(vars, self.annot_body_code, indent).generate() # return the transformed code return transformed_code
def transform(self): '''To apply loop transformations on the annotated code''' # parse the code to get the AST stmts = parser.getParser(self.line_no).parse(self.module_body_code) # apply transformations t = transformator.Transformator(self.perf_params, self.verbose) transformed_stmts = t.transform(stmts) # generate code for the transformed ASTs indent = ' ' * self.indent_size extra_indent = ' ' cgen = codegen.CodeGen() transformed_code = '' for s in transformed_stmts: transformed_code += cgen.generate(s, indent, extra_indent) # return the transformed code return transformed_code
def transform(self, leader_annot_info, annot_body_code, trailer_annot_code, lang): '''To transform the annotated code region''' # extract leader annotation information leader_annot_code, indent, line_no, module_name, module_body = leader_annot_info # parse the module body p = parser.getParser(line_no) vars = p.parse(module_body) # semantic check for v in vars: if lang == src.main.C_CPP: v.rowMajorCheck() elif lang == src.main.FORTRAN: v.columnMajorCheck() else: print 'internal error: unknown source language' sys.exit(1) # unparse if lang == src.main.C_CPP: extra_indent = ' ' * 2 transformed_code = unparser.unparseToC(vars, annot_body_code, indent, extra_indent) elif lang == src.main.FORTRAN: indent = ' ' * 6 extra_indent = ' ' * 2 transformed_code = unparser.unparseToFortran( vars, annot_body_code, indent, extra_indent) else: print 'internal error: unknown source language' sys.exit(1) return leader_annot_code + transformed_code + trailer_annot_code
def testSpaceParser(self): self.assertEqual(type(parser.getParser('space')), parser.SpaceParser)
def testBaseParser(self): self.assertEqual(type(parser.getParser('base')), parser.BaseParser)
def main(): global msglogger # Parse arguments prsr = parser.getParser() distiller.knowledge_distillation.add_distillation_args(prsr, ALL_MODEL_NAMES, True) args = prsr.parse_args() if not os.path.exists(args.output_dir): os.makedirs(args.output_dir) msglogger = apputils.config_pylogger(os.path.join(script_dir, 'logging.conf'), args.name, args.output_dir) # Log various details about the execution environment. It is sometimes useful # to refer to past experiment executions and this information may be useful. apputils.log_execution_env_state(sys.argv, gitroot=module_path) msglogger.debug("Distiller: %s", distiller.__version__) start_epoch = 0 best_epochs = [distiller.MutableNamedTuple({'epoch': 0, 'top1': 0, 'sparsity': 0}) for i in range(args.num_best_scores)] if args.deterministic: # Experiment reproducibility is sometimes important. Pete Warden expounded about this # in his blog: https://petewarden.com/2018/03/19/the-machine-learning-reproducibility-crisis/ # In Pytorch, support for deterministic execution is still a bit clunky. if args.workers > 1: msglogger.error('ERROR: Setting --deterministic requires setting --workers/-j to 0 or 1') exit(1) # Use a well-known seed, for repeatability of experiments torch.manual_seed(0) random.seed(0) np.random.seed(0) cudnn.deterministic = True else: # This issue: https://github.com/pytorch/pytorch/issues/3659 # Implies that cudnn.benchmark should respect cudnn.deterministic, but empirically we see that # results are not re-produced when benchmark is set. So enabling only if deterministic mode disabled. cudnn.benchmark = True if args.cpu or not torch.cuda.is_available(): # Set GPU index to -1 if using CPU args.device = 'cpu' args.gpus = -1 else: args.device = 'cuda' if args.gpus is not None: try: args.gpus = [int(s) for s in args.gpus.split(',')] except ValueError: msglogger.error('ERROR: Argument --gpus must be a comma-separated list of integers only') exit(1) available_gpus = torch.cuda.device_count() for dev_id in args.gpus: if dev_id >= available_gpus: msglogger.error('ERROR: GPU device ID {0} requested, but only {1} devices available' .format(dev_id, available_gpus)) exit(1) # Set default device in case the first one on the list != 0 torch.cuda.set_device(args.gpus[0]) # Infer the dataset from the model name args.dataset = 'cifar10' if 'cifar' in args.arch else 'imagenet' args.num_classes = 10 if args.dataset == 'cifar10' else 1000 if args.earlyexit_thresholds: args.num_exits = len(args.earlyexit_thresholds) + 1 args.loss_exits = [0] * args.num_exits args.losses_exits = [] args.exiterrors = [] # Create the model model = create_model(args.pretrained, args.dataset, args.arch, parallel=not args.load_serialized, device_ids=args.gpus) compression_scheduler = None # Create a couple of logging backends. TensorBoardLogger writes log files in a format # that can be read by Google's Tensor Board. PythonLogger writes to the Python logger. tflogger = TensorBoardLogger(msglogger.logdir) pylogger = PythonLogger(msglogger) # capture thresholds for early-exit training if args.earlyexit_thresholds: msglogger.info('=> using early-exit threshold values of %s', args.earlyexit_thresholds) # We can optionally resume from a checkpoint if args.resume: model, compression_scheduler, start_epoch = apputils.load_checkpoint(model, chkpt_file=args.resume) model.to(args.device) # Define loss function (criterion) and optimizer criterion = nn.CrossEntropyLoss().to(args.device) optimizer = torch.optim.SGD(model.parameters(), lr=args.lr, momentum=args.momentum, weight_decay=args.weight_decay) msglogger.info('Optimizer Type: %s', type(optimizer)) msglogger.info('Optimizer Args: %s', optimizer.defaults) if args.ADC: return automated_deep_compression(model, criterion, optimizer, pylogger, args) # This sample application can be invoked to produce various summary reports. if args.summary: return summarize_model(model, args.dataset, which_summary=args.summary) # Load the datasets: the dataset to load is inferred from the model name passed # in args.arch. The default dataset is ImageNet, but if args.arch contains the # substring "_cifar", then cifar10 is used. train_loader, val_loader, test_loader, _ = apputils.load_data( args.dataset, os.path.expanduser(args.data), args.batch_size, args.workers, args.validation_size, args.deterministic) msglogger.info('Dataset sizes:\n\ttraining=%d\n\tvalidation=%d\n\ttest=%d', len(train_loader.sampler), len(val_loader.sampler), len(test_loader.sampler)) activations_collectors = create_activation_stats_collectors(model, *args.activation_stats) if args.sensitivity is not None: sensitivities = np.arange(args.sensitivity_range[0], args.sensitivity_range[1], args.sensitivity_range[2]) return sensitivity_analysis(model, criterion, test_loader, pylogger, args, sensitivities) if args.evaluate: return evaluate_model(model, criterion, test_loader, pylogger, activations_collectors, args, compression_scheduler) if args.compress: # The main use-case for this sample application is CNN compression. Compression # requires a compression schedule configuration file in YAML. compression_scheduler = distiller.file_config(model, optimizer, args.compress, compression_scheduler) # Model is re-transferred to GPU in case parameters were added (e.g. PACTQuantizer) model.to(args.device) elif compression_scheduler is None: compression_scheduler = distiller.CompressionScheduler(model) if args.thinnify: #zeros_mask_dict = distiller.create_model_masks_dict(model) assert args.resume is not None, "You must use --resume to provide a checkpoint file to thinnify" distiller.remove_filters(model, compression_scheduler.zeros_mask_dict, args.arch, args.dataset, optimizer=None) apputils.save_checkpoint(0, args.arch, model, optimizer=None, scheduler=compression_scheduler, name="{}_thinned".format(args.resume.replace(".pth.tar", "")), dir=msglogger.logdir) print("Note: your model may have collapsed to random inference, so you may want to fine-tune") return args.kd_policy = None if args.kd_teacher: teacher = create_model(args.kd_pretrained, args.dataset, args.kd_teacher, device_ids=args.gpus) if args.kd_resume: teacher, _, _ = apputils.load_checkpoint(teacher, chkpt_file=args.kd_resume) dlw = distiller.DistillationLossWeights(args.kd_distill_wt, args.kd_student_wt, args.kd_teacher_wt) args.kd_policy = distiller.KnowledgeDistillationPolicy(model, teacher, args.kd_temp, dlw) compression_scheduler.add_policy(args.kd_policy, starting_epoch=args.kd_start_epoch, ending_epoch=args.epochs, frequency=1) msglogger.info('\nStudent-Teacher knowledge distillation enabled:') msglogger.info('\tTeacher Model: %s', args.kd_teacher) msglogger.info('\tTemperature: %s', args.kd_temp) msglogger.info('\tLoss Weights (distillation | student | teacher): %s', ' | '.join(['{:.2f}'.format(val) for val in dlw])) msglogger.info('\tStarting from Epoch: %s', args.kd_start_epoch) for epoch in range(start_epoch, start_epoch + args.epochs): # This is the main training loop. msglogger.info('\n') if compression_scheduler: compression_scheduler.on_epoch_begin(epoch) # Train for one epoch with collectors_context(activations_collectors["train"]) as collectors: train(train_loader, model, criterion, optimizer, epoch, compression_scheduler, loggers=[tflogger, pylogger], args=args) distiller.log_weights_sparsity(model, epoch, loggers=[tflogger, pylogger]) distiller.log_activation_statsitics(epoch, "train", loggers=[tflogger], collector=collectors["sparsity"]) if args.masks_sparsity: msglogger.info(distiller.masks_sparsity_tbl_summary(model, compression_scheduler)) # evaluate on validation set with collectors_context(activations_collectors["valid"]) as collectors: top1, top5, vloss = validate(val_loader, model, criterion, [pylogger], args, epoch) distiller.log_activation_statsitics(epoch, "valid", loggers=[tflogger], collector=collectors["sparsity"]) save_collectors_data(collectors, msglogger.logdir) stats = ('Peformance/Validation/', OrderedDict([('Loss', vloss), ('Top1', top1), ('Top5', top5)])) distiller.log_training_progress(stats, None, epoch, steps_completed=0, total_steps=1, log_freq=1, loggers=[tflogger]) if compression_scheduler: compression_scheduler.on_epoch_end(epoch, optimizer) # Update the list of top scores achieved so far, and save the checkpoint is_best = top1 > best_epochs[-1].top1 if top1 > best_epochs[0].top1: best_epochs[0].epoch = epoch best_epochs[0].top1 = top1 # Keep best_epochs sorted such that best_epochs[0] is the lowest top1 in the best_epochs list best_epochs = sorted(best_epochs, key=lambda score: score.top1) for score in reversed(best_epochs): if score.top1 > 0: msglogger.info('==> Best Top1: %.3f on Epoch: %d', score.top1, score.epoch) apputils.save_checkpoint(epoch, args.arch, model, optimizer, compression_scheduler, best_epochs[-1].top1, is_best, args.name, msglogger.logdir) # Finally run results on the test set test(test_loader, model, criterion, [pylogger], activations_collectors, args=args)
def _cyan(string): return '\033[96m%s\033[m' % string if __name__ == '__main__': root_dir = os.getcwd() locale_dir = os.path.join(root_dir, 'locale') base_locale = os.path.join(locale_dir, 'en-US') locales = [f for f in os.listdir(locale_dir) if f != 'en-US'] locales.sort() files = [f for f in os.listdir(base_locale)] files.sort() for string_file in files: print _blue(string_file) parser = getParser(string_file) parser.readFile(os.path.join(base_locale, string_file)) entities = [e for e in parser] for locale in locales: added = 0 same = 0 translated = 0 path = os.path.join(locale_dir, locale, string_file) if os.path.exists(path): parser.readFile(path) locale_entities = {e.key: e for e in parser} else: locale_entities = {} output = codecsopen(path, 'w', 'utf-8')