def n_return_if_stmt(self, node): if self.__params['isLambda']: self.preorder(node[0]) self.prune() else: self.write(self.indent, 'return') if self.return_none or node != AST('return_stmt', [AST('ret_expr', [NONE]), Token('RETURN_END_IF')]): self.write(' ') self.preorder(node[0]) self.print_() self.prune() # stop recursing
def n_delete_subscr(self, node): n = node[-2][0] if n == 'build_list' and n[-1].type.startswith('BUILD_TUPLE'): if n[-1] != 'BUILD_TUPLE_0': n.type = 'build_tuple2' elif n == 'LOAD_CONST' and type(n.pattr) == tuple and len(n.pattr) > 0: node[-2][0] = AST('build_tuple2', [Token('LOAD_CONST', None, x) for x in n.pattr] + [None]) self.default(node)
def n_yield(self, node): self.write('yield') if node != AST('yield', [NONE, Token('YIELD_VALUE')]): self.write(' ') self.preorder(node[0]) self.prune() # stop recursing
import sys, re, cStringIO from types import ListType, TupleType, DictType, \ EllipsisType, IntType, CodeType from spark import GenericASTTraversal import Parser from Parser import AST from Scanner import Token, Code minint = -sys.maxint - 1 # Some ASTs used for comparing code fragments (like 'return None' at # the end of functions). RETURN_LOCALS = AST( 'return_stmt', [AST('expr', [Token('LOAD_LOCALS')]), Token('RETURN_VALUE')]) NONE = AST('expr', [Token('LOAD_CONST', pattr=None)]) RETURN_NONE = AST('stmt', [AST('return_stmt', [NONE, Token('RETURN_VALUE')])]) PASS = AST('stmts', [AST('sstmt', [AST('stmt', [AST('passstmt', [])])])]) ASSIGN_DOC_STRING = lambda doc_string: \ AST('stmt', [ AST('assign', [ AST('expr', [ Token('LOAD_CONST', pattr=doc_string) ]), AST('designator', [ Token('STORE_NAME', pattr='__doc__')]) ])])
def extract_hyperparameters(self, filename, module): astObj = AST() hyperParams = astObj.ParseAst(filename, self.project_title) for i in range(0, len(hyperParams)): if 'epochs' in hyperParams[i]: self.experiment_metadata['epochs'] = hyperParams[i].split( '=')[1] if 'batch_size' in hyperParams[i]: self.experiment_metadata['batch_size'] = hyperParams[i].split( '=')[1] #Extract metadata from the model file. #Adding Callbacks and Checkpoint Code Dynamically flag = 1 temp = module with open(filename) as myfile: if 'CSVLogger' in myfile.read(): return temp_file_location = get_python_lib() temp_file_location = temp_file_location.replace('\\', '/') with open(temp_file_location + '/temp.py', 'w') as file: source_lines = inspect.getsourcelines(temp) for code in source_lines: if (isinstance(code, (list))): for line in range(0, len(code)): #Adding import functions and call backs if 'import' in code[line] and flag == 1: file.write( str('from keras.callbacks import CSVLogger, ModelCheckpoint\n' )) file.write(str(file.write(code[line])) + '\n') flag = 0 elif '.fit(' in code[line]: csv_file_path = temp_file_location + '/training.log' file.write("csv_logger = CSVLogger('" + csv_file_path + "')" + '\n') weights_file_path = temp_file_location + '/weights-improvement-{epoch:02d}-{acc:.2f}.hdf5' file.write( str('filepath="' + weights_file_path + '"') + '\n') file.write( str("checkpoint = ModelCheckpoint(filepath, monitor='acc', verbose=1, save_best_only=False, mode='max')" + '\n')) file.write( str("callbacks_list = checkpoint" + '\n')) code[line] = code[line].replace( ')', ',callbacks=[csv_logger, callbacks_list])') file.write(code[line]) else: file.write(code[line]) #Executing the temp file os.system(sys.executable + ' ' + temp_file_location + '/temp.py') model_file = glob.glob(temp_file_location + '/*.hdf5') epochs = int(self.experiment_metadata['epochs']) #Loading model and parameters model = load_model(model_file[epochs - 1]) self.experiment_metadata['input_shape'] = model.input_shape self.experiment_metadata['layers_count'] = len(model.layers) self.experiment_metadata['output_shape'] = model.output_shape self.experiment_metadata[ 'Optimizer'] = model.optimizer.__class__.__name__ self.experiment_metadata['LossFunction'] = model.loss #Model and Log file self.experiment_metadata[ 'callbacks_log'] = temp_file_location + '/training.log' self.experiment_metadata['model_file'] = model_file[epochs - 1] #Generate Predict Function # output = self.generate_predict(model) # with open(temp_file_location + '/auto_predict.py', 'w') as file: # file.write(output) self.experiment_metadata[ 'predict_function'] = temp_file_location + '/auto_predict.py' print(self.experiment_metadata) self.saveToLocalDB(self.experiment_metadata) #Python_file, Metadata_dict(), project name #self.saveToDB(filename, self.experiment_metadata, self.project_title) #Deleting all the files that are generated by ModelKB os.remove(temp_file_location + '/temp.py') os.remove(temp_file_location + '/training.log') os.remove(temp_file_location + '/auto_predict.py') for i in range(0, len(model_file)): os.remove(model_file[i]) sys.exit(0)