def translate_vensim(mdl_file): """ Translate a vensim model file into a python class. Supported functionality:\n\n""" # Todo: Test translation independent of load # We can make smaller unit tests that way... # Step 1 parser gets the subscript dictionary with open(mdl_file, 'rU') as file: text = file.read() # Todo: consider making this easier to understand, maybe with a second parsimonious grammar f2 = re.findall( r'([a-zA-Z][^:~\n.\[\]\+\-\!/\(\)\\\&\=]+:+[^:~.\[\]\+\-\!/\(\),\\\&\=]+,+[^:~.\[\]\+\-\!/\(\)\\\&\=]+~)+', text) for i in range(len(f2)): f2[i] = re.sub(r'[\n\t~]', '', f2[i]) dictofsubs = {} for i in f2: Family = builder.make_python_identifier(i.split(":")[0]) Elements = i.split(":")[1].split(",") for i in range(len(Elements)): Elements[i] = builder.make_python_identifier(Elements[i].strip()) dictofsubs[Family] = dict(zip(Elements, range(len(Elements)))) for i in dictofsubs: for j in dictofsubs: try: if set(dictofsubs[j].keys()).issubset( dictofsubs[i].keys()) and j != i: tempj = [] for key in (dictofsubs[i]): if key in dictofsubs[j].keys(): tempj.append(dictofsubs[i][key]) tempj.append(i) dictofsubs[j] = sorted(tempj) except: pass # Step 2 parser writes the file outfile_name = mdl_file[:-4] + '.py' parser = TextParser(file_grammar, outfile_name, text, dictofsubs) parser.builder.write() return parser.filename
def visit_Identifier(self, n, vc): string = n.text return builder.make_python_identifier(string)
def visit_Identifier(self, n, vc): string = n.text return builder.make_python_identifier(string)
def translate_jitia(mdl_file): """ Translate a jitia model file into a python class. Supported functionality:\n\n""" # model=mdl_file.replace('.dyn','.py') # builder.Builder(model,dictofsubs) f1 = open(mdl_file, 'r') f1 = ' '.join(f1) f1 = re.sub(r'(MACRO(.|\n)*MEND)*', '', f1) f2 = re.findall(r'([A-Z]+ [^\n]*(\n [^\n]*)*)', f1) for i, j in enumerate(f2): f2[i] = j[0] f3 = [] for i in f2: f3.append(i.split('\n ')) exceptions = ('FOR ', 'N TIME=', 'SPEC', 'SAVE') symbols = r'(\,|\^|\(|\)|\[|\]|\=|\<|\>|(?<![0-9]e)\+(?![0-9])|(?<![0-9]e)\-(?![0-9])|(?<=[0-9])\+(?=[0-9])|(?<=[0-9])\-(?=[0-9])|\*|\/)' numbers = r'(?i)((\+|\-)*(?<!([a-zA-Z]|[0-9]|\_))([0-9]*\.?[0-9]+(e(\+|\-)?[0-9]*)*)(?!([a-zA-Z]|[0-9]|\_)))' functions = [ 'MAX', 'MIN', 'POS', 'IF', 'THEN', 'ELSE', 'AND', 'OR', 'SUM', 'SUMV', 'SAMPLE', 'TABLE', 'STEP', 'WEIGHT', 'TUNER', 'TUN', 'FIFZE', 'FIFGE', 'DELAY3M', 'DATAOFF', 'SCLPRD', 'FAVAIL', 'PRDV', 'TUNE1', 'DATAOO', 'SMOOTH', 'DELAY', 'SUPPOS', 'AVVAL', 'SUPPOS', 'TABHL' ] functiondic = { 'MISSING()': '0', 'MAX': 'np.maximum', 'MIN': 'np.minimum', 'SUM': 'np.sum', 'if_then_else': 'np.where', 'SUMV': 'functions.sumv', 'ABS': 'abs', 'INTEGER': 'int', 'EXP': 'np.exp', 'PI': 'np.pi', 'SIN': 'np.sin', 'COS': 'np.cos', 'SQRT': 'np.sqrt', 'TAN': 'np.tan', 'LOGNORMAL': 'np.random.lognormal', 'RANDOM NORMAL': 'functions.bounded_normal', 'POISSON': 'np.random.poisson', 'LN': 'np.log', 'EXPRND': 'np.random.exponential', 'RANDOM UNIFORM': 'np.random.rand', 'ARCCOS': 'np.arccos', 'ARCSIN': 'np.arcsin', 'ARCTAN': 'np.arctan', 'STEP': 'functions.step', 'MODULO': 'np.mod', 'PULSE': 'functions.pulse', 'PULSE TRAIN': 'functions.pulse_train', 'RAMP': 'functions.ramp', 'FIFZE': 'functions.fifze', 'FIFGE': 'functions.fifge', 'SCLPRD': 'functions.sclprd', 'PRDV': 'functions.prdv', 'SMOOTH3I': '_smooth3i_', 'SMOOTH3': '_smooth3_', 'SMOOTHI': '_smoothi_', 'SMOOTH': '_smooth_', 'LOGN': 'np.log', 'ROUND': 'np.round', 'DELAY3I': '_delay3i_', 'DELAY3': '_delay3_', 'DELAY1I': '_delayi_', 'DELAY1': '_delay_', 'DELAYI': '_delayi_', 'DELAY': '_delay_', 'FLOOR': 'np.floor', 'VECTORMIN': 'np.min', 'VECTORMAX': 'np.max', 'PROD': 'np.prod', 'TABHL': 'functions.tabhl' } functiondiccase = '(' + '|'.join(functiondic) + ')' def unique(seq): Set = set(seq) return list(Set) checking = [] subscripts = [] for variable in f3: if variable[0].startswith('FOR '): checking.append(''.join(variable).split(' ', 1)[-1]) subscripts.append(''.join(variable).split(' ', 1)[-1]) elif variable[0].startswith('SAVE '): checking.append(''.join(variable).split(' ', 1)[-1]) else: checking.append(','.join( re.findall(r'((?<=[A-Z] )[^\=]*(?=\=))', variable[0]))) checking.append(','.join(functions)) checking.append(','.join(functiondic.keys())) checking = ' '.join(checking) checking = re.sub(r'(\,|\=|\^|\(|\))', ' ', checking) checking = checking.split(' ') checking = filter(None, checking) checking = unique(checking) checking = '|'.join(checking) checking = '(?i)^(' + re.sub(r'(%s|%s)' % (symbols, numbers), '', checking).replace('||', '|') + ')$' subscripts = ','.join(subscripts) subscripts = re.sub('\^', '', subscripts) subscripts = re.sub('=', ',', subscripts) subscripts = subscripts.split(',') subscripts.append('\*') subscripts = filter(None, subscripts) # for i,j in enumerate(subscripts): # if re.search(r'\d+\-\d+',j): # subscripts[i]='' # j=j.split('-') # for k in range(int(j[0]),int(j[1])): # subscripts[i]+=str(k)+'|' # subscripts[i].strip('|') subs = r'(?i)(' + '|'.join(subscripts) + ')' # subs = re.sub('(\d+)-(\d+)',lambda s:'|'.join(['%s'%x for x in range(int(s.group(1)),int(s.group(2))+1)]),subs) for position, variable in enumerate(f3): for j in range(len(variable)): if not variable[-1].endswith(' ') and not variable[0].startswith( exceptions): variable.pop(-1) else: break for counter, value in enumerate(variable): value = re.sub(symbols, ' ', value) value = re.sub(numbers, ' ', value) value = re.sub('\s+', ' ', value) value = value.split(' ') if counter != 0: if len(variable[counter] ) < 40 and variable[counter][-1] != '^': variable[counter + 1:] = '' if not re.search( '(\d|\=|\>|\<|\+|\-|\*|\/|\(|\)|\[|\]|\:|\,)', variable[counter]) and len(variable[counter]) > 55: variable[counter:] = '' for word in value: if not re.search(checking, word) and word != '' and not re.search( r'(?i)(else|then)', word): variable[counter:] = '' break for j in range(len(variable)): if not variable[-1].endswith(' ') and not variable[0].startswith( exceptions) and variable[0] != variable[-1]: variable.pop(-1) else: break f3[position] = ''.join(variable).strip().replace('^', ' ').replace( '\n', ' ') f3[position] = re.sub(r'^(.*?)\((.*?)\)(\=)', r'\1[\2]\3', f3[position]) f3[position] = re.sub('\((?=%s\s*(\,|\)))' % subs, '[', f3[position]) f3[position] = re.sub(r'(\[%s\s*(\,\s*%s)*\s*)\)' % (subs, subs), r'\1]', f3[position]) # if re.match('(?i)\w *c school duration',f3[position]): # print f3[position] f3[position] = re.sub(r'(?i)(%s)' % functiondiccase, lambda s: s.group(1).upper(), f3[position]) # if f3[position].startswith('N LKNPE'): # print f3[position] auxiliaries = [] levels = [] tables = [] initials = [] subscripts = [] for variable in f3: if variable.startswith('FOR '): subscripts.append(variable.split(' ', 1)[-1]) elif variable.startswith(('A ', 'C ', 'R ')): auxiliaries.append(variable.split(' ', 1)[-1]) elif variable.startswith('SPEC '): auxiliaries.append(variable.split(' ', 1)[-1]) elif variable.startswith('L '): levels.append(variable.split(' ', 1)[-1]) elif variable.startswith('T '): tables.append(variable.split(' ', 1)[-1]) elif variable.startswith('N '): if variable.startswith('N TIME'): variable = variable.replace('TIME', ' INITIAL TIME') initials.append(variable.split(' ', 1)[-1]) dictofsubs = {} for i in subscripts: Family = builder.make_python_identifier(i.split("=")[0]) if not re.search('-', i): Elements = i.split("=")[1].split(",") for i in range(len(Elements)): Elements[i] = builder.make_python_identifier( Elements[i].strip()) dictofsubs[Family] = dict(zip(Elements, range(len(Elements)))) elif not re.search(r'[a-zA-Z]', i.split('=')[1]): Elements = i.split("=")[1].split("-") Elements = range(int(Elements[0]), int(Elements[1]) + 1) for i, j in enumerate(Elements): Elements[i] = str(j) dictofsubs[Family] = dict(zip(Elements, range(len(Elements)))) elif re.search('-', i.split('=')[1]): Elements = i.split('=')[1] dictofsubs[Family] = re.sub( r'([\w\s]+)', lambda s: builder.make_python_identifier(s.group(1)), Elements) for subfamily in dictofsubs: if isinstance(dictofsubs[subfamily], str): fromelem = dictofsubs[subfamily].split('-')[0] toelem = dictofsubs[subfamily].split('-')[1] for family in dictofsubs: if fromelem in dictofsubs[family] and toelem in dictofsubs[ family] and isinstance(dictofsubs[family], dict): dictofsubs[subfamily] = range( dictofsubs[family][fromelem], dictofsubs[family][toelem] + 1) dictofsubs[subfamily].append(family) def getelempos(allelements): getelempos.sumfor = '' position = [] elements = allelements.replace(' ', '').split(',') for pos, element in enumerate(elements): if element in dictofsubs.keys(): if isinstance(dictofsubs[element], dict): position.append(':') else: position.append(sorted(dictofsubs[element][:-1])) else: if element == '*': position.append(':') if len(position) == 1 or (position[-2] == ':' and len(position) > 1): sumat = pos else: sumat = pos - 1 getelempos.sumfor = ',' + str(sumat) for d in dictofsubs.itervalues(): try: position.append(d[element]) except: pass if len(re.findall(r'\*', allelements)) > 1: getelempos.sumfor = '' return tuple(position) getelempos.sumfor = '' def getnumofelements(element): """ Parameters ---------- element <string of subscripts> returns a list of the sizes of the dimensions. A 4x3x6 array would return [4,3,6] """ # todo: make this elementstr or something if element == '': # or ("subscript_"+element) in subscripts: return 0 position = [] elements = element.replace('!', '').replace('', '').split(',') for element in elements: if element in dictofsubs.keys(): if isinstance(dictofsubs[element], list): position.append( (getnumofelements(dictofsubs[element][-1]))[0]) else: position.append(len(dictofsubs[element])) else: if element == '*': position.append(1) for d in dictofsubs.itervalues(): try: (d[element]) except: pass else: position.append(len(d)) return position pysd.builder.dictofsubs = dictofsubs pysd.builder.getelempos = getelempos pysd.builder.getnumofelements = getnumofelements ####################################################### new_model = mdl_file.replace('.dyn', '.py') Builder = builder.Builder(new_model, dictofsubs) macrolist = macroextractor(mdl_file) ####################################################### lookups = [] for line in tables: line = re.findall(r'([^\[\=]*)(\[([^\]]*)\])?\=(.*)', line)[0] identifier = builder.make_python_identifier(line[0]) sub = re.sub( r'([^\[\]\(\)\+\-\*\/\,\.]+)', lambda s: builder.make_python_identifier(s.group(1)) if not re.match('\d*$', s.group(1)) else s.group(1), line[2]) expression = line[3].split(',') # if line[0].startswith('Data Table for Inventory by LOS and Rank'): # print sub,identifier copairlist = [] for i in range(len(expression) / 2): copair = (expression[2 * i], expression[2 * i + 1]) copairlist.append(copair) lookups.append([identifier, [sub], [copairlist]]) for pos1, value1 in enumerate(lookups): for pos2, value2 in enumerate(lookups): if pos1 != pos2: try: if value1[0] == value2[0]: lookups[pos1][1] += value2[1] lookups[pos1][2] += value2[2] lookups[pos2] = '' except: pass lookups = filter(None, lookups) lookuptofix = [] for line in lookups: lookuptofix.append(line[0]) try: Builder.add_lookup(line[0], '', line[1], line[2]) except Exception as e: print line[0] print e lookuptofix = '(?i)(' + '|'.join(lookuptofix) + ')\(\)\[*[^\]\(]*\]*\(' stocks = [] init_val = [] for line in levels: line = re.findall(r'([^\[\=]*)(\[([^\]]*)\])?\=.*?\+DT\*\((.*)\)', line)[0] identifier = builder.make_python_identifier(line[0]) sub = re.sub(r'([^\[\]\(\)\+\-\*\/\,\.]+)', lambda s: builder.make_python_identifier(s.group(1)), line[2]) expression = formatexpression(line[3], identifier, functiondic, macrolist) expression = re.sub( r'\[([^\]]*)\]', lambda s: '[' + ','.join( map(str, getelempos(s.group(1).replace('()', '')))) + ']' + getelempos.sumfor, expression) expression = re.sub( r'\,(%s\(\))\)' % subs, lambda s: ',' + ','.join( map(str, getnumofelements(s.group(1).replace('()', '')))) + ')', expression) expression = re.sub(r'%s' % lookuptofix, r'\1(', expression) subinexpression = '[' + ','.join(map(str, getelempos(sub))) + ']' expression = re.sub( r'(?<!\(|\w|,)(\w+)\(\)\[\:(\,\:)*\]', r'functions.shorthander(\1(),\1.dimension_dir,loc_dimension_dir,_subscript_dict)' + subinexpression, expression.replace(' ', '')) stocks.append([identifier, [sub], [expression], [], []]) for line in initials: line = re.findall(r'([^\[\=]*)(\[([^\]]*)\])?\=(.*)', line)[0] identifier = builder.make_python_identifier(line[0]) sub = re.sub(r'([^\[\]\(\)\+\-\*\/\,\.]+)', lambda s: builder.make_python_identifier(s.group(1)), line[2]) expression = formatexpression(line[3], identifier, functiondic, macrolist) expression = re.sub( r'\[([^\]]*)\]', lambda s: '[' + ','.join( map(str, getelempos(s.group(1).replace('()', '')))) + ']' + getelempos.sumfor, expression) expression = re.sub( r'\,(%s\(\))\)' % subs, lambda s: ',' + ','.join( map(str, getnumofelements(s.group(1).replace('()', '')))) + ')', expression) expression = re.sub(r'%s' % lookuptofix, r'\1(', expression) subinexpression = '[' + ','.join(map(str, getelempos(sub))) + ']' if not re.search( r'(np.sum|functions.sumv|functions.prdv|functions.sclprd|smooth|delay|functions.tabhl)', expression): expression = re.sub( r'(\w+)\(\)\[\:(\,\:)*\]', r'functions.shorthander(\1(),\1.dimension_dir,loc_dimension_dir,_subscript_dict)' + subinexpression, expression.replace(' ', '')) # expression=re.sub(r'((?<!(\w|\[|\]|\+|\-|\.))[\d\.]+(?!(\w|\[|\]|\+|\-|\.)))',lambda s: s.group(1) if re.search(r'\.',s.group(1)) else s.group(1)+'.0',expression) init_val.append([identifier, [sub], [expression]]) for pos1, value1 in enumerate(stocks): for pos2, value2 in enumerate(stocks): if pos1 != pos2: try: if value1[0] == value2[0]: stocks[pos1][1] += value2[1] stocks[pos1][2] += value2[2] stocks[pos2] = '' except: pass stocks = filter(None, stocks) for pos1, value1 in enumerate(stocks): for pos2, value2 in enumerate(init_val): try: if value1[0] == value2[0]: stocks[pos1][3] += value2[1] stocks[pos1][4] += value2[2] init_val[pos2] = '' except: pass init_val = filter(None, init_val) for line in stocks: try: Builder.add_stock(line[0], line[1], line[2], line[4], line[3]) except: print line[0] flaux = [] for line in auxiliaries: line = re.findall(r'([^\[\=]*)(\[([^\]]*)\])?\=(.*)', line)[0] identifier = builder.make_python_identifier(line[0]) sub = re.sub(r'([^\[\]\(\)\+\-\*\/\,\.]+)', lambda s: builder.make_python_identifier(s.group(1)), line[2]) expression = formatexpression(line[3], identifier, functiondic, macrolist) expression = re.sub( r'\[([^\]]*)\]', lambda s: '[' + ','.join( map(str, getelempos(s.group(1).replace('()', '')))) + ']' + getelempos.sumfor, expression) expression = re.sub( r'(\,(\w*)\(\)([\,\)]))', lambda s: ',' + ','.join( map(str, getnumofelements(s.group(2)))) + s.group(3) if s.group(2) in dictofsubs.keys() else s.group(1), expression) expression = re.sub(r'(?<!\w)%s' % lookuptofix, r'\1(', expression) subinexpression = '[' + ','.join(map(str, getelempos(sub))) + ']' if not re.search( r'(np.sum|functions.sumv|functions.prdv|functions.sclprd|smooth|delay|functions.tabhl)', expression): expression = re.sub( r'(\w+)\(\)\[\:(\,\:)*\]', r'functions.shorthander(\1(),\1.dimension_dir,loc_dimension_dir,_subscript_dict)' + subinexpression, expression.replace(' ', '')) # expression=re.sub(r'((?<!(\w|\[|\]|\+|\-|\.))[\d\.]+(?!(\w|\[|\]|\+|\-|\.)))',lambda s: s.group(1) if re.search(r'\.',s.group(1)) else s.group(1)+'.0',expression) expression = smooth(expression, [sub], Builder) expression = delay(expression, [sub], Builder) # if len(re.findall(r'(?<!\,|\[)[\+\-\*\/]',expression))!=len(re.findall(r'(?<!\,|\[)[\+\-\*\/]',line[3])): # print identifier flaux.append([identifier, [sub], [expression]]) for pos, line_init in enumerate(init_val): for line_aux in flaux: if line_init[0] == line_aux[0]: init_val[pos][0] = '_init_' + line_init[0] # init_val[pos][2]=['functions.initial("%s",'%(line_init[0]+line_init[1][0])+line_init[2][0]+')'] flaux = flaux + init_val for pos1, value1 in enumerate(flaux): for pos2, value2 in enumerate(flaux): if pos1 != pos2: try: if value1[0] == value2[0]: flaux[pos1][1] += value2[1] flaux[pos1][2] += value2[2] flaux[pos2] = '' except: pass flaux = filter(None, flaux) for line in flaux: Builder.add_flaux(line[0], line[1], line[2]) Builder.write() macros(mdl_file, checking, functiondic) return new_model