def writeobj(obj, write_directory=None, name=None, clean=False): if name is None: name = Output._format_name(obj) if write_directory is None: write_directory = "\\\\?\\" + cfg.workingdir if not os.path.exists(write_directory): os.makedirs(write_directory) if util.is_numeric(obj) or (type(obj) is str) or (type(obj) is bool): Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name, obj]], writetype='ab') elif (type(obj) == dict) or (type(obj) == defaultdict) or (type(obj) == OrderedDict): Output._writedict(obj, write_directory, name) elif type(obj) == list or type(obj) == tuple: Output._writelist(obj, write_directory, name) elif type(obj) == pd.core.frame.DataFrame: obj = Output.clean_df(obj) obj.to_csv(os.path.join(write_directory, str(name) + '.csv')) elif type(obj) is pd.tseries.index.DatetimeIndex: pd.DataFrame(obj).to_csv( os.path.join(write_directory, str(name) + '.csv')) elif inspect.isclass(type(obj)) and hasattr(obj, '__dict__'): Output._writeclass(obj, write_directory, name)
def _writelist(obj, write_directory, name): csv_elements = [ item for item in obj if util.is_numeric(item) or type(item) == str ] Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name] + csv_elements], writetype='ab')
def ingest_tsv(filepath): log.info('ingesting %s as tsv file', filepath) save_file_metadata(filepath, status='parsing', filetype='tsv') with open(filepath, 'rU') as fid: reader = csv.reader(fid, delimiter='\t') header = reader.next() log.debug("%d columns: %s", len(header), ", ".join(header)) if len(header) == 0: raise ValueError('header row must contain at least one column') keys = [normalize_column_name(h) for h in header] def parse(row): if len(keys) == len(row): return dict(zip(keys, row)) parsed = [parse(row) for row in reader] parsed = [v for v in parsed if v is not None] header = [{'raw': h, 'key': k} for h, k in itertools.izip(header, keys)] for h in header: data = [p[h['key']] for p in parsed] if all(is_boolean(d) for d in data): h['datatype'] = 'boolean' elif all(is_numeric(d) for d in data): h['datatype'] = 'numeric' else: h['datatype'] = 'string' save_file_metadata(filepath, headers=header) return parsed
def _writedict(obj, write_directory, name): if not len(obj): return # two types of dictionaries one is key and class obj value if all([util.is_numeric(val) or (type(val) is str) or (type(val) is bool) or (type(val) is list) or (type(val) == tuple) or (type(val) == np.ndarray) for val in obj.values()]): Output.csvwrite(os.path.join(write_directory, str(name)+'.csv'), obj.items()) else: new_directory = os.path.join(write_directory, str(name)) for key, value in obj.items(): Output.writeobj(value, new_directory, Output._clean_string(key))
def _format_name(obj, default=None): name = Output._get_name(obj) if default is None or name==default: return Output._clean_string(name) if util.is_numeric(default): name = str(default) + '--' + str(name) else: name = str(default) return Output._clean_string(name)
def _format_name(obj, default=None): name = Output._get_name(obj) if default is None or name == default: return Output._clean_string(name) if util.is_numeric(default): name = str(default) + '--' + str(name) else: name = str(default) return Output._clean_string(name)
def create(root): for func in root.children: global lv local_var_count = 0; stmts = ""; for i, param in enumerate(func.children[0].children): symbol_table[str(param)] = par_template.substitute(var_num= str(i+1)); for i, define in enumerate(func.children[1].children): symbol_table[str(define)] = getdefine_template.substitute(var_num= str(i+1)); local_var_count += 1; for statement in func.children[2].children: if(str(statement) == '='): if str(statement.children[1]) in op_table: if str(str(statement.children[0])) not in symbol_table: symbol_table[str(statement.children[0])] = getdefine_template.substitute(var_num= local_var_count +1) local_var_count += 1 stmts += is_operator(statement.children[1], statement.children[0], local_var_count) else: if str(statement.children[0]) not in symbol_table: symbol_table[str(statement.children[0])] = getdefine_template.substitute(var_num= local_var_count +1); local_var_count += 1; if util.is_numeric(str(statement.children[1])): if str(statement.children[1]) not in symbol_table: print consttable_template.substitute(val = float(str(statement.children[1]))) symbol_table[str(statement.children[1])] = constaddr_template.substitute(val = float(str(statement.children[1]))) ifn_c = "" elif str(statement.children[1]).isalpha(): ifn_c = "addl $16, %ebx" sa = symbol_table[str(statement.children[1])] % {"destreg": "%ebx"} da = symbol_table[str(statement.children[0])] % {"destreg": "%eax"} stmts += equ_template.substitute(sourceaddr = sa, destaddr = da, loop_val = lv, ifnot_constant = ifn_c) lv += 1 else: invoke_asm = "" total_args = len(statement.children) arg_template = """\n%(arg)s\n\tmovl %%eax, %(N)s(%%esp)""" for arg in statement.children: invoke_asm += arg_template % {"arg": symbol_table[str(arg)] % {"destreg": "%eax"}, "N": str(total_args*4)} total_args -= 1 stmts += invoke_template.substitute(name = str(statement), Nplus1 = len(statement.children) + 1, args = invoke_asm) fbody = stmts; #temporary variable management is 'create space for as many temp variables as will be needed' allocate = setdefine_template.substitute(var_num= util.max_locals(func), body = fbody) print func_template.substitute(name= str(func), body= allocate)
def _writedict(obj, write_directory, name): if not len(obj): return # two types of dictionaries one is key and class obj value if all([ util.is_numeric(val) or (type(val) is str) or (type(val) is bool) or (type(val) is list) or (type(val) == tuple) or (type(val) == np.ndarray) for val in obj.values() ]): Output.csvwrite(os.path.join(write_directory, str(name) + '.csv'), obj.items()) else: new_directory = os.path.join(write_directory, str(name)) for key, value in obj.items(): Output.writeobj(value, new_directory, Output._clean_string(key))
def init_var(val, name, is_global=False): """Helper to create variables with numpy or TF initial values.""" if isinstance(val, tf.Tensor): var = u.get_variable(name=name, initializer=val, reuse=is_global) else: val = np.array(val) assert u.is_numeric(val), "Non-numeric type." var_struct = u.get_var(name=name, initializer=val, reuse=is_global) holder = var_struct.val_ init_dict[holder] = val var = var_struct.var if is_global: global_vars.append(var) else: local_vars.append(var) return var
def is_operator(op_parent, destination, local_var_count): global lv asm = "" for child in op_parent.children: if str(child) in op_table: local_var_count += 1 tvar = getdefine_template.substitute(var_num = str(local_var_count)) asm += is_operator(child, tvar, local_var_count) if int(str(child.getChildIndex())) == 0: ifn_c1 = "addl $16, %ebx" sa1 = tvar % {"destreg": "%ebx"} else: ifn_c2 = "addl $16, %edx" sa2 = tvar % {"destreg": "%edx"} else: if util.is_numeric(str(child)): if str(child) not in symbol_table: print consttable_template.substitute(val = float(str(child))) symbol_table[str(child)] = constaddr_template.substitute(val = float(str(child))) if int(str(child.getChildIndex())) == 0: ifn_c1 = "" else: ifn_c2 = "" elif str(child).isalpha(): if int(str(child.getChildIndex())) == 0: ifn_c1 = "addl $16, %ebx" else: ifn_c2 = "addl $16, %edx" if int(str(child.getChildIndex())) == 0: sa1 = symbol_table[str(child)] % {"destreg": "%ebx"} else: sa2 = symbol_table[str(child)] % {"destreg": "%edx"} if str(destination) in symbol_table: da = symbol_table[str(destination)] % {"destreg": "%eax"} else: da = destination % {"destreg": "%eax"} asm += equWithOp_template.substitute(sourceaddr_1 = sa1, sourceaddr_2 = sa2, destaddr = da, loop_val = lv, operation = op_table[str(op_parent)], ifnot_constant_1 = ifn_c1, ifnot_constant_2 = ifn_c2) lv += 1 return asm
def init_var(val, name, is_global=False): """Helper to create variables with numpy or TF initial values.""" if isinstance(val, tf.Tensor): var = u.get_variable(name=name, initializer=val, reuse=is_global) else: val = np.array(val) assert u.is_numeric(val), "Unknown type" holder = tf.placeholder(dtype, shape=val.shape, name=name + "_holder") var = u.get_variable(name=name, initializer=holder, reuse=is_global) init_dict[holder] = val if is_global: global_vars.append(var) else: local_vars.append(var) return var
def writeobj(obj, write_directory=None, name=None, clean=False): if name is None: name = Output._format_name(obj) if write_directory is None: write_directory = "\\\\?\\" + cfg.workingdir if not os.path.exists(write_directory): os.makedirs(write_directory) if util.is_numeric(obj) or (type(obj) is str) or (type(obj) is bool): Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name, obj]], writetype='ab') elif (type(obj) == dict) or (type(obj) == defaultdict) or (type(obj) == OrderedDict): Output._writedict(obj, write_directory, name) elif type(obj) == list or type(obj) == tuple: Output._writelist(obj, write_directory, name) elif type(obj) == pd.core.frame.DataFrame: obj = Output.clean_df(obj) obj.to_csv(os.path.join(write_directory, str(name)+'.csv')) elif type(obj) is pd.tseries.index.DatetimeIndex: pd.DataFrame(obj).to_csv(os.path.join(write_directory, str(name)+'.csv')) elif inspect.isclass(type(obj)) and hasattr(obj, '__dict__'): Output._writeclass(obj, write_directory, name)
def _writelist(obj, write_directory, name): csv_elements = [item for item in obj if util.is_numeric(item) or type(item) == str] Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name] + csv_elements], writetype='ab')