def writeobj(obj, write_directory=None, name=None, clean=False):
if name is None:
name = Output._format_name(obj)
if write_directory is None:
write_directory = "\\\\?\\" + cfg.workingdir
if not os.path.exists(write_directory):
os.makedirs(write_directory)
if util.is_numeric(obj) or (type(obj) is str) or (type(obj) is bool):
Output.csvwrite(os.path.join(write_directory, 'vars.csv'),
[[name, obj]],
writetype='ab')
elif (type(obj) == dict) or (type(obj)
== defaultdict) or (type(obj)
== OrderedDict):
Output._writedict(obj, write_directory, name)
elif type(obj) == list or type(obj) == tuple:
Output._writelist(obj, write_directory, name)
elif type(obj) == pd.core.frame.DataFrame:
obj = Output.clean_df(obj)
obj.to_csv(os.path.join(write_directory, str(name) + '.csv'))
elif type(obj) is pd.tseries.index.DatetimeIndex:
pd.DataFrame(obj).to_csv(
os.path.join(write_directory,
str(name) + '.csv'))
elif inspect.isclass(type(obj)) and hasattr(obj, '__dict__'):
Output._writeclass(obj, write_directory, name)
def _writelist(obj, write_directory, name):
csv_elements = [
item for item in obj if util.is_numeric(item) or type(item) == str
]
Output.csvwrite(os.path.join(write_directory, 'vars.csv'),
[[name] + csv_elements],
writetype='ab')
def ingest_tsv(filepath):
log.info('ingesting %s as tsv file', filepath)
save_file_metadata(filepath, status='parsing', filetype='tsv')
with open(filepath, 'rU') as fid:
reader = csv.reader(fid, delimiter='\t')
header = reader.next()
log.debug("%d columns: %s", len(header), ", ".join(header))
if len(header) == 0:
raise ValueError('header row must contain at least one column')
keys = [normalize_column_name(h) for h in header]
def parse(row):
if len(keys) == len(row):
return dict(zip(keys, row))
parsed = [parse(row) for row in reader]
parsed = [v for v in parsed if v is not None]
header = [{'raw': h, 'key': k} for h, k in itertools.izip(header, keys)]
for h in header:
data = [p[h['key']] for p in parsed]
if all(is_boolean(d) for d in data):
h['datatype'] = 'boolean'
elif all(is_numeric(d) for d in data):
h['datatype'] = 'numeric'
else:
h['datatype'] = 'string'
save_file_metadata(filepath, headers=header)
return parsed
def _writedict(obj, write_directory, name):
if not len(obj):
return
# two types of dictionaries one is key and class obj value
if all([util.is_numeric(val) or (type(val) is str) or (type(val) is bool) or (type(val) is list) or (type(val) == tuple) or (type(val) == np.ndarray) for val in obj.values()]):
Output.csvwrite(os.path.join(write_directory, str(name)+'.csv'), obj.items())
else:
new_directory = os.path.join(write_directory, str(name))
for key, value in obj.items():
Output.writeobj(value, new_directory, Output._clean_string(key))
def _format_name(obj, default=None):
name = Output._get_name(obj)
if default is None or name==default:
return Output._clean_string(name)
if util.is_numeric(default):
name = str(default) + '--' + str(name)
else:
name = str(default)
return Output._clean_string(name)
def _format_name(obj, default=None):
name = Output._get_name(obj)
if default is None or name == default:
return Output._clean_string(name)
if util.is_numeric(default):
name = str(default) + '--' + str(name)
else:
name = str(default)
return Output._clean_string(name)
def create(root):
for func in root.children:
global lv
local_var_count = 0;
stmts = "";
for i, param in enumerate(func.children[0].children):
symbol_table[str(param)] = par_template.substitute(var_num= str(i+1));
for i, define in enumerate(func.children[1].children):
symbol_table[str(define)] = getdefine_template.substitute(var_num= str(i+1));
local_var_count += 1;
for statement in func.children[2].children:
if(str(statement) == '='):
if str(statement.children[1]) in op_table:
if str(str(statement.children[0])) not in symbol_table:
symbol_table[str(statement.children[0])] = getdefine_template.substitute(var_num= local_var_count +1)
local_var_count += 1
stmts += is_operator(statement.children[1], statement.children[0], local_var_count)
else:
if str(statement.children[0]) not in symbol_table:
symbol_table[str(statement.children[0])] = getdefine_template.substitute(var_num= local_var_count +1);
local_var_count += 1;
if util.is_numeric(str(statement.children[1])):
if str(statement.children[1]) not in symbol_table:
print consttable_template.substitute(val = float(str(statement.children[1])))
symbol_table[str(statement.children[1])] = constaddr_template.substitute(val = float(str(statement.children[1])))
ifn_c = ""
elif str(statement.children[1]).isalpha():
ifn_c = "addl $16, %ebx"
sa = symbol_table[str(statement.children[1])] % {"destreg": "%ebx"}
da = symbol_table[str(statement.children[0])] % {"destreg": "%eax"}
stmts += equ_template.substitute(sourceaddr = sa, destaddr = da, loop_val = lv, ifnot_constant = ifn_c)
lv += 1
else:
invoke_asm = ""
total_args = len(statement.children)
arg_template = """\n%(arg)s\n\tmovl %%eax, %(N)s(%%esp)"""
for arg in statement.children:
invoke_asm += arg_template % {"arg": symbol_table[str(arg)] % {"destreg": "%eax"}, "N": str(total_args*4)}
total_args -= 1
stmts += invoke_template.substitute(name = str(statement), Nplus1 = len(statement.children) + 1, args = invoke_asm)
fbody = stmts;
#temporary variable management is 'create space for as many temp variables as will be needed'
allocate = setdefine_template.substitute(var_num= util.max_locals(func), body = fbody)
print func_template.substitute(name= str(func), body= allocate)
def _writedict(obj, write_directory, name):
if not len(obj):
return
# two types of dictionaries one is key and class obj value
if all([
util.is_numeric(val) or (type(val) is str)
or (type(val) is bool) or (type(val) is list)
or (type(val) == tuple) or (type(val) == np.ndarray)
for val in obj.values()
]):
Output.csvwrite(os.path.join(write_directory,
str(name) + '.csv'), obj.items())
else:
new_directory = os.path.join(write_directory, str(name))
for key, value in obj.items():
Output.writeobj(value, new_directory,
Output._clean_string(key))
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Non-numeric type."
var_struct = u.get_var(name=name, initializer=val, reuse=is_global)
holder = var_struct.val_
init_dict[holder] = val
var = var_struct.var
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def is_operator(op_parent, destination, local_var_count):
global lv
asm = ""
for child in op_parent.children:
if str(child) in op_table:
local_var_count += 1
tvar = getdefine_template.substitute(var_num = str(local_var_count))
asm += is_operator(child, tvar, local_var_count)
if int(str(child.getChildIndex())) == 0:
ifn_c1 = "addl $16, %ebx"
sa1 = tvar % {"destreg": "%ebx"}
else:
ifn_c2 = "addl $16, %edx"
sa2 = tvar % {"destreg": "%edx"}
else:
if util.is_numeric(str(child)):
if str(child) not in symbol_table:
print consttable_template.substitute(val = float(str(child)))
symbol_table[str(child)] = constaddr_template.substitute(val = float(str(child)))
if int(str(child.getChildIndex())) == 0:
ifn_c1 = ""
else:
ifn_c2 = ""
elif str(child).isalpha():
if int(str(child.getChildIndex())) == 0:
ifn_c1 = "addl $16, %ebx"
else:
ifn_c2 = "addl $16, %edx"
if int(str(child.getChildIndex())) == 0:
sa1 = symbol_table[str(child)] % {"destreg": "%ebx"}
else:
sa2 = symbol_table[str(child)] % {"destreg": "%edx"}
if str(destination) in symbol_table:
da = symbol_table[str(destination)] % {"destreg": "%eax"}
else:
da = destination % {"destreg": "%eax"}
asm += equWithOp_template.substitute(sourceaddr_1 = sa1, sourceaddr_2 = sa2, destaddr = da, loop_val = lv, operation = op_table[str(op_parent)], ifnot_constant_1 = ifn_c1, ifnot_constant_2 = ifn_c2)
lv += 1
return asm
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Non-numeric type."
var_struct = u.get_var(name=name, initializer=val, reuse=is_global)
holder = var_struct.val_
init_dict[holder] = val
var = var_struct.var
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def init_var(val, name, is_global=False):
"""Helper to create variables with numpy or TF initial values."""
if isinstance(val, tf.Tensor):
var = u.get_variable(name=name, initializer=val, reuse=is_global)
else:
val = np.array(val)
assert u.is_numeric(val), "Unknown type"
holder = tf.placeholder(dtype,
shape=val.shape,
name=name + "_holder")
var = u.get_variable(name=name,
initializer=holder,
reuse=is_global)
init_dict[holder] = val
if is_global:
global_vars.append(var)
else:
local_vars.append(var)
return var
def writeobj(obj, write_directory=None, name=None, clean=False):
if name is None:
name = Output._format_name(obj)
if write_directory is None:
write_directory = "\\\\?\\" + cfg.workingdir
if not os.path.exists(write_directory):
os.makedirs(write_directory)
if util.is_numeric(obj) or (type(obj) is str) or (type(obj) is bool):
Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name, obj]], writetype='ab')
elif (type(obj) == dict) or (type(obj) == defaultdict) or (type(obj) == OrderedDict):
Output._writedict(obj, write_directory, name)
elif type(obj) == list or type(obj) == tuple:
Output._writelist(obj, write_directory, name)
elif type(obj) == pd.core.frame.DataFrame:
obj = Output.clean_df(obj)
obj.to_csv(os.path.join(write_directory, str(name)+'.csv'))
elif type(obj) is pd.tseries.index.DatetimeIndex:
pd.DataFrame(obj).to_csv(os.path.join(write_directory, str(name)+'.csv'))
elif inspect.isclass(type(obj)) and hasattr(obj, '__dict__'):
Output._writeclass(obj, write_directory, name)
def _writelist(obj, write_directory, name):
csv_elements = [item for item in obj if util.is_numeric(item) or type(item) == str]
Output.csvwrite(os.path.join(write_directory, 'vars.csv'), [[name] + csv_elements], writetype='ab')
