def __init__(self, tool='', fileobj = None):
'''Initialize self.'''
self.newdoc = not fileobj
tmpio = strio()
tmpattr = '____daedoc_pseudo_attr'
if not self.newdoc:
tmpio = open(str(fileobj), buffering=0, encoding='UTF-8')
tmpstr = tmpio.read()
tmpstr = tmpstr.replace('xmlns=', tmpattr+'=', 1)
tmpio = strio(tmpstr)
else:
tmpio = strio(daedoc.__template)
self.tree = xet.parse(tmpio)
self.root = self.tree.getroot()
if self.root.attrib.has_key(tmpattr):
self.root.attrib['xmlns'] = self.root.attrib[tmpattr]
self.root.attrib.pop(tmpattr)
else:
self.root.attrib['xmlns']='http://www.collada.org/2005/11/COLLADASchema'
tmpel = self.root.find('asset/contributor/author')
tmpel.text = str(getpass.getuser())
tmpel = self.root.find('asset/contributor/authoring_tool')
tmpel.text = 'pyllada ' + __version__ + ' | ' + tool
def loadSplitData(csv_str, symbol):
splitconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db"))
splitcurs = splitconn.cursor()
cols = ['TYPE', 'DATE', 'VALUE']
params = (symbol, ','.join(cols))
splitcurs.execute("drop table if exists ds_%s" % params[0])
create = "create table ds_%s (%s)" % params
print create
splitcurs.execute(create)
reader = list(csv.DictReader(strio(csv_str), fieldnames=cols))
data = filter(lambda d: d['TYPE'] in ('DIVIDEND', 'SPLIT'), reader)
vals = [(row['TYPE'][0:3], '-'.join([
row['DATE'].strip()[0:4], row['DATE'].strip()[4:6],
row['DATE'].strip()[6:]
]), row['VALUE']) for row in data]
qlst = []
for unused in xrange(len(cols)):
qlst.append('?')
qms = ','.join(qlst)
ins = 'insert into ds_%s (TYPE,DATE,VALUE) values (%s)' % (params[0], qms)
print ins
splitcurs.executemany(ins, vals)
splitconn.commit()
print
def get_content_type_and_body(fields, files={}):
boundary = '------BOUNDARY-%d'%datetime.datetime.now().microsecond
crlf = '\r\n'
items = []
for key, value in fields.iteritems():
items += [
''.join(['--', boundary]),
'Content-Disposition: form-data; name="%s"' % str(key),
'',
str(value),
]
for key, values in files.iteritems():
filename, value = values
if isinstance(value, file):
value = strio(value.read())
if not isinstance(value, (cStringIO.InputType, StringIO.StringIO)):
raise ValueError, "Files must be of type <StringIO> or <file>"
items += [
''.join(['--', boundary]),
'Content-Disposition: form-data; name="%s"; filename="%s"' % (str(key), str(filename)),
'Content-Type: %s' % (mimetypes.guess_type(filename)[0] or 'application/octet-stream'),
'',
value.getvalue(),
]
items += [
''.join(['--', boundary, '--']),
''
]
body = crlf.join(items)
content_type = 'Multipart/form-data; boundary=%s' % boundary
return content_type, body
def loadEodData(csv_str, symbol):
eodconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db"))
eodcurs = eodconn.cursor()
reader = csv.DictReader(strio(csv_str))
for row in reader:
cols = map(lambda s: s.replace(' ', '_').upper(), sorted(row.keys()))
params = (symbol, ','.join(cols))
eodcurs.execute("drop table if exists _%s" % params[0])
create = "create table _%s (%s)" % params
print create
eodcurs.execute(create)
break
vals = [(
row['Date'],
row['Open'],
row['High'],
row['Low'],
row['Close'],
row['Volume'],
row['Adj Close']) for row in reader]
qlst = []
for unused in xrange(len(vals[0])):
qlst.append('?')
qms = ','.join(qlst)
ins = 'insert into _%s (DATE,OPEN,HIGH,LOW,CLOSE,VOLUME,ADJ_CLOSE) values (%s)' % (params[0], qms)
print ins
eodcurs.executemany(ins, vals)
eodconn.commit()
print
def loadSplitData(csv_str, symbol):
splitconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db"))
splitcurs = splitconn.cursor()
cols = ["TYPE", "DATE", "VALUE"]
params = (symbol, ",".join(cols))
splitcurs.execute("drop table if exists ds_%s" % params[0])
create = "create table ds_%s (%s)" % params
print create
splitcurs.execute(create)
reader = list(csv.DictReader(strio(csv_str), fieldnames=cols))
data = filter(lambda d: d["TYPE"] in ("DIVIDEND", "SPLIT"), reader)
vals = [
(
row["TYPE"][0:3],
"-".join([row["DATE"].strip()[0:4], row["DATE"].strip()[4:6], row["DATE"].strip()[6:]]),
row["VALUE"],
)
for row in data
]
qlst = []
for unused in xrange(len(cols)):
qlst.append("?")
qms = ",".join(qlst)
ins = "insert into ds_%s (TYPE,DATE,VALUE) values (%s)" % (params[0], qms)
print ins
splitcurs.executemany(ins, vals)
splitconn.commit()
print
def loadEodData(csv_str, symbol):
eodconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db"))
eodcurs = eodconn.cursor()
reader = csv.DictReader(strio(csv_str))
for row in reader:
cols = map(lambda s: s.replace(' ', '_').upper(), sorted(row.keys()))
params = (symbol, ','.join(cols))
eodcurs.execute("drop table if exists _%s" % params[0])
create = "create table _%s (%s)" % params
print create
eodcurs.execute(create)
break
vals = [(row['Date'], row['Open'], row['High'], row['Low'], row['Close'],
row['Volume'], row['Adj Close']) for row in reader]
qlst = []
for unused in xrange(len(vals[0])):
qlst.append('?')
qms = ','.join(qlst)
ins = 'insert into _%s (DATE,OPEN,HIGH,LOW,CLOSE,VOLUME,ADJ_CLOSE) values (%s)' % (
params[0], qms)
print ins
eodcurs.executemany(ins, vals)
eodconn.commit()
print
def process(fname):
ic = open(fname,'rb')
encrypted, d, offset = Mjo.header(ic)
#Get entrypoint details
ic.seek(0x10)
primary = Util.read_int(ic)
ic.seek(0x18)
record_count = Util.read_int(ic)
entrypoints = {}
for i in range(record_count):
code = Util.read_int32(ic)
offs = Util.read_int(ic)
entrypoints[offs] = (code, offs == primary)
#Get generic channel for bytecode
if encrypted:
s = Mjo.crypt(ic)
bc = strio(s)
ic.seek(0)
open(fname+'.dec','wb').write(ic.read(offset) + s)
else:
ic.seek(offset)
s = ic.read()
bc = strio(s)
endPos = len(s) - 1
#Read commands
cmds = []
targets = {}
# try:
if 1:
while bc.tell()<endPos:
cpos = bc.tell()
cmd = read_cmd(bc, cpos + offset)
cmds.append((cpos, cmd))
if cmd.__class__ == Cmd:
for i in cmd[2]:
targets[i] = 0
# except:
if 0:
print 'warning: this mjo may be truncated.'
import pdb
pdb.set_trace()
#Order labels
targets = targets.keys()
targets.sort(reverse=True)
labels = {}
for n in range(len(targets)):
labels[targets[n]] = n + 1
#Open output file
dir, basename = os.path.split(fname)
output_dir = g_output_dir
if not output_dir:
output_dir = dir
noext = os.path.splitext(basename)[0]
oc = open(output_dir + '\\' + noext + '.mjs', 'w')
res = resstrs(output_dir + '\\' + noext + '.sjs')
#Output code
def do_lbl(pos):
if pos in labels:
oc.write('\n @%d\n'%labels[pos])
res.finalise()
if pos in targets:
targets.remove(pos)
def do_pos(pos):
if include_offsets == 'Bytecode': oc.write("{-%04x-} "%pos)
elif include_offsets == 'File': oc.write("{-%04x-} "%(pos+offset))
stack = []
def pop_all():
if stack:
res.finalise()
while stack:
pos, elt = stack.pop()
do_lbl(pos)
do_pos(pos)
oc.write('push %s\n'%elt)
def pop_n(n, clear = True):
skip = None
l = []
for n in range(n):
if skip or not stack:
l.append('pop')
else:
pos, elt = stack.pop()
if not skip and pos in labels:
skip = pos
l.append(elt)
if clear or skip:
pop_all()
if skip: do_lbl(skip)
return ', '.join(l)
for pos, cmd in cmds:
if pos in entrypoints:
code, is_entry = entrypoints[pos]
del entrypoints[pos]
f = "function"
if is_entry: f = "entrypoint"
oc.write("\n#%s $%08lx\n"%(f, code&0xFFFFFFFF))
res.finalise()
if cmd == None:
pop_all()
do_lbl(pos)
elif cmd.__class__ == Stack:
s, n, b = cmd
res.finalise()
if n: s = "%s(%s)"%(s,pop_n(n, False))
elif b:
if res.enabled: s=res.add_whole(s)
else: s = repr(s)
stack.append((pos,s))
elif cmd.__class__ == Cmd:
s, n, l = cmd
if n: args = " (%s)"%pop_n(n)
else:
args = ""
pop_all()
do_lbl(pos)
do_pos(pos)
res.finalise()
oc.write(s)
oc.write(args)
if len(l) == 1:
oc.write(' @%d'%labels[l[0]])
elif l:
oc.write(' { %s }'%', '.join(['@%d'%labels[n] for n in l]))
oc.write('\n')
else:#text
pop_all()
do_lbl(pos)
do_pos(pos)
if res.enabled:
r = res.add_part(cmd)
if r: oc.write("%s\n"%r)
else: oc.write(repr(cmd))
pop_all()
#Clean up
if targets:
print "Warning: labels not generated: %s" % ", ".join(['@%d'%labels[n] for n in targets])
if entrypoints:
print "Warning: entrypoints not generated: %s"%", ".join(['$%08lx'%entrypoints[n][0]&0xFFFFFFFF for n in entrypoints])
oc.close()
res.close()
opInfo[op] = ('op%03x'%op, [4])
for op in (0x1b0, 0x1b1, 0x1b2, 0x1b3, 0x1b5, 0x1b8, 0x1c0, 0x1d0, 0x1d2,
0x1d8, 0x1d9, 0x270, 0x272, 0x290, 0x2c0, 0x802, 0x837):
opInfo[op] = ('op%03x'%op, [8])
ic = open(sys.argv[1],'rb')
encrypted, l, offset = Mjo.header(ic)
ic.seek(16)
primary = struct.unpack('I', ic.read(4))[0]
ic.seek(0x18)
entrypoints = {}
for i in range(struct.unpack('I', ic.read(4))[0]):
code, offs = struct.unpack('II', ic.read(8))
entrypoints[offs] = (code, offs == primary)
if encrypted:
s = strio(Mjo.crypt(ic))
else:
ic.seek(offset)
s = strio(ic.read(l))
dasm = []
labels={}
ident = 0
while 1:
curoff = s.tell()
if curoff in entrypoints:
code, isEntry = entrypoints[curoff]
if isEntry: dasm.append((curoff, '#entry 0x%08x'%code))
else: dasm.append((curoff, '#function 0x%08x'%code))
op = s.read(2)
def process(fname):
ic = open(fname, 'rb')
encrypted, d, offset = Mjo.header(ic)
#Get entrypoint details
ic.seek(0x10)
primary = Util.read_int(ic)
ic.seek(0x18)
record_count = Util.read_int(ic)
entrypoints = {}
for i in range(record_count):
code = Util.read_int32(ic)
offs = Util.read_int(ic)
entrypoints[offs] = (code, offs == primary)
#Get generic channel for bytecode
if encrypted:
s = Mjo.crypt(ic)
bc = strio(s)
ic.seek(0)
open(fname + '.dec', 'wb').write(ic.read(offset) + s)
else:
ic.seek(offset)
s = ic.read()
bc = strio(s)
endPos = len(s) - 1
#Read commands
cmds = []
targets = {}
# try:
if 1:
while bc.tell() < endPos:
cpos = bc.tell()
cmd = read_cmd(bc, cpos + offset)
cmds.append((cpos, cmd))
if cmd.__class__ == Cmd:
for i in cmd[2]:
targets[i] = 0
# except:
if 0:
print 'warning: this mjo may be truncated.'
import pdb
pdb.set_trace()
#Order labels
targets = targets.keys()
targets.sort(reverse=True)
labels = {}
for n in range(len(targets)):
labels[targets[n]] = n + 1
#Open output file
dir, basename = os.path.split(fname)
output_dir = g_output_dir
if not output_dir:
output_dir = dir
noext = os.path.splitext(basename)[0]
oc = open(output_dir + '\\' + noext + '.mjs', 'w')
res = resstrs(output_dir + '\\' + noext + '.sjs')
#Output code
def do_lbl(pos):
if pos in labels:
oc.write('\n @%d\n' % labels[pos])
res.finalise()
if pos in targets:
targets.remove(pos)
def do_pos(pos):
if include_offsets == 'Bytecode': oc.write("{-%04x-} " % pos)
elif include_offsets == 'File': oc.write("{-%04x-} " % (pos + offset))
stack = []
def pop_all():
if stack:
res.finalise()
while stack:
pos, elt = stack.pop()
do_lbl(pos)
do_pos(pos)
oc.write('push %s\n' % elt)
def pop_n(n, clear=True):
skip = None
l = []
for n in range(n):
if skip or not stack:
l.append('pop')
else:
pos, elt = stack.pop()
if not skip and pos in labels:
skip = pos
l.append(elt)
if clear or skip:
pop_all()
if skip: do_lbl(skip)
return ', '.join(l)
for pos, cmd in cmds:
if pos in entrypoints:
code, is_entry = entrypoints[pos]
del entrypoints[pos]
f = "function"
if is_entry: f = "entrypoint"
oc.write("\n#%s $%08lx\n" % (f, code & 0xFFFFFFFF))
res.finalise()
if cmd == None:
pop_all()
do_lbl(pos)
elif cmd.__class__ == Stack:
s, n, b = cmd
res.finalise()
if n: s = "%s(%s)" % (s, pop_n(n, False))
elif b:
if res.enabled: s = res.add_whole(s)
else: s = repr(s)
stack.append((pos, s))
elif cmd.__class__ == Cmd:
s, n, l = cmd
if n: args = " (%s)" % pop_n(n)
else:
args = ""
pop_all()
do_lbl(pos)
do_pos(pos)
res.finalise()
oc.write(s)
oc.write(args)
if len(l) == 1:
oc.write(' @%d' % labels[l[0]])
elif l:
oc.write(' { %s }' % ', '.join(['@%d' % labels[n] for n in l]))
oc.write('\n')
else: #text
pop_all()
do_lbl(pos)
do_pos(pos)
if res.enabled:
r = res.add_part(cmd)
if r: oc.write("%s\n" % r)
else: oc.write(repr(cmd))
pop_all()
#Clean up
if targets:
print "Warning: labels not generated: %s" % ", ".join(
['@%d' % labels[n] for n in targets])
if entrypoints:
print "Warning: entrypoints not generated: %s" % ", ".join(
['$%08lx' % entrypoints[n][0] & 0xFFFFFFFF for n in entrypoints])
oc.close()
res.close()
def download_upload(url, key, bucket, extra={}, redundancy=True):
res = requests.get(url, stream=True)
mp = bucket.initiate_multipart_upload(key, metadata=extra, reduced_redundancy=not redundancy)
idx = count()
for block in res.iter_content:
mp.upload_part_from_file(strio(block), next(idx))
def __str__(self):
'''Return string representation of self.'''
tmpstr = strio();
self.write(tmpstr)
return tmpstr.getvalue()
opInfo[op] = ('op%03x' % op, [4])
for op in (0x1b0, 0x1b1, 0x1b2, 0x1b3, 0x1b5, 0x1b8, 0x1c0, 0x1d0, 0x1d2,
0x1d8, 0x1d9, 0x270, 0x272, 0x290, 0x2c0, 0x802, 0x837):
opInfo[op] = ('op%03x' % op, [8])
ic = open(sys.argv[1], 'rb')
encrypted, l, offset = Mjo.header(ic)
ic.seek(16)
primary = struct.unpack('I', ic.read(4))[0]
ic.seek(0x18)
entrypoints = {}
for i in range(struct.unpack('I', ic.read(4))[0]):
code, offs = struct.unpack('II', ic.read(8))
entrypoints[offs] = (code, offs == primary)
if encrypted:
s = strio(Mjo.crypt(ic))
else:
ic.seek(offset)
s = strio(ic.read(l))
dasm = []
labels = {}
ident = 0
while 1:
curoff = s.tell()
if curoff in entrypoints:
code, isEntry = entrypoints[curoff]
if isEntry: dasm.append((curoff, '#entry 0x%08x' % code))
else: dasm.append((curoff, '#function 0x%08x' % code))
op = s.read(2)
