def __init__(self, tool='', fileobj = None): '''Initialize self.''' self.newdoc = not fileobj tmpio = strio() tmpattr = '____daedoc_pseudo_attr' if not self.newdoc: tmpio = open(str(fileobj), buffering=0, encoding='UTF-8') tmpstr = tmpio.read() tmpstr = tmpstr.replace('xmlns=', tmpattr+'=', 1) tmpio = strio(tmpstr) else: tmpio = strio(daedoc.__template) self.tree = xet.parse(tmpio) self.root = self.tree.getroot() if self.root.attrib.has_key(tmpattr): self.root.attrib['xmlns'] = self.root.attrib[tmpattr] self.root.attrib.pop(tmpattr) else: self.root.attrib['xmlns']='http://www.collada.org/2005/11/COLLADASchema' tmpel = self.root.find('asset/contributor/author') tmpel.text = str(getpass.getuser()) tmpel = self.root.find('asset/contributor/authoring_tool') tmpel.text = 'pyllada ' + __version__ + ' | ' + tool
def loadSplitData(csv_str, symbol): splitconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db")) splitcurs = splitconn.cursor() cols = ['TYPE', 'DATE', 'VALUE'] params = (symbol, ','.join(cols)) splitcurs.execute("drop table if exists ds_%s" % params[0]) create = "create table ds_%s (%s)" % params print create splitcurs.execute(create) reader = list(csv.DictReader(strio(csv_str), fieldnames=cols)) data = filter(lambda d: d['TYPE'] in ('DIVIDEND', 'SPLIT'), reader) vals = [(row['TYPE'][0:3], '-'.join([ row['DATE'].strip()[0:4], row['DATE'].strip()[4:6], row['DATE'].strip()[6:] ]), row['VALUE']) for row in data] qlst = [] for unused in xrange(len(cols)): qlst.append('?') qms = ','.join(qlst) ins = 'insert into ds_%s (TYPE,DATE,VALUE) values (%s)' % (params[0], qms) print ins splitcurs.executemany(ins, vals) splitconn.commit() print
def get_content_type_and_body(fields, files={}): boundary = '------BOUNDARY-%d'%datetime.datetime.now().microsecond crlf = '\r\n' items = [] for key, value in fields.iteritems(): items += [ ''.join(['--', boundary]), 'Content-Disposition: form-data; name="%s"' % str(key), '', str(value), ] for key, values in files.iteritems(): filename, value = values if isinstance(value, file): value = strio(value.read()) if not isinstance(value, (cStringIO.InputType, StringIO.StringIO)): raise ValueError, "Files must be of type <StringIO> or <file>" items += [ ''.join(['--', boundary]), 'Content-Disposition: form-data; name="%s"; filename="%s"' % (str(key), str(filename)), 'Content-Type: %s' % (mimetypes.guess_type(filename)[0] or 'application/octet-stream'), '', value.getvalue(), ] items += [ ''.join(['--', boundary, '--']), '' ] body = crlf.join(items) content_type = 'Multipart/form-data; boundary=%s' % boundary return content_type, body
def loadEodData(csv_str, symbol): eodconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db")) eodcurs = eodconn.cursor() reader = csv.DictReader(strio(csv_str)) for row in reader: cols = map(lambda s: s.replace(' ', '_').upper(), sorted(row.keys())) params = (symbol, ','.join(cols)) eodcurs.execute("drop table if exists _%s" % params[0]) create = "create table _%s (%s)" % params print create eodcurs.execute(create) break vals = [( row['Date'], row['Open'], row['High'], row['Low'], row['Close'], row['Volume'], row['Adj Close']) for row in reader] qlst = [] for unused in xrange(len(vals[0])): qlst.append('?') qms = ','.join(qlst) ins = 'insert into _%s (DATE,OPEN,HIGH,LOW,CLOSE,VOLUME,ADJ_CLOSE) values (%s)' % (params[0], qms) print ins eodcurs.executemany(ins, vals) eodconn.commit() print
def loadSplitData(csv_str, symbol): splitconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db")) splitcurs = splitconn.cursor() cols = ["TYPE", "DATE", "VALUE"] params = (symbol, ",".join(cols)) splitcurs.execute("drop table if exists ds_%s" % params[0]) create = "create table ds_%s (%s)" % params print create splitcurs.execute(create) reader = list(csv.DictReader(strio(csv_str), fieldnames=cols)) data = filter(lambda d: d["TYPE"] in ("DIVIDEND", "SPLIT"), reader) vals = [ ( row["TYPE"][0:3], "-".join([row["DATE"].strip()[0:4], row["DATE"].strip()[4:6], row["DATE"].strip()[6:]]), row["VALUE"], ) for row in data ] qlst = [] for unused in xrange(len(cols)): qlst.append("?") qms = ",".join(qlst) ins = "insert into ds_%s (TYPE,DATE,VALUE) values (%s)" % (params[0], qms) print ins splitcurs.executemany(ins, vals) splitconn.commit() print
def loadEodData(csv_str, symbol): eodconn = sqlite3.connect(os.path.join(dllib.DB_DIR, "div_splits.db")) eodcurs = eodconn.cursor() reader = csv.DictReader(strio(csv_str)) for row in reader: cols = map(lambda s: s.replace(' ', '_').upper(), sorted(row.keys())) params = (symbol, ','.join(cols)) eodcurs.execute("drop table if exists _%s" % params[0]) create = "create table _%s (%s)" % params print create eodcurs.execute(create) break vals = [(row['Date'], row['Open'], row['High'], row['Low'], row['Close'], row['Volume'], row['Adj Close']) for row in reader] qlst = [] for unused in xrange(len(vals[0])): qlst.append('?') qms = ','.join(qlst) ins = 'insert into _%s (DATE,OPEN,HIGH,LOW,CLOSE,VOLUME,ADJ_CLOSE) values (%s)' % ( params[0], qms) print ins eodcurs.executemany(ins, vals) eodconn.commit() print
def process(fname): ic = open(fname,'rb') encrypted, d, offset = Mjo.header(ic) #Get entrypoint details ic.seek(0x10) primary = Util.read_int(ic) ic.seek(0x18) record_count = Util.read_int(ic) entrypoints = {} for i in range(record_count): code = Util.read_int32(ic) offs = Util.read_int(ic) entrypoints[offs] = (code, offs == primary) #Get generic channel for bytecode if encrypted: s = Mjo.crypt(ic) bc = strio(s) ic.seek(0) open(fname+'.dec','wb').write(ic.read(offset) + s) else: ic.seek(offset) s = ic.read() bc = strio(s) endPos = len(s) - 1 #Read commands cmds = [] targets = {} # try: if 1: while bc.tell()<endPos: cpos = bc.tell() cmd = read_cmd(bc, cpos + offset) cmds.append((cpos, cmd)) if cmd.__class__ == Cmd: for i in cmd[2]: targets[i] = 0 # except: if 0: print 'warning: this mjo may be truncated.' import pdb pdb.set_trace() #Order labels targets = targets.keys() targets.sort(reverse=True) labels = {} for n in range(len(targets)): labels[targets[n]] = n + 1 #Open output file dir, basename = os.path.split(fname) output_dir = g_output_dir if not output_dir: output_dir = dir noext = os.path.splitext(basename)[0] oc = open(output_dir + '\\' + noext + '.mjs', 'w') res = resstrs(output_dir + '\\' + noext + '.sjs') #Output code def do_lbl(pos): if pos in labels: oc.write('\n @%d\n'%labels[pos]) res.finalise() if pos in targets: targets.remove(pos) def do_pos(pos): if include_offsets == 'Bytecode': oc.write("{-%04x-} "%pos) elif include_offsets == 'File': oc.write("{-%04x-} "%(pos+offset)) stack = [] def pop_all(): if stack: res.finalise() while stack: pos, elt = stack.pop() do_lbl(pos) do_pos(pos) oc.write('push %s\n'%elt) def pop_n(n, clear = True): skip = None l = [] for n in range(n): if skip or not stack: l.append('pop') else: pos, elt = stack.pop() if not skip and pos in labels: skip = pos l.append(elt) if clear or skip: pop_all() if skip: do_lbl(skip) return ', '.join(l) for pos, cmd in cmds: if pos in entrypoints: code, is_entry = entrypoints[pos] del entrypoints[pos] f = "function" if is_entry: f = "entrypoint" oc.write("\n#%s $%08lx\n"%(f, code&0xFFFFFFFF)) res.finalise() if cmd == None: pop_all() do_lbl(pos) elif cmd.__class__ == Stack: s, n, b = cmd res.finalise() if n: s = "%s(%s)"%(s,pop_n(n, False)) elif b: if res.enabled: s=res.add_whole(s) else: s = repr(s) stack.append((pos,s)) elif cmd.__class__ == Cmd: s, n, l = cmd if n: args = " (%s)"%pop_n(n) else: args = "" pop_all() do_lbl(pos) do_pos(pos) res.finalise() oc.write(s) oc.write(args) if len(l) == 1: oc.write(' @%d'%labels[l[0]]) elif l: oc.write(' { %s }'%', '.join(['@%d'%labels[n] for n in l])) oc.write('\n') else:#text pop_all() do_lbl(pos) do_pos(pos) if res.enabled: r = res.add_part(cmd) if r: oc.write("%s\n"%r) else: oc.write(repr(cmd)) pop_all() #Clean up if targets: print "Warning: labels not generated: %s" % ", ".join(['@%d'%labels[n] for n in targets]) if entrypoints: print "Warning: entrypoints not generated: %s"%", ".join(['$%08lx'%entrypoints[n][0]&0xFFFFFFFF for n in entrypoints]) oc.close() res.close()
opInfo[op] = ('op%03x'%op, [4]) for op in (0x1b0, 0x1b1, 0x1b2, 0x1b3, 0x1b5, 0x1b8, 0x1c0, 0x1d0, 0x1d2, 0x1d8, 0x1d9, 0x270, 0x272, 0x290, 0x2c0, 0x802, 0x837): opInfo[op] = ('op%03x'%op, [8]) ic = open(sys.argv[1],'rb') encrypted, l, offset = Mjo.header(ic) ic.seek(16) primary = struct.unpack('I', ic.read(4))[0] ic.seek(0x18) entrypoints = {} for i in range(struct.unpack('I', ic.read(4))[0]): code, offs = struct.unpack('II', ic.read(8)) entrypoints[offs] = (code, offs == primary) if encrypted: s = strio(Mjo.crypt(ic)) else: ic.seek(offset) s = strio(ic.read(l)) dasm = [] labels={} ident = 0 while 1: curoff = s.tell() if curoff in entrypoints: code, isEntry = entrypoints[curoff] if isEntry: dasm.append((curoff, '#entry 0x%08x'%code)) else: dasm.append((curoff, '#function 0x%08x'%code)) op = s.read(2)
def process(fname): ic = open(fname, 'rb') encrypted, d, offset = Mjo.header(ic) #Get entrypoint details ic.seek(0x10) primary = Util.read_int(ic) ic.seek(0x18) record_count = Util.read_int(ic) entrypoints = {} for i in range(record_count): code = Util.read_int32(ic) offs = Util.read_int(ic) entrypoints[offs] = (code, offs == primary) #Get generic channel for bytecode if encrypted: s = Mjo.crypt(ic) bc = strio(s) ic.seek(0) open(fname + '.dec', 'wb').write(ic.read(offset) + s) else: ic.seek(offset) s = ic.read() bc = strio(s) endPos = len(s) - 1 #Read commands cmds = [] targets = {} # try: if 1: while bc.tell() < endPos: cpos = bc.tell() cmd = read_cmd(bc, cpos + offset) cmds.append((cpos, cmd)) if cmd.__class__ == Cmd: for i in cmd[2]: targets[i] = 0 # except: if 0: print 'warning: this mjo may be truncated.' import pdb pdb.set_trace() #Order labels targets = targets.keys() targets.sort(reverse=True) labels = {} for n in range(len(targets)): labels[targets[n]] = n + 1 #Open output file dir, basename = os.path.split(fname) output_dir = g_output_dir if not output_dir: output_dir = dir noext = os.path.splitext(basename)[0] oc = open(output_dir + '\\' + noext + '.mjs', 'w') res = resstrs(output_dir + '\\' + noext + '.sjs') #Output code def do_lbl(pos): if pos in labels: oc.write('\n @%d\n' % labels[pos]) res.finalise() if pos in targets: targets.remove(pos) def do_pos(pos): if include_offsets == 'Bytecode': oc.write("{-%04x-} " % pos) elif include_offsets == 'File': oc.write("{-%04x-} " % (pos + offset)) stack = [] def pop_all(): if stack: res.finalise() while stack: pos, elt = stack.pop() do_lbl(pos) do_pos(pos) oc.write('push %s\n' % elt) def pop_n(n, clear=True): skip = None l = [] for n in range(n): if skip or not stack: l.append('pop') else: pos, elt = stack.pop() if not skip and pos in labels: skip = pos l.append(elt) if clear or skip: pop_all() if skip: do_lbl(skip) return ', '.join(l) for pos, cmd in cmds: if pos in entrypoints: code, is_entry = entrypoints[pos] del entrypoints[pos] f = "function" if is_entry: f = "entrypoint" oc.write("\n#%s $%08lx\n" % (f, code & 0xFFFFFFFF)) res.finalise() if cmd == None: pop_all() do_lbl(pos) elif cmd.__class__ == Stack: s, n, b = cmd res.finalise() if n: s = "%s(%s)" % (s, pop_n(n, False)) elif b: if res.enabled: s = res.add_whole(s) else: s = repr(s) stack.append((pos, s)) elif cmd.__class__ == Cmd: s, n, l = cmd if n: args = " (%s)" % pop_n(n) else: args = "" pop_all() do_lbl(pos) do_pos(pos) res.finalise() oc.write(s) oc.write(args) if len(l) == 1: oc.write(' @%d' % labels[l[0]]) elif l: oc.write(' { %s }' % ', '.join(['@%d' % labels[n] for n in l])) oc.write('\n') else: #text pop_all() do_lbl(pos) do_pos(pos) if res.enabled: r = res.add_part(cmd) if r: oc.write("%s\n" % r) else: oc.write(repr(cmd)) pop_all() #Clean up if targets: print "Warning: labels not generated: %s" % ", ".join( ['@%d' % labels[n] for n in targets]) if entrypoints: print "Warning: entrypoints not generated: %s" % ", ".join( ['$%08lx' % entrypoints[n][0] & 0xFFFFFFFF for n in entrypoints]) oc.close() res.close()
def download_upload(url, key, bucket, extra={}, redundancy=True): res = requests.get(url, stream=True) mp = bucket.initiate_multipart_upload(key, metadata=extra, reduced_redundancy=not redundancy) idx = count() for block in res.iter_content: mp.upload_part_from_file(strio(block), next(idx))
def __str__(self): '''Return string representation of self.''' tmpstr = strio(); self.write(tmpstr) return tmpstr.getvalue()
opInfo[op] = ('op%03x' % op, [4]) for op in (0x1b0, 0x1b1, 0x1b2, 0x1b3, 0x1b5, 0x1b8, 0x1c0, 0x1d0, 0x1d2, 0x1d8, 0x1d9, 0x270, 0x272, 0x290, 0x2c0, 0x802, 0x837): opInfo[op] = ('op%03x' % op, [8]) ic = open(sys.argv[1], 'rb') encrypted, l, offset = Mjo.header(ic) ic.seek(16) primary = struct.unpack('I', ic.read(4))[0] ic.seek(0x18) entrypoints = {} for i in range(struct.unpack('I', ic.read(4))[0]): code, offs = struct.unpack('II', ic.read(8)) entrypoints[offs] = (code, offs == primary) if encrypted: s = strio(Mjo.crypt(ic)) else: ic.seek(offset) s = strio(ic.read(l)) dasm = [] labels = {} ident = 0 while 1: curoff = s.tell() if curoff in entrypoints: code, isEntry = entrypoints[curoff] if isEntry: dasm.append((curoff, '#entry 0x%08x' % code)) else: dasm.append((curoff, '#function 0x%08x' % code)) op = s.read(2)