def __init__(self, *a, **kw): from cStringIO import StringIO self.__csio = StringIO(*a, **kw)
def moderatesignups(): global commentHashesAndComments commentHashesAndComments = {} stringio = StringIO() stringio.write('<html>\n<head>\n</head>\n\n') # redditSession = loginAndReturnRedditSession() redditSession = loginOAuthAndReturnRedditSession() submissions = getSubmissionsForRedditSession(redditSession) flat_comments = getCommentsForSubmissions(submissions) retiredHashes = retiredCommentHashes() i = 1 stringio.write( '<iframe name="invisibleiframe" style="display:none;"></iframe>\n') stringio.write("<h3>") stringio.write(os.getcwd()) stringio.write("<br>\n") for submission in submissions: stringio.write(submission.title) stringio.write("<br>\n") stringio.write("</h3>\n\n") stringio.write( '<form action="copydisplayduringsignuptoclipboard.html" method="post" target="invisibleiframe">' ) stringio.write( '<input type="submit" value="Copy display-during-signup.py stdout to clipboard">' ) stringio.write('</form>') for comment in flat_comments: # print comment.is_root # print comment.score i += 1 commentHash = sha1() commentHash.update(comment.fullname) commentHash.update(comment.body.encode('utf-8')) commentHash = commentHash.hexdigest() if commentHash not in retiredHashes: commentHashesAndComments[commentHash] = comment authorName = str( comment.author ) # can be None if author was deleted. So check for that and skip if it's None. stringio.write("<hr>\n") stringio.write('<font color="blue"><b>') stringio.write( authorName ) # can be None if author was deleted. So check for that and skip if it's None. stringio.write('</b></font><br>') if ParticipantCollection().hasParticipantNamed(authorName): stringio.write( ' <small><font color="green">(member)</font></small>') # if ParticipantCollection().participantNamed(authorName).isStillIn: # stringio.write(' <small><font color="green">(in)</font></small>') # else: # stringio.write(' <small><font color="red">(out)</font></small>') else: stringio.write( ' <small><font color="red">(not a member)</font></small>') stringio.write( '<form action="takeaction.html" method="post" target="invisibleiframe">' ) stringio.write( '<input type="submit" name="actiontotake" value="Signup" style="color:white;background-color:green">' ) # stringio.write('<input type="submit" name="actiontotake" value="Signup and checkin">') # stringio.write('<input type="submit" name="actiontotake" value="Relapse">') # stringio.write('<input type="submit" name="actiontotake" value="Reinstate">') stringio.write( '<input type="submit" name="actiontotake" value="Skip comment">' ) stringio.write( '<input type="submit" name="actiontotake" value="Skip comment and don\'t upvote">' ) stringio.write('<input type="hidden" name="username" value="' + b64encode(authorName) + '">') stringio.write('<input type="hidden" name="commenthash" value="' + commentHash + '">') # stringio.write('<input type="hidden" name="commentpermalink" value="' + comment.permalink + '">') stringio.write('</form>') stringio.write( bleach.clean(markdown.markdown(comment.body.encode('utf-8')), tags=['p'])) stringio.write("\n<br><br>\n\n") stringio.write('</html>') pageString = stringio.getvalue() stringio.close() return Response(pageString, mimetype='text/html')
def changeFormatImage( input, outformat, **options ): """ Function to change the format of the input image. Usage: >>> from NIST.fingerprint.functions import changeFormatImage To convert an PIL image to a RAW string, use the following commands: >>> from PIL import Image >>> imgPIL = Image.new( "L", ( 500, 500 ), 255 ) >>> imgRAW = changeFormatImage( imgPIL, "RAW" ) >>> imgRAW == chr( 255 ) * 500 * 500 True All format supported by PIL are supported as output format: >>> changeFormatImage( imgPIL, "TIFF" ) # doctest: +ELLIPSIS <PIL.TiffImagePlugin.TiffImageFile image mode=L size=500x500 at ...> >>> changeFormatImage( imgPIL, "PNG" ) # doctest: +ELLIPSIS <PIL.PngImagePlugin.PngImageFile image mode=L size=500x500 at ...> You can also convert a StringIO buffer: >>> from cStringIO import StringIO >>> imgBuffer = StringIO() >>> imgPIL.save( imgBuffer, 'JPEG' ) >>> imgRAW2 = changeFormatImage( imgBuffer, "RAW" ) >>> imgRAW == imgRAW2 True A `notImplemented` is raised if the input format is not supported or if the output format is not implemented in this function or by PIL: >>> changeFormatImage( None, "RAW" ) Traceback (most recent call last): ... notImplemented: Input format not supported >>> changeFormatImage( imgPIL, "WSQ" ) Traceback (most recent call last): ... notImplemented: Output format not supported by PIL """ outformat = outformat.upper() # Convert the input data to PIL format if isinstance( input, Image.Image ): img = input elif isinstance( input, str ): try: buff = StringIO( input ) img = Image.open( buff ) except: if string_to_hex( input[ 0 : 4 ] ) in [ "FFA0FFA4", "FFA0FFA5", "FFA0FFA6", "FFA0FFA2", "FFA0FFA8" ]: img = RAWToPIL( WSQ().decode( input ), **options ) else: if outformat == "RAW": return input else: img = RAWToPIL( input, **options ) elif isinstance( input, ( OutputType, InputType ) ): img = Image.open( input ) else: raise notImplemented( "Input format not supported" ) # Convert the PIL format to the output format if outformat == "PIL": return img elif outformat == "RAW": return PILToRAW( img ) else: try: buff = StringIO() img.save( buff, format = outformat ) return Image.open( buff ) except: raise notImplemented( "Output format not supported by PIL" )
def SPA4Symbol(modelType="fixed"): ''' model for profit test of each parameter ''' t0 = time.time() if modelType == "fixed": n_rvs = range(5, 55, 5) hist_periods = range(50, 130, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7", "0.75", "0.8", "0.85", "0.9", "0.95") myDir = os.path.join(ExpResultsDir, "fixedSymbolSPPortfolio", "LargestMarketValue_200501") resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Fixed_eachParam_Profit.csv") elif modelType == "dynamic": n_rvs = range(5, 55, 5) hist_periods = range(90, 120 + 10, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7") myDir = os.path.join(ExpResultsDir, "dynamicSymbolSPPortfolio", "LargestMarketValue_200501_rv50") resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Dynamic_eachParam_Profit.csv") avgIO = StringIO() if not os.path.exists(resFile): with open(resFile, 'ab') as fout: fout.write( 'n, h, alpha, SPA_Q, sampling, n_rule, n_period, P-value\n') for n_rv in n_rvs: #load exp ROI for period in hist_periods: if n_rv == 50 and period == 50: continue for alpha in alphas: t1 = time.time() if modelType == "fixed": dirName = "fixedSymbolSPPortfolio_n%s_p%s_s200_a%s" % ( n_rv, period, alpha) elif modelType == "dynamic": dirName = "dynamicSymbolSPPortfolio_n%s_p%s_s200_a%s" % ( n_rv, period, alpha) exps = glob(os.path.join(myDir, dirName, "20050103-20131231_*")) if len(exps) > 3: exps = exps[:3] #set base ROI n_periods = 2236 base_rois = np.zeros(n_periods) diffobj = ROIDiffObject(base_rois) for exp in exps: #load comparison rois df = pd.read_pickle(os.path.join(exp, 'wealthProcess.pkl')) proc = df.sum(axis=1) exp_finalWealth = proc[-1] if exp_finalWealth >= 0: wrois = proc.pct_change() wrois[0] = 0 diffobj.setCompareROIs(wrois) print " SPA4Symbol n-h-alpha:%s-%s-%s, load data OK, %.3f secs" % ( n_rv, period, alpha, time.time() - t1) t2 = time.time() #SPA test Q = 0.5 n_samplings = 5000 verbose = True try: pvalue = SPATest.SPATest(diffobj, Q, n_samplings, "SPA_C", verbose) except AssertionError: continue print "n-h-alpha:%s-%s-%s, (n_rules, n_periods):(%s, %s), SPA_C:%s elapsed:%.3f secs" % ( n_rv, period, alpha, diffobj.n_rules, diffobj.n_periods, pvalue, time.time() - t2) avgIO.write("%s,%s,%s,%s,%s,%s,%s,%s\n" % (n_rv, period, alpha, Q, n_samplings, diffobj.n_rules, diffobj.n_periods, pvalue)) with open(resFile, 'ab') as fout: fout.write(avgIO.getvalue()) avgIO.close() print "SPA4Symbol SPA, elapsed %.3f secs" % (time.time() - t0)
def SPA4BHSymbol(modelType="fixed", years=None): ''' buy-and-hold versus. SP model ''' t0 = time.time() if modelType == "fixed": n_rvs = range(5, 55, 5) hist_periods = range(50, 130, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7", "0.75", "0.8", "0.85", "0.9", "0.95") myDir = os.path.join(ExpResultsDir, "fixedSymbolSPPortfolio", "LargestMarketValue_200501") if not years: resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Fixed_BetterBH.csv") else: resFile = os.path.join(ExpResultsDir, "SPA", "SPA_year_Fixed_BetterBH.csv") elif modelType == "dynamic": n_rvs = range(5, 55, 5) hist_periods = range(90, 120 + 10, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7") myDir = os.path.join(ExpResultsDir, "dynamicSymbolSPPortfolio", "LargestMarketValue_200501_rv50") if not years: resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Dynamic_BetterBH.csv") else: resFile = os.path.join(ExpResultsDir, "SPA", "SPA_year_Dynamic_BetterBH.csv") bhDir = os.path.join(ExpResultsDir, "BuyandHoldPortfolio") #stats file avgIO = StringIO() if not os.path.exists(resFile): if not years: avgIO.write('n_rv, SPA_Q, sampling, n_rule, n_period, P-value\n') else: avgIO.write( 'startDate, endDate, n_rv, SPA_Q, sampling, n_rule, n_period, P-value\n' ) bh_rois = None tgt_rois = [] for n_rv in n_rvs: t1 = time.time() #load buyhold ROI bh_df = pd.read_pickle(os.path.join(bhDir, "wealthSum_n%s.pkl" % n_rv)) bh_finalWealth = bh_df[-1] bh_rois = bh_df.pct_change() bh_rois[0] = 0 #load model ROI for period in hist_periods: for alpha in alphas: if modelType == "fixed": dirName = "fixedSymbolSPPortfolio_n%s_p%s_s200_a%s" % ( n_rv, period, alpha) elif modelType == "dynamic": dirName = "dynamicSymbolSPPortfolio_n%s_p%s_s200_a%s" % ( n_rv, period, alpha) exps = glob(os.path.join(myDir, dirName, "20050103-20131231_*")) if len(exps) > 3: exps = exps[:3] for exp in exps: #load comparison rois df = pd.read_pickle(os.path.join(exp, 'wealthProcess.pkl')) proc = df.sum(axis=1) exp_finalWealth = proc[-1] if exp_finalWealth >= bh_finalWealth: wrois = proc.pct_change() wrois[0] = 0 tgt_rois.append(wrois) print " SPA4BHSymbol n_rv: %s, load data OK, %.3f secs" % ( n_rv, time.time() - t1) if not years: #set diff obj diffobj = ROIDiffObject(bh_rois) for rois in tgt_rois: diffobj.setCompareROIs(rois) #SPA test t2 = time.time() Q = 0.5 n_samplings = 5000 verbose = True pvalue = SPATest.SPATest(diffobj, Q, n_samplings, "SPA_C", verbose) print "full n_rv:%s, (n_rules, n_periods):(%s, %s), SPA_C:%s elapsed:%.3f secs" % ( n_rv, diffobj.n_rules, diffobj.n_periods, pvalue, time.time() - t2) avgIO.write("%s,%s,%s,%s,%s,%s\n" % (n_rv, Q, n_samplings, diffobj.n_rules, diffobj.n_periods, pvalue)) else: for year in years: startDate, endDate = date(year, 1, 1), date(year, 12, 31) diffobj = ROIDiffObject(bh_rois[startDate:endDate]) for rois in tgt_rois: diffobj.setCompareROIs(rois[startDate:endDate]) t2 = time.time() Q = 0.5 n_samplings = 5000 verbose = True pvalue = SPATest.SPATest(diffobj, Q, n_samplings, "SPA_C", verbose) print "year:%s n_rv:%s, (n_rules, n_periods):(%s, %s), SPA_C:%s elapsed:%.3f secs" % ( year, n_rv, diffobj.n_rules, diffobj.n_periods, pvalue, time.time() - t2) avgIO.write( "%s, %s,%s,%s,%s,%s,%s,%s\n" % (bh_rois[startDate:endDate].index[0], bh_rois[startDate:endDate].index[-1], n_rv, Q, n_samplings, diffobj.n_rules, diffobj.n_periods, pvalue)) with open(resFile, 'ab') as fout: fout.write(avgIO.getvalue()) avgIO.close() print "SPA4BHSymbol SPA, elapsed %.3f secs" % (time.time() - t0)
def __init__(self, force_type=None): self.buf = StringIO() self.force_type = force_type self.in_json = False
def get_marker_bytes(): img = make_marker(radius, fill_color, stroke_color, stroke_width, opacity) img_sio = StringIO() img.save(img_sio, 'PNG') return img_sio.getvalue()
def test_Publisher(self): import rospy from rospy.impl.registration import get_topic_manager, Registration from rospy.topics import Publisher, DEFAULT_BUFF_SIZE # Publisher(self, name, data_class, subscriber_listener=None, tcp_nodelay=False, latch=False, headers=None) name = 'foo' rname = rospy.resolve_name('foo') data_class = test_rospy.msg.Val # test invalid params for n in [None, '', 1]: try: Publisher(n, data_class) self.fail("should not allow invalid name") except ValueError: pass for d in [None, 1, TestRospyTopics]: try: Publisher(name, d) self.fail("should now allow invalid data_class") except ValueError: pass try: Publisher(name, None) self.fail("None should not be allowed for data_class") except ValueError: pass # round 1: test basic params pub = Publisher(name, data_class) self.assertEquals(rname, pub.resolved_name) # - pub.name is left in for backwards compatiblity, but resolved_name is preferred self.assertEquals(rname, pub.name) self.assertEquals(data_class, pub.data_class) self.assertEquals('test_rospy/Val', pub.type) self.assertEquals(data_class._md5sum, pub.md5sum) self.assertEquals(Registration.PUB, pub.reg_type) # verify impl as well impl = get_topic_manager().get_impl(Registration.PUB, rname) self.assert_(impl == pub.impl) self.assertEquals(rname, impl.resolved_name) self.assertEquals(data_class, impl.data_class) self.failIf(impl.is_latch) self.assertEquals(None, impl.latch) self.assertEquals(0, impl.seq) self.assertEquals(1, impl.ref_count) self.assertEquals(b'', impl.buff.getvalue()) self.failIf(impl.closed) self.failIf(impl.has_connections()) # check publish() fall-through from test_rospy.msg import Val impl.publish(Val('hello world-1')) # check stats self.assertEquals(0, impl.message_data_sent) # check acquire/release don't bomb impl.acquire() impl.release() # do a single publish with connection override. The connection # override is a major cheat as the object isn't even an actual # connection. I will need to add more integrated tests later co1 = ConnectionOverride('co1') self.failIf(impl.has_connection('co1')) impl.add_connection(co1) self.assert_(impl.has_connection('co1')) self.assert_(impl.has_connections()) impl.publish(Val('hello world-1'), connection_override=co1) try: from cStringIO import StringIO except ImportError: from io import BytesIO as StringIO buff = StringIO() Val('hello world-1').serialize(buff) # - check equals, but strip length field first self.assertEquals(co1.data[4:], buff.getvalue()) self.assertEquals(None, impl.latch) # Now enable latch pub = Publisher(name, data_class, latch=True) impl = get_topic_manager().get_impl(Registration.PUB, rname) # have to verify latching in pub impl self.assert_(impl == pub.impl) self.assertEquals(True, impl.is_latch) self.assertEquals(None, impl.latch) self.assertEquals(2, impl.ref_count) co2 = ConnectionOverride('co2') self.failIf(impl.has_connection('co2')) impl.add_connection(co2) for n in ['co1', 'co2']: self.assert_(impl.has_connection(n)) self.assert_(impl.has_connections()) v = Val('hello world-2') impl.publish(v, connection_override=co2) self.assert_(v == impl.latch) buff = StringIO() Val('hello world-2').serialize(buff) # - strip length and check value self.assertEquals(co2.data[4:], buff.getvalue()) # test that latched value is sent to connections on connect co3 = ConnectionOverride('co3') self.failIf(impl.has_connection('co3')) impl.add_connection(co3) for n in ['co1', 'co2', 'co3']: self.assert_(impl.has_connection(n)) self.assert_(impl.has_connections()) self.assertEquals(co3.data[4:], buff.getvalue()) # TODO: tcp_nodelay # TODO: suscribe listener self.assert_(impl.has_connection('co1')) impl.remove_connection(co1) self.failIf(impl.has_connection('co1')) self.assert_(impl.has_connections()) self.assert_(impl.has_connection('co3')) impl.remove_connection(co3) self.failIf(impl.has_connection('co3')) self.assert_(impl.has_connections()) self.assert_(impl.has_connection('co2')) impl.remove_connection(co2) self.failIf(impl.has_connection('co2')) self.failIf(impl.has_connections()) # test publish() latch on a new Publisher object (this was encountered in testing, so I want a test case for it) pub = Publisher('bar', data_class, latch=True, queue_size=0) v = Val('no connection test') pub.impl.publish(v) self.assert_(v == pub.impl.latch) # test connection header h = {'foo': 'bar', 'fuga': 'hoge'} pub = Publisher('header_test', data_class, headers=h, queue_size=0) self.assertEquals(h, pub.impl.headers)
def loads(cls, text): return cls.load(StringIO(text))
def to_print(self): self.ensure_one() style_default = xlwt.easyxf('font: height 240') style_header = xlwt.easyxf('font: height 280, bold on') style_bold = xlwt.easyxf('font: height 240, bold on; align: horz center; ' 'borders: left thin, top thin, bottom thin, right thin') style_table = xlwt.easyxf('font: height 240; borders: left thin, bottom thin, right thin') wb = xlwt.Workbook("UTF-8") ws = wb.add_sheet('SPJ') ws.footer_str = '' title = "SURAT PERINTAH JALAN" y = 0 x = 0 ws.col(x).width = 5000 ws.col(x + 1).width = 12000 ws.col(x + 2).width = 6000 ws.row(0).height_mismatch = 1 ws.row(0).height = 300 ws.row(1).height_mismatch = 1 ws.row(1).height = 280 ws.row(2).height_mismatch = 1 ws.row(2).height = 280 ws.row(3).height_mismatch = 1 ws.row(3).height = 280 ws.row(4).height_mismatch = 1 ws.row(4).height = 280 ws.row(5).height_mismatch = 1 ws.row(5).height = 280 # ws.col(x + 3).width = 4500 # ws.col(x + 4).width = 6000 ws.write(y, x, "{} {}".format(title, self.name), style=style_header) y += 1 ws.write(y, x, "Tanggal SPJ", style=style_default) ws.write(y, x + 1, self.dt_spj, style=style_default) y += 1 ws.write(y, x, "Customer", style=style_default) ws.write(y, x + 1, self.customer_id.name, style=style_default) y += 1 ws.write(y, x, "Tempat Angkut", style=style_default) ws.write(y, x + 1, "{} - {}".format(self.source_group_loc_id.name, self.source_loc_id.name), style=style_default) y += 1 ws.write(y, x, "Tempat Bongkar", style=style_default) ws.write(y, x + 1, "{} - {}".format(self.dest_group_loc_id.name, self.dest_loc_id.name), style=style_default) y += 1 ws.write(y, x, "No Bukti", style=style_default) ws.write(y, x + 1, self.no_bukti, style=style_default) y += 2 ws.write(y, x, "No", style=style_bold) ws.write(y, x + 1, "Nama Barang", style=style_bold) ws.write(y, x + 2, "QTY", style=style_bold) y += 1 # idx = 0 # sum_qty = sum(self.invoice_line_ids.mapped("quantity")) # for inv_line_id in self.invoice_line_ids: # ws.row(y).height_mismatch = 1 # ws.row(y).height = 280 # idx += 1 # tax_name = "" # for tax_id in inv_line_id.invoice_line_tax_ids: # tax_name += tax_id.name ws.write(y, x, 1, style=style_table) ws.write(y, x + 1, self.product_id.name, style=style_table) ws.write(y, x + 2, self.jumlah_barang, style=style_table) y += 1 ws.write(y, x + 1, "Jml. Qty:", style=xlwt.easyxf('font: height 240; align: horiz right')) ws.write(y, x + 2, self.jumlah_barang, style=style_table) y += 3 ws.write(y, x, "Adm. Penjualan, Pengambil Mengetahui, ", style=style_default) y += 3 ws.write(y, x, "(____________) (___________) (___________)", style=style_default) fp = StringIO() wb.save(fp) fp.seek(0) data = fp.read() fp.close() return self.env["ss.download"].download( "spj_{}.xls".format(self.name), data )
def build_select_stmt(self, quals, columns, allow_filtering, verbose=False): stmt_str = StringIO() usedQuals = {} filteredColumns = [] rowid = None binding_values = [] for col in columns: if col != self.ROWIDCOLUMN: filteredColumns.append(col) if (self.query): stmt_str.write(self.query) else: for col in self.rowIdColumns: if col not in filteredColumns: filteredColumns.append(col) stmt_str.write(u"SELECT {0} FROM {1}.{2}".format( ",".join(map(lambda c: '"{0}"'.format(c), filteredColumns)), self.keyspace, self.columnfamily)) isWhere = None eqRestricted = None rangeUsed = False if self.prepare_select_stmt: formatting_str = '?' else: formatting_str = '%s' for qual in quals: if qual.field_name == self.ROWIDCOLUMN: rowid = qual.value if qual.field_name in self.queryableColumns: qual.componentIdx = self.querableColumnsIdx[ qual.field_name] else: qual.componentIdx = 10000 if rowid is not None: ids = json.loads(rowid) for i in range(0, len(self.rowIdColumns)): columnName = self.rowIdColumns[i] binding_values.append( types_mapper.map_object_to_type( ids[i], self.columnsTypes[columnName])) stmt_str.write(u" WHERE {0}".format(u" AND ".join( map(lambda str: str + u" = " + formatting_str, self.rowIdColumns)))) else: sortedQuals = sorted(quals, key=lambda qual: qual.componentIdx) last_clustering_key_idx = 0 for qual in sortedQuals: # Partition key and clustering column can't be null if qual.componentIdx < self.IDX_QUERY_COST and qual.value is None: return None if ISDEBUG or verbose: logger.log( u"qual field {0}; qual index {1}; qual type {2}; qual operator: {4}; qual value {3}" .format(qual.field_name, qual.componentIdx, type(qual.operator), qual.value, qual.operator)) if qual.operator == "=": if (qual.field_name in self.queryableColumns and self.queryableColumns[qual.field_name] != self.REGULAR_QUERY_COST): if self.queryableColumns[ qual. field_name] == self.CLUSTERING_KEY_QUERY_COST: if last_clustering_key_idx == 0 and qual.componentIdx != self.CLUSTERING_KEY_QUERY_COST: eqRestricted = True elif qual.componentIdx - 1 != last_clustering_key_idx and last_clustering_key_idx != 0: eqRestricted = True if (qual.field_name not in usedQuals and not eqRestricted): usedQuals[qual.field_name] = qual.value if self.queryableColumns[ qual. field_name] == self.CLUSTERING_KEY_QUERY_COST: last_clustering_key_idx = qual.componentIdx formatted = u" {0} = {1} ".format( qual.field_name, formatting_str) binding_values.append( types_mapper.map_object_to_type( qual.value, self.columnsTypes[qual.field_name])) if isWhere: stmt_str.write(u" AND ") stmt_str.write(formatted) else: stmt_str.write(u" WHERE ") stmt_str.write(formatted) isWhere = 1 elif allow_filtering: formatted = u" {0} = {1} ".format( qual.field_name, formatting_str) binding_values.append( types_mapper.map_object_to_type( qual.value, self.columnsTypes[qual.field_name])) if isWhere: stmt_str.write(u" AND ") stmt_str.write(formatted) else: stmt_str.write(u" WHERE ") stmt_str.write(formatted) isWhere = 1 elif allow_filtering: formatted = u" {0} = {1} ".format( qual.field_name, formatting_str) binding_values.append( types_mapper.map_object_to_type( qual.value, self.columnsTypes[qual.field_name])) if isWhere: stmt_str.write(u" AND ") stmt_str.write(formatted) else: stmt_str.write(u" WHERE ") stmt_str.write(formatted) isWhere = 1 # IN operator elif qual.operator == (u"=", True): if (qual.field_name in self.queryableColumns): if (self.queryableColumns[qual.field_name] == self.CLUSTERING_KEY_QUERY_COST or self.queryableColumns[qual.field_name] == self.PARTITION_KEY_QUERY_COST): if (qual.field_name not in usedQuals and not eqRestricted and not rangeUsed): usedQuals[qual.field_name] = qual.value formatted = u"{0} IN {1}".format( qual.field_name, formatting_str) binding_value = [] for el in qual.value: binding_value.append( types_mapper.map_object_to_type( el, self.columnsTypes[ qual.field_name])) if self.prepare_select_stmt: binding_values.append(binding_value) else: binding_values.append( ValueSequence(binding_value)) if isWhere: stmt_str.write(u" AND ") stmt_str.write(formatted) else: stmt_str.write(u" WHERE ") stmt_str.write(formatted) isWhere = 1 elif ( qual.operator == "~" or qual.operator == "~~" ) and qual.field_name in self.indexes and self.indexes[ qual. field_name] == "org.apache.cassandra.index.sasi.SASIIndex": if qual.operator == "~": val = "%{0}%".format(qual.value) else: val = qual.value stmt_str.write(u" AND {0} LIKE {1}".format( qual.field_name, formatting_str)) binding_values.append( types_mapper.map_object_to_type( val, self.columnsTypes[qual.field_name])) else: if (qual.operator == ">" or qual.operator == "<" or qual.operator == ">=" or qual.operator == "<="): if (qual.field_name in self.queryableColumns and (self.queryableColumns[qual.field_name] == self.CLUSTERING_KEY_QUERY_COST # only SASI indexes support <,>,>=,<= or (qual.field_name in self.indexes and self.indexes[qual.field_name] == "org.apache.cassandra.index.sasi.SASIIndex")) or (allow_filtering and self.queryableColumns[qual.field_name] != self.PARTITION_KEY_QUERY_COST)): rangeUsed = True if isWhere: stmt_str.write(u" AND {0} {1} {2}".format( qual.field_name, qual.operator, formatting_str)) binding_values.append( types_mapper.map_object_to_type( qual.value, self.columnsTypes[ qual.field_name])) else: stmt_str.write( u" WHERE {0} {1} {2}".format( qual.field_name, qual.operator, formatting_str)) isWhere = 1 binding_values.append( types_mapper.map_object_to_type( qual.value, self.columnsTypes[ qual.field_name])) if (self.limit): stmt_str.write(u" LIMIT {0}".format(self.limit)) if allow_filtering: stmt_str.write(u" ALLOW FILTERING ") statement = stmt_str.getvalue() stmt_str.close() if ISDEBUG: logger.log(u"CQL query: {0}".format(statement), INFO) return (statement, binding_values, filteredColumns)
def read_file(self, path, maxsize): """ path = relative path/filename in project It: - *must* resolve to be under self._projects/project_id or get an error - it must have size in bytes less than the given limit - to download the directory blah/foo, request blah/foo.zip Returns base64-encoded file as an object: {'base64':'... contents ...'} or {'error':"error message..."} in case of an error. """ abspath = os.path.abspath(os.path.join(self.project_path, path)) base, ext = os.path.splitext(abspath) if not abspath.startswith(self.project_path): raise RuntimeError( "path (=%s) must be contained in project path %s" % (path, self.project_path)) if not os.path.exists(abspath): if ext != '.zip': raise RuntimeError("path (=%s) does not exist" % path) else: if os.path.exists(base) and os.path.isdir(base): abspath = os.path.splitext(abspath)[0] else: raise RuntimeError( "path (=%s) does not exist and neither does %s" % (path, base)) filename = os.path.split(abspath)[-1] if os.path.isfile(abspath): # a regular file # TODO: compress the file before base64 encoding (and corresponding decompress # in hub before sending to client) size = os.lstat(abspath).st_size if size > maxsize: raise RuntimeError( "path (=%s) must be at most %s bytes, but it is %s bytes" % (path, maxsize, size)) content = open(abspath).read() else: # a zip file in memory from a directory tree # REFERENCES: # - http://stackoverflow.com/questions/1855095/how-to-create-a-zip-archive-of-a-directory # - https://support.google.com/accounts/answer/6135882 import zipfile from cStringIO import StringIO output = StringIO() relroot = os.path.abspath(os.path.join(abspath, os.pardir)) size = 0 zip = zipfile.ZipFile(output, "w", zipfile.ZIP_DEFLATED, False) for root, dirs, files in os.walk(abspath): # add directory (needed for empty dirs) zip.write(root, os.path.relpath(root, relroot)) for file in files: filename = os.path.join(root, file) if os.path.isfile(filename): # regular files only size += os.lstat(filename).st_size if size > maxsize: raise RuntimeError( "path (=%s) must be at most %s bytes, but it is at least %s bytes" % (path, maxsize, size)) arcname = os.path.join(os.path.relpath(root, relroot), file) zip.write(filename, arcname) # Mark the files as having been created on Windows so that # Unix permissions are not inferred as 0000. for zfile in zip.filelist: zfile.create_system = 0 zip.close() content = output.getvalue() import base64 return {'base64': base64.b64encode(content)}
def from_string_cnet(self, string, keep_constants=True, **kwargs): """ Instanciates a Boolean Network from a string in cnet format. Args: string (string): A cnet format representation of a Boolean Network. Returns: (BooleanNetwork) Examples: String should be structured as follow ``` #.v = number of nodes .v 1 #.l = node label .l 1 node-a .l 2 node-b #.n = (node number) (in-degree) (input node 1) … (input node k) .n 1 2 4 5 01 1 # transition rule ``` See also: :func:`from_file` :func:`from_dict` Note: see examples for more information. """ network_file = StringIO(string) logic = defaultdict(dict) line = network_file.readline() while line != "": if line[0] != '#' and line != '\n': # .v <#-nodes> if '.v' in line: Nnodes = int(line.split()[1]) for inode in range(Nnodes): logic[inode] = {'name':'','in':[],'out':[]} # .l <node-id> <node-name> elif '.l' in line: logic[int(line.split()[1])-1]['name'] = line.split()[2] # .n <node-id> <#-inputs> <input-node-id> elif '.n' in line: inode = int(line.split()[1]) - 1 indegree = int(line.split()[2]) for jnode in range(indegree): logic[inode]['in'].append(int(line.split()[3 + jnode])-1) logic[inode]['out'] = [0 for i in range(2**indegree) if indegree > 0] logic_line = network_file.readline().strip() if indegree <= 0: if logic_line == '': logic[inode]['in'] = [inode] logic[inode]['out'] = [0,1] else: logic[inode]['out'] = [int(logic_line)] else: while logic_line != '\n' and logic_line != '' and len(logic_line)>1: for nlogicline in expand_logic_line(logic_line): logic[inode]['out'][binstate_to_statenum(nlogicline.split()[0])] = int(nlogicline.split()[1]) logic_line = network_file.readline().strip() # .e = end of file elif '.e' in line: break line = network_file.readline() return self.from_dict(logic, keep_constants=keep_constants, **kwargs)
def __init__(self, finished): self._finished = finished self._bodybuf = StringIO()
def generate_response(self, response): f = StringIO() json.dump(response, f) l = f.tell() f.seek(0) return f, l
def test_unified_build_finder(self): self.create_both('chrome.manifest', 'a\nb\nc\n') self.create_one('a', 'chrome/chrome.manifest', 'a\nb\nc\n') self.create_one('b', 'chrome/chrome.manifest', 'b\nc\na\n') self.create_one( 'a', 'chrome/browser/foo/buildconfig.html', '\n'.join([ '<html>', '<body>', '<h1>about:buildconfig</h1>', '<div>foo</div>', '</body>', '</html>', ])) self.create_one( 'b', 'chrome/browser/foo/buildconfig.html', '\n'.join([ '<html>', '<body>', '<h1>about:buildconfig</h1>', '<div>bar</div>', '</body>', '</html>', ])) finder = UnifiedBuildFinder(FileFinder(self.tmppath('a')), FileFinder(self.tmppath('b'))) self.assertEqual( sorted([(f, c.open().read()) for f, c in finder.find('**/chrome.manifest')]), [('chrome.manifest', 'a\nb\nc\n'), ('chrome/chrome.manifest', 'a\nb\nc\n')]) self.assertEqual( sorted([(f, c.open().read()) for f, c in finder.find('**/buildconfig.html')]), [('chrome/browser/foo/buildconfig.html', '\n'.join([ '<html>', '<body>', '<h1>about:buildconfig</h1>', '<div>foo</div>', '<hr> </hr>', '<div>bar</div>', '</body>', '</html>', ]))]) xpi = MockDest() with JarWriter(fileobj=xpi, compress=True) as jar: jar.add('foo', 'foo') jar.add('bar', 'bar') foo_xpi = xpi.read() self.create_both('foo.xpi', foo_xpi) with JarWriter(fileobj=xpi, compress=True) as jar: jar.add('foo', 'bar') self.create_one('a', 'bar.xpi', foo_xpi) self.create_one('b', 'bar.xpi', xpi.read()) errors.out = StringIO() with self.assertRaises(AccumulatedErrors), errors.accumulate(): self.assertEqual([(f, c.open().read()) for f, c in finder.find('*.xpi')], [('foo.xpi', foo_xpi)]) errors.out = sys.stderr
def __repr__(self): from cStringIO import StringIO from pprint import pprint sio = StringIO() pprint(self._adapterCache, sio) return sio.getvalue()
def __init__(self): self._response_header = StringIO() self._response_body = StringIO()
def main(argv): indelible_model = 'JC' indelible_model = 'LAV0.01a' theta = 0.01 mu = 1 #k = (1,2) k = (1,2,3,4,5) m = 100 n = 5 nr_genes = 10 nr_sims = 1 nr_rows = 3 nr_cols = 3 a_max = 0.74 b_max = 0.74 #a_max = 0.3 #b_max = 0.2 kmer_methods = ['CoalescentJCNJ', 'CoalescentJCLS', 'JCNJ','dstarNJ','concatdJCNJ'] #kmer_methods = ['dstarNJ','concatdJCNJ'] distance_formulas = ['ARS2015', 'alignment_based'] multiple_alignment_method = 'clustalo' alignment_method = 'stretcher' N = theta/mu db_file = 'db.sql' try: opts, args = getopt.getopt(argv,"hk:m:n:o:",["indelible_model=","theta=","genes=","sims=","rows=","cols=","a_max=","b_max="]) except getopt.GetoptError as err: # print usage information and exit: print(str(err)) usage() sys.exit(2) for opt, arg in opts: if opt == '-h': usage() sys.exit() elif opt == "--theta": theta = float(arg) N = theta elif opt == "-k": k = tuple([int(k_i) for k_i in re.sub("[()]","",arg).split(",")]) elif opt == "-m": m = int(arg) elif opt == "-o": db_file = arg elif opt == "-n": n = int(arg) elif opt == "--genes": nr_genes = int(arg) elif opt == "--method": method = arg elif opt == "--distance_formula": distance_formula = arg elif opt == "--sims": nr_sims = int(arg) elif opt == "--rows": nr_rows = int(arg) elif opt == "--cols": nr_cols = int(arg) elif opt == "--a_max": a_max = float(arg) elif opt == "--b_max": b_max = float(arg) elif opt == "--indelible_model": indelible_model = arg elif opt == "--reconstruct_only": reconstruct_only = True usage() import logging sqla_logger = logging.getLogger('sqlalchemy.engine.base.Engine') sqla_logger.propagate = False sqla_logger.addHandler(logging.FileHandler('/tmp/sqla.log')) from sqlalchemy import create_engine engine = create_engine('sqlite:///{:s}'.format(os.path.abspath(db_file)), echo=True, convert_unicode=True) from sqlalchemy.orm import sessionmaker Session = sessionmaker(bind=engine) session = Session() Base.metadata.create_all(engine) import resource sim_set = None print('Simulating sequence data for a {:d}x{:d} Huelsenbeck diagram with {:d} simulations for each tree, and {:d} gene trees per simulation, using the following parameters: theta={:.2e}, m={:d}, indelible_model={:s}\n'.format(nr_rows,nr_cols,nr_sims,nr_genes,theta,m,indelible_model)) sim_set = HuelsenbeckSimulationSet(rows=nr_rows, cols=nr_cols, nr_sims=nr_sims, theta=theta, indelible_model=indelible_model, genes=nr_genes, m=m, n=n, a_max=a_max, b_max=b_max) for method in ['raxml']: tree_estimate_set = HuelsenbeckTreeEstimateSet(simulation_set=sim_set, method=method, distance_formula=None, alignment_method=multiple_alignment_method, k=None) session.add(sim_set) session.add(tree_estimate_set) for row,b in enumerate([(row+1)*(b_max)/nr_rows for row in range(nr_rows)]): for col,a in enumerate([(col+1)*(a_max)/nr_cols for col in range(nr_cols)]): t_a = abs(-3.0/4.0*log(1-4.0/3.0*a)/mu) t_b = abs(-3.0/4.0*log(1-4.0/3.0*b)/mu) tree = huelsenbeck_tree(t_a,t_b,5) tree_newick = ")".join(tree.format('newick').split(")")[:-1])+")" print(tree_newick) xtree = XTree(tree,dict((clade,set([clade.name])) for clade in tree.get_terminals())) #print(','.join([''.join(split[0])+'|'+''.join(split[1]) for split in xtree.get_splits()])+" (t_a,t_b) = ({:f},{:f}): ".format(t_a,t_b)) species_set = sorted(tree.get_terminals(),key=lambda species: species.name) n = len(species_set) species_names = [species.name for species in species_set] genes = [GeneLineage(name='s{:d}'.format(i)) for i,_ in enumerate(range(len(species_set)))] gene_embedding = dict(zip(species_set,[[gene] for gene in genes])) for sim_it in range(nr_sims): # Create simulation objects sim = Simulation(tree=tree_newick, theta=theta, indelible_model=indelible_model, genes=nr_genes, m=m, n=n) session.add(sim) huel_sim = HuelsenbeckSimulation(simulation_set=sim_set, simulation=sim, row=row, col=col) session.add(huel_sim) # Prepare kmer distance matrices to be used to compute averages kmer_distance_matrices = dict() finite_counts_matrices = dict() for distance_formula in ['dstar', 'ARS2015']: kmer_distance_matrices[distance_formula] = dict() finite_counts_matrices[distance_formula] = dict() for k_i in k: kmer_distance_matrices[distance_formula][k_i] = zero_distance_matrix(species_names) finite_counts_matrices[distance_formula][k_i] = zero_distance_matrix(species_names) # Prepare concatenated sequence object sample_ids = [sample.name for sample in itertools.chain.from_iterable(gene_embedding.values())] concatenated_sequences = dict((sample_id,SeqRecord(Seq('',DNAAlphabet()),id=sample_id,name=sample_id,description=sample_id)) for sample_id in sample_ids) for gene in range(nr_genes): # generate gene tree and sequences # for each set of sequence, and for each distance formula and value of k, generate a k-mer distance matrix. sum these matrices for all genes # also store the concatenated sequences coalescent = EmbeddedGeneForest(tree, gene_embedding) coalescent.coalesce(theta) genetree = coalescent.genetree() with TemporaryDirectory() as tmpdir: sequences = mutate_indelible(genetree, m, tmpdir, indelible_model, aligned=False) aligned_sequences = SeqIO.to_dict(align_sequences(sequences, multiple_alignment_method, tmpdir)) for sample_id in sample_ids: concatenated_sequences[sample_id] += aligned_sequences[sample_id] for distance_formula in ['dstar', 'ARS2015']: for k_i in k: if distance_formula == 'ARS2015': dm = kmer_distance_matrix(sequences, k_i, normalized_kmer_distance, grouping=gene_embedding) #print(dm) elif distance_formula == 'dstar': dm = kmer_distance_matrix(sequences, k_i, dstar_kmer_distance, grouping=gene_embedding) else: raise Exception finite_counts_matrices[distance_formula][k_i] += dm.isfinite() kmer_distance_matrices[distance_formula][k_i] += dm.nantozero() #print(kmer_distance_matrices[distance_formula][k_i]) for distance_formula in ['dstar', 'ARS2015']: for k_i in k: #print(finite_counts_matrices[distance_formula][k_i]) avg_dm = kmer_distance_matrices[distance_formula][k_i]/finite_counts_matrices[distance_formula][k_i] #if distance_formula == 'dstar': # print(finite_counts_matrices[distance_formula][k_i]) # print(avg_dm) kdm = kmer_distance_matrix_from_dm(avg_dm, sim, distance_formula, None, k_i) session.add(kdm) jc_dm = kmer_distance_matrix(concatenated_sequences.values(), 1, aligned_kmer_distance, alignment_fn=stretcher_alignment, grouping=gene_embedding) #print(jc_dm) kdm = kmer_distance_matrix_from_dm(jc_dm, sim, 'concatdJC', alignment_method, 1) session.add(kdm) # reconstruct from concatenated sequences using raxml with TemporaryDirectory() as tmpdir: t0 = time.clock() xtreehat = RAxML(concatenated_sequences.values(), gene_embedding, tmpdir) t1 = time.clock() success = int(xtree.displays(xtreehat)) print(success) tree_estimate = TreeEstimate(simulation=sim, method=tree_estimate_set.method, distance_formula=tree_estimate_set.distance_formula, k=tree_estimate_set.k, splits=','.join([''.join(split[0])+'|'+''.join(split[1]) for split in xtreehat.get_splits()]), success=int(xtree.displays(xtreehat)), dt=t1-t0) session.add(tree_estimate) #session.commit() huel_tree_estimate = HuelsenbeckTreeEstimate(tree_estimate_set=tree_estimate_set, tree_estimate=tree_estimate, huelsenbeck_simulation=huel_sim) session.add(huel_tree_estimate) session.commit() # create tree_estimate sets for method in kmer_methods: if method in ['CoalescentJCNJ', 'CoalescentJCLS', 'JCNJ']: distance_formula = 'ARS2015' tree_estimate_set = HuelsenbeckTreeEstimateSet(simulation_set=sim_set, method=method, distance_formula=distance_formula, alignment_method=None, k=",".join([str(k_i) for k_i in k])) elif method == 'dstarNJ': distance_formula = 'dstar' tree_estimate_set = HuelsenbeckTreeEstimateSet(simulation_set=sim_set, method=method, distance_formula=distance_formula, alignment_method=None, k=",".join([str(k_i) for k_i in k])) elif method == 'concatdJCNJ': distance_formula = 'concatdJC' #alignment_method = 'clustalo' tree_estimate_set = HuelsenbeckTreeEstimateSet(simulation_set=sim_set, method=method, distance_formula=distance_formula, alignment_method=alignment_method, k="1") session.add(tree_estimate_set) session.commit() # fetch tree_estimate sets that do not require full sequence data tree_estimate_sets = session.query(HuelsenbeckTreeEstimateSet).\ join(HuelsenbeckTreeEstimateSet.simulation_set).\ filter(HuelsenbeckTreeEstimateSet.method.in_(kmer_methods)). \ filter(HuelsenbeckTreeEstimateSet.simulation_set==sim_set).all() # run tree_estimates for tree_estimate_set in tree_estimate_sets: method = tree_estimate_set.method print(method) distance_formula = tree_estimate_set.distance_formula #alignment_method = tree_estimate_set.alignment_method try: k = [int(k_i) for k_i in tree_estimate_set.k.split(",")] except AttributeError: k = None for huel_sim in tree_estimate_set.simulation_set.huelsenbeck_simulations: sim = huel_sim.simulation treedata = sim.tree handle = StringIO(treedata) #print(handle.read()) tree = Phylo.read(handle, "newick") xtree = XTree(tree,dict((clade,set([clade.name])) for clade in tree.get_terminals())) kmer_distance_matrices = dict((kdm.k,kdm.to_dm()) for kdm in sim.kmer_distance_matrices if kdm.k in k and kdm.distance_formula==distance_formula) t0 = time.clock() if method == 'CoalescentJCLS': xtreehat = TreeMinDistanceFromFiveTaxonCoalescentJCExpectedKmerDistanceParameterizationMap(kmer_distance_matrices) elif method == 'CoalescentJCNJ': xtreehat = NJArgMinSumOfDistancesFromCoalescentJCExpectedKmerPairDistanceParameterizationMap(kmer_distance_matrices) elif method == 'JCNJ': #for k,dm in kmer_distance_matrices.items(): # print dm adjusted_distance_matrices = dict((k,JCKmerDistanceMatrixAdjustment(kmer_distance_matrix,k)) for k,kmer_distance_matrix in kmer_distance_matrices.items()) #for k,dm in adjusted_distance_matrices.items(): # print dm xtreehat = NJ(adjusted_distance_matrices) elif method == 'dstarNJ': #for _,dm in kmer_distance_matrices.items(): # print dm xtreehat = NJ(kmer_distance_matrices) elif method == 'concatdJCNJ': adjusted_distance_matrices = {1:JCKmerDistanceMatrixAdjustment(kmer_distance_matrices[1],1)} xtreehat = NJ(adjusted_distance_matrices) else: raise(Exception) t1 = time.clock() success = int(xtree.displays(xtreehat)) print(','.join([''.join(split[0])+'|'+''.join(split[1]) for split in xtree.get_splits()])+" (t_a,t_b) = ({:f},{:f}): ".format(t_a,t_b)+','.join([''.join(split[0])+'|'+''.join(split[1]) for split in xtreehat.get_splits()])+" ({:d})".format(success)) #print(k) tree_estimate = TreeEstimate(simulation=sim, method=method, distance_formula=distance_formula, k=",".join([str(k_i) for k_i in k]), splits=','.join([''.join(split[0])+'|'+''.join(split[1]) for split in xtreehat.get_splits()]), success=int(xtree.displays(xtreehat)), dt=t1-t0) session.add(tree_estimate) #session.commit() huel_tree_estimate = HuelsenbeckTreeEstimate(tree_estimate_set=tree_estimate_set, tree_estimate=tree_estimate, huelsenbeck_simulation=huel_sim) session.add(huel_tree_estimate) session.commit()
def skip_leading_wsp(f): "Works on a file, returns a file-like object" return StringIO("\n".join(map(string.strip, f.readlines())))
def install_from_urls(self, cr, uid, urls, context=None): OPENERP = 'openerp' tmp = tempfile.mkdtemp() _logger.debug('Install from url: %r', urls) try: # 1. Download & unzip missing modules for module_name, url in urls.items(): if not url: continue # nothing to download, local version is already the last one try: _logger.info('Downloading module `%s` from PengERP Apps', module_name) content = urllib2.urlopen(url).read() except Exception: _logger.exception('Failed to fetch module %s', module_name) raise osv.except_osv( _('Module not found'), _('The `%s` module appears to be unavailable at the moment, please try again later.' ) % module_name) else: zipfile.ZipFile(StringIO(content)).extractall(tmp) assert os.path.isdir(os.path.join(tmp, module_name)) # 2a. Copy/Replace module source in addons path for module_name, url in urls.items(): if module_name == OPENERP or not url: continue # OPENERP is special case, handled below, and no URL means local module module_path = modules.get_module_path(module_name, downloaded=True, display_warning=False) bck = backup(module_path, False) _logger.info('Copy downloaded module `%s` to `%s`', module_name, module_path) shutil.move(os.path.join(tmp, module_name), module_path) if bck: shutil.rmtree(bck) # 2b. Copy/Replace server+base module source if downloaded if urls.get(OPENERP, None): # special case. it contains the server and the base module. # extract path is not the same base_path = os.path.dirname(modules.get_module_path('base')) # copy all modules in the SERVER/openerp/addons directory to the new "openerp" module (except base itself) for d in os.listdir(base_path): if d != 'base' and os.path.isdir(os.path.join( base_path, d)): destdir = os.path.join( tmp, OPENERP, 'addons', d) # XXX 'openerp' subdirectory ? shutil.copytree(os.path.join(base_path, d), destdir) # then replace the server by the new "base" module server_dir = openerp.tools.config[ 'root_path'] # XXX or dirname() bck = backup(server_dir) _logger.info('Copy downloaded module `openerp` to `%s`', server_dir) shutil.move(os.path.join(tmp, OPENERP), server_dir) #if bck: # shutil.rmtree(bck) self.update_list(cr, uid, context=context) with_urls = [m for m, u in urls.items() if u] downloaded_ids = self.search(cr, uid, [('name', 'in', with_urls)], context=context) already_installed = self.search(cr, uid, [('id', 'in', downloaded_ids), ('state', '=', 'installed')], context=context) to_install_ids = self.search(cr, uid, [('name', 'in', urls.keys()), ('state', '=', 'uninstalled')], context=context) post_install_action = self.button_immediate_install( cr, uid, to_install_ids, context=context) if already_installed: # in this case, force server restart to reload python code... cr.commit() openerp.service.restart_server() return { 'type': 'ir.actions.client', 'tag': 'home', 'params': { 'wait': True }, } return post_install_action finally: shutil.rmtree(tmp)
def write_archive(self, treeish, archive, timestamp=None, prefix=''): """Write treeish into an archive If no timestamp is provided and 'treeish' is a commit, its committer timestamp will be used. Otherwise the current time will be used. All path names in the archive are added to 'prefix', which defaults to an empty string. Arguments: treeish The treeish to write. archive An archive from the 'tarfile' module timestamp Timestamp to use for the files in the archive. prefix Extra prefix to add to the path names in the archive. Example:: >>> import tarfile, pygit2 >>>> with tarfile.open('foo.tar', 'w') as archive: >>>> repo = pygit2.Repsitory('.') >>>> repo.write_archive(archive, repo.head.target) """ # Try to get a tree form whatever we got if isinstance(treeish, Tree): tree = treeish if isinstance(treeish, Oid) or is_string(treeish): treeish = self[treeish] # if we don't have a timestamp, try to get it from a commit if not timestamp: try: commit = treeish.peel(Commit) timestamp = commit.committer.time except Exception: pass # as a last resort, use the current timestamp if not timestamp: timestamp = int(time()) tree = treeish.peel(Tree) index = Index() index.read_tree(tree) for entry in index: content = self[entry.id].read_raw() info = tarfile.TarInfo(prefix + entry.path) info.size = len(content) info.mtime = timestamp info.uname = info.gname = 'root' # just because git does this if entry.mode == GIT_FILEMODE_LINK: info.type = archive.SYMTYPE info.linkname = content info.mode = 0o777 # symlinks get placeholder info.size = 0 archive.addfile(info) else: archive.addfile(info, StringIO(content))
def parseSPA4SymbolProfit(modelType="fixed"): ''' csv to latex format ''' if modelType == "fixed": n_rvs = range(5, 55, 5) hist_periods = range(50, 130, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7", "0.75", "0.8", "0.85", "0.9", "0.95") resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Fixed_eachParam_Profit.csv") elif modelType == "dynamic": n_rvs = range(5, 55, 5) hist_periods = range(90, 120 + 10, 10) alphas = ("0.5", "0.55", "0.6", "0.65", "0.7") resFile = os.path.join(ExpResultsDir, "SPA", "SPA_Dynamic_eachParam_Profit.csv") #read resfile SPA = {} reader = csv.reader(open(resFile)) reader.next() for row in reader: print row key = "%s-%s-%s" % (row[0], row[1], row[2]) pvalue = float(row[7]) if key not in SPA.keys(): SPA[key] = pvalue elif pvalue > SPA[key]: SPA[key] = pvalue if modelType == "fixed": outFile = os.path.join(ExpResultsDir, "SPA", "SPA_Fixed_Latex_Profit.txt") elif modelType == "dynamic": outFile = os.path.join(ExpResultsDir, "SPA", "SPA_Dynamic_Latex_Profit.txt") for n_rv in n_rvs: if modelType == "fixed": tableHead = r''' \begin{table} \fontsize{8pt}{8pt}\selectfont \caption{The SPA test $p$-value of our SP model with portfolio size $n=%s$.} \begin{center} \begin{tabular}{| r | r |r |r |r |r |r |r| r | } \hline ''' % (n_rv) tableTail = r'''\end{tabular} \end{center} \label{tab:SPA_SP_n%s} \end{table} ''' % (n_rv) elif modelType == "dynamic": tableHead = r''' \begin{table} \fontsize{8pt}{8pt}\selectfont \caption{The SPA test $p$-value of our SIP model with portfolio size $n=%s$.} \begin{center} \begin{tabular}{| r | r |r |r |r |} \hline ''' % (n_rv) tableTail = r'''\end{tabular} \end{center} \label{tab:SPA_SIP_n%s} \end{table} ''' % (n_rv) statIO = StringIO() with open(outFile, 'ab') as fout: fout.write(tableHead) fout.write(r'$\alpha$ & $h=%4s$ & ' % (hist_periods[0])) fout.write(r' & '.join(str(p) for p in hist_periods[1:])) fout.write('\\\ \hline \n') #load exp ROI for alpha in alphas: statIO.write('%4s &' % (alpha)) for hdx, period in enumerate(hist_periods): if n_rv == 50 and period == 50: statIO.write('- &') continue key = "%s-%s-%s" % (n_rv, period, alpha) if SPA[key] <= 0.01: statIO.write('***%4.4f ' % (SPA[key])) elif SPA[key] <= 0.05: statIO.write('**%4.4f ' % (SPA[key])) elif SPA[key] <= 0.1: statIO.write('*%4.4f ' % (SPA[key])) else: statIO.write('%4.4f ' % (SPA[key])) if hdx != len(hist_periods) - 1: statIO.write(' & ') else: statIO.write('\\\ \hline \n') with open(outFile, 'ab') as fout: fout.write(statIO.getvalue()) fout.write(tableTail) statIO.close()
"{0}: write_var,index 0".format(n)) p.write_var(all_vars, self.tsvalues[:], self.tsvar, index=1) self.assertTrue( (all_vars[self.tsvar][1] == self.tsvalues[0]).all(), "{0}: write_var,index 1".format(n)) def test_close(self): for p in self.io_ports.itervalues(): f = p.open_file(self.slice) f = p.close_file(f) #=============================================================================== # Command-Line Operation #=============================================================================== if __name__ == "__main__": hline = '=' * 70 print hline print 'STANDARD OUTPUT FROM ALL TESTS:' print hline mystream = StringIO() tests = unittest.TestLoader().loadTestsFromTestCase(climIOTests) unittest.TextTestRunner(stream=mystream).run(tests) print hline print 'TESTS RESULTS:' print hline print str(mystream.getvalue())
def __enter__(self): self._stdout = sys.stdout sys.stdout = self._stringio = StringIO() return self
def process_message(self, peer, mailfrom, rcpttos, data): from cStringIO import StringIO from Mailman import Utils from Mailman import Message from Mailman import MailList # If the message is to a Mailman mailing list, then we'll invoke the # Mailman script directly, without going through the real smtpd. # Otherwise we'll forward it to the local proxy for disposition. listnames = [] for rcpt in rcpttos: local = rcpt.lower().split('@')[0] # We allow the following variations on the theme # listname # listname-admin # listname-owner # listname-request # listname-join # listname-leave parts = local.split('-') if len(parts) > 2: continue listname = parts[0] if len(parts) == 2: command = parts[1] else: command = '' if not Utils.list_exists(listname) or command not in ( '', 'admin', 'owner', 'request', 'join', 'leave'): continue listnames.append((rcpt, listname, command)) # Remove all list recipients from rcpttos and forward what we're not # going to take care of ourselves. Linear removal should be fine # since we don't expect a large number of recipients. for rcpt, listname, command in listnames: rcpttos.remove(rcpt) # If there's any non-list destined recipients left, print >> DEBUGSTREAM, 'forwarding recips:', ' '.join(rcpttos) if rcpttos: refused = self._deliver(mailfrom, rcpttos, data) # TBD: what to do with refused addresses? print >> DEBUGSTREAM, 'we got refusals:', refused # Now deliver directly to the list commands mlists = {} s = StringIO(data) msg = Message.Message(s) # These headers are required for the proper execution of Mailman. All # MTAs in existance seem to add these if the original message doesn't # have them. if not msg.getheader('from'): msg['From'] = mailfrom if not msg.getheader('date'): msg['Date'] = time.ctime(time.time()) for rcpt, listname, command in listnames: print >> DEBUGSTREAM, 'sending message to', rcpt mlist = mlists.get(listname) if not mlist: mlist = MailList.MailList(listname, lock=0) mlists[listname] = mlist # dispatch on the type of command if command == '': # post msg.Enqueue(mlist, tolist=1) elif command == 'admin': msg.Enqueue(mlist, toadmin=1) elif command == 'owner': msg.Enqueue(mlist, toowner=1) elif command == 'request': msg.Enqueue(mlist, torequest=1) elif command in ('join', 'leave'): # TBD: this is a hack! if command == 'join': msg['Subject'] = 'subscribe' else: msg['Subject'] = 'unsubscribe' msg.Enqueue(mlist, torequest=1)
def read_info(name): data = get_data(__package__, "info/{name}.conf".format(name=name)) parser = SafeConfigParser(allow_no_value=True) parser.readfp(StringIO(data)) return parser
puncts = set([c for c in string.punctuation]) sentences = [] for line in fin: # each sentence is a list of words, we lowercase and remove punctuations # same as the Scala code sentences.append([w for w in nltk.word_tokenize(line.strip().lower()) if w not in puncts]) fin.close() # train word2vec with sentences model = word2vec.Word2Vec(sentences, size=100, window=4, min_count=1, workers=4) model.init_sims(replace=True) # find 10 words closest to "day" print "words most similar to 'day':" print model.most_similar(positive=["day"], topn=10) # find closest word to "he" print "words most similar to 'he':" print model.most_similar(positive=["he"], topn=1) # for each word in the vocabulary, write out the word vectors to a file fvec = open("/tmp/word_vectors.txt", 'wb') for word in model.vocab.keys(): vec = model[word] for i in range(vec.shape[0]): s = StringIO() np.savetxt(s, vec, fmt="%.5f", newline=",") fvec.write("%s%s\n" % (s.getvalue(), word)) fvec.close()
def send(cls, r, resource): """ Method to retrieve updates for a subscription, render the notification message and send it - responds to POST?format=msg requests to the respective resource. @param r: the S3Request @param resource: the S3Resource """ _debug = current.log.debug _debug("S3Notifications.send()") json_message = current.xml.json_message # Read subscription data source = r.body source.seek(0) data = source.read() subscription = json.loads(data) #_debug("Notify PE #%s by %s on %s of %s since %s" % \ # (subscription["pe_id"], # str(subscription["method"]), # str(subscription["notify_on"]), # subscription["resource"], # subscription["last_check_time"], # )) # Check notification settings notify_on = subscription["notify_on"] methods = subscription["method"] if not notify_on or not methods: return json_message(message="No notifications configured " "for this subscription") # Authorization (pe_id must not be None) pe_id = subscription["pe_id"] if not pe_id: r.unauthorised() # Fields to extract fields = resource.list_fields(key="notify_fields") if "created_on" not in fields: fields.append("created_on") # Extract the data data = resource.select(fields, represent=True, raw_data=True) rows = data["rows"] # How many records do we have? numrows = len(rows) if not numrows: return json_message(message="No records found") #_debug("%s rows:" % numrows) # Prepare meta-data get_config = resource.get_config settings = current.deployment_settings page_url = subscription["page_url"] crud_strings = current.response.s3.crud_strings.get(resource.tablename) if crud_strings: resource_name = crud_strings.title_list else: resource_name = string.capwords(resource.name, "_") last_check_time = s3_decode_iso_datetime( subscription["last_check_time"]) email_format = subscription["email_format"] if not email_format: email_format = settings.get_msg_notify_email_format() filter_query = subscription.get("filter_query") meta_data = { "systemname": settings.get_system_name(), "systemname_short": settings.get_system_name_short(), "resource": resource_name, "page_url": page_url, "notify_on": notify_on, "last_check_time": last_check_time, "filter_query": filter_query, "total_rows": numrows, } # Render contents for the message template(s) renderer = get_config("notify_renderer") if not renderer: renderer = settings.get_msg_notify_renderer() if not renderer: renderer = cls._render contents = {} if email_format == "html" and "EMAIL" in methods: contents["html"] = renderer(resource, data, meta_data, "html") contents["default"] = contents["html"] if email_format != "html" or "EMAIL" not in methods or len( methods) > 1: contents["text"] = renderer(resource, data, meta_data, "text") contents["default"] = contents["text"] # Subject line subject = get_config("notify_subject") if not subject: subject = settings.get_msg_notify_subject() if callable(subject): subject = subject(resource, data, meta_data) from string import Template subject = Template(subject).safe_substitute(S="%(systemname)s", s="%(systemname_short)s", r="%(resource)s") subject = subject % meta_data # Attachment attachment = subscription.get("attachment", False) document_ids = None if attachment: attachment_fnc = settings.get_msg_notify_attachment() if attachment_fnc: document_ids = attachment_fnc(resource, data, meta_data) # Helper function to find templates from a priority list join = lambda *f: os.path.join(current.request.folder, *f) def get_template(path, filenames): for fn in filenames: filepath = join(path, fn) if os.path.exists(filepath): try: return open(filepath, "rb") except: pass return None # Render and send the message(s) themes = settings.get_template() prefix = resource.get_config("notify_template", "notify") send = current.msg.send_by_pe_id success = False errors = [] for method in methods: error = None # Get the message template template = None filenames = ["%s_%s.html" % (prefix, method.lower())] if method == "EMAIL" and email_format: filenames.insert(0, "%s_email_%s.html" % (prefix, email_format)) if themes != "default": location = settings.get_template_location() if not isinstance(themes, (tuple, list)): themes = (themes, ) for theme in themes[::-1]: path = join(location, "templates", theme, "views", "msg") template = get_template(path, filenames) if template is not None: break if template is None: path = join("views", "msg") template = get_template(path, filenames) if template is None: template = StringIO( s3_str(current.T("New updates are available."))) # Select contents format if method == "EMAIL" and email_format == "html": output = contents["html"] else: output = contents["text"] # Render the message try: message = current.response.render(template, output) except: exc_info = sys.exc_info()[:2] error = ("%s: %s" % (exc_info[0].__name__, exc_info[1])) errors.append(error) continue if not message: continue # Send the message #_debug("Sending message per %s" % method) #_debug(message) try: sent = send(pe_id, subject=s3_truncate(subject, 78), message=message, contact_method=method, system_generated=True, document_ids=document_ids) except: exc_info = sys.exc_info()[:2] error = ("%s: %s" % (exc_info[0].__name__, exc_info[1])) sent = False if sent: # Successful if at least one notification went out success = True else: if not error: error = current.session.error if isinstance(error, list): error = "/".join(error) if error: errors.append(error) # Done if errors: message = ", ".join(errors) else: message = "Success" return json_message(success=success, statuscode=200 if success else 403, message=message)
gains = CalcGainsFromFps(suppl, fps, topN=details.topN, actName=details.actCol, nActs=details.nActs, biasList=details.biasList) t2 = time.time() message("\tThat took %.2f seconds.\n" % (t2 - t1)) if details.gainsName: outF = open(details.gainsName, 'w+') OutputGainsData(outF, gains, cat, nActs=details.nActs) else: if details.gainsName: inF = open(details.gainsName, 'r') gains = ProcessGainsData(inF) if details.doDetails: if not cat: message("We require a catalog to get details\n") sys.exit(-2) if not gains: message("We require gains data to get details\n") sys.exit(-2) io = StringIO() io.write('id,SMILES,gain\n') ShowDetails(cat, gains, nToDo=details.nBits, outF=io) if details.detailsName: open(details.detailsName, 'w+').write(io.getvalue()) else: sys.stderr.write(io.getvalue())