def dailyForexNews(): startContext = DailyFxSpiderUtils.retrunStartContext( 'http://cdn.dailyfx.com.hk/livenews/index.html') length = len(DailyFxSpiderUtils.findAllTarget(startContext)) currentList = [] for i in range(length): dict = DailyFxSpiderUtils.divisionTarget( startContext, '<tr class="record" valign="top">', '</tr>') startContext = dict['nextContext'] #Save TargetContext sectionDict = dict['targetContext'] #print sectionDict(the final result) currentDict = filterFinalTarget(sectionDict) keyid = str(uuid.uuid1()) if currentDict['descriptdetails'] != '': currentList.append([ keyid, currentDict['titletime'], currentDict['descriptcontext'], currentDict['descriptdetails'] ]) else: currentList.append([ keyid, currentDict['titletime'], currentDict['descriptcontext'], '' ]) return currentList
def dailyForexNews(): startContext = DailyFxSpiderUtils.retrunStartContext('http://cdn.dailyfx.com.hk/livenews/index.html') length = len(DailyFxSpiderUtils.findAllTarget(startContext)) currentList = [] for i in range(length): dict = DailyFxSpiderUtils.divisionTarget(startContext,'<tr class="record" valign="top">', '</tr>') startContext = dict['nextContext'] #Save TargetContext sectionDict = dict['targetContext'] #print sectionDict(the final result) currentDict = filterFinalTarget(sectionDict) keyid = str(uuid.uuid1()) if currentDict['descriptdetails'] !='': currentList.append([keyid,currentDict['titletime'],currentDict['descriptcontext'],currentDict['descriptdetails']]) else: currentList.append([keyid,currentDict['titletime'],currentDict['descriptcontext'],'']) return currentList
def writeDailyForexNews(): currentList = dailyForexNews() conn = DailyFxSpiderUtils.getMySQLConn() cursor = conn.cursor() try: cursor.execute("DELETE FROM DAILY_FOREX_NEWS_RESOURCE_TABLE") conn.commit() except conn.Error,e: print "Mysql Error %d: %s" % (e.args[0], e.args[1]) conn.rollback()
def writeDailyForexNews(): currentList = dailyForexNews() conn = DailyFxSpiderUtils.getMySQLConn() cursor = conn.cursor() try: cursor.execute("DELETE FROM DAILY_FOREX_NEWS_RESOURCE_TABLE") conn.commit() except conn.Error, e: print "Mysql Error %d: %s" % (e.args[0], e.args[1]) conn.rollback()
def filterFinalTarget(sectionDict): if DailyFxSpiderUtils.filterTargetFlag(sectionDict, '<td style="background-color:#FFFFFF" width="100">'): titleTimeSubSet = DailyFxSpiderUtils.divisionTarget(sectionDict, '<td style="background-color:#FFFFFF" width="100">', '</td>') descriptSubSet = DailyFxSpiderUtils.divisionTarget(titleTimeSubSet['nextContext'],'<td style="background-color:#FFFFFF" width="550" class="stitle">','</td>') titletime = DailyFxSpiderUtils.filterSubSetContext(titleTimeSubSet['targetContext'], '<td style="background-color:#FFFFFF" width="100">','</td>') descriptcontext = DailyFxSpiderUtils.filterSubSetContext(descriptSubSet['targetContext'],'<td style="background-color:#FFFFFF" width="550" class="stitle">','</td>') descriptdetails ='' if DailyFxSpiderUtils.filterTargetFlag(descriptcontext,'class="openDiv">'): currentdescriptcontext = DailyFxSpiderUtils.filterSubSetContext(descriptcontext, 'class="openDiv">', '</a>') descriptdetails = DailyFxSpiderUtils.filterSubSetContext(descriptcontext,'style="display:none;">','</div>') descriptcontext = currentdescriptcontext return {'titletime':titletime,'descriptcontext':descriptcontext,'descriptdetails':descriptdetails} elif DailyFxSpiderUtils.filterTargetFlag(sectionDict, '<td style="background-color:#F2F2F2" width="100">'): titleTimeSubSet = DailyFxSpiderUtils.divisionTarget(sectionDict, '<td style="background-color:#F2F2F2" width="100">','</td>') titletime = DailyFxSpiderUtils.filterSubSetContext(titleTimeSubSet['targetContext'],'<td style="background-color:#F2F2F2" width="100">','</td>') descriptSubSet = DailyFxSpiderUtils.divisionTarget(titleTimeSubSet['nextContext'],'<td style="background-color:#F2F2F2" width="550" class="stitle">','</td>') descriptcontext = DailyFxSpiderUtils.filterSubSetContext(descriptSubSet['targetContext'],'<td style="background-color:#F2F2F2" width="550" class="stitle">','</td>') descriptdetails ='' if DailyFxSpiderUtils.filterTargetFlag(descriptcontext,'class="openDiv">'): currentdescriptcontext = DailyFxSpiderUtils.filterSubSetContext(descriptcontext, 'class="openDiv">', '</a>') descriptdetails = DailyFxSpiderUtils.filterSubSetContext(descriptcontext,'style="display:none;">','</div>') descriptcontext = currentdescriptcontext return {'titletime':titletime,'descriptcontext':descriptcontext,'descriptdetails':descriptdetails}
def filterFinalTarget(sectionDict): if DailyFxSpiderUtils.filterTargetFlag( sectionDict, '<td style="background-color:#FFFFFF" width="100">'): titleTimeSubSet = DailyFxSpiderUtils.divisionTarget( sectionDict, '<td style="background-color:#FFFFFF" width="100">', '</td>') descriptSubSet = DailyFxSpiderUtils.divisionTarget( titleTimeSubSet['nextContext'], '<td style="background-color:#FFFFFF" width="550" class="stitle">', '</td>') titletime = DailyFxSpiderUtils.filterSubSetContext( titleTimeSubSet['targetContext'], '<td style="background-color:#FFFFFF" width="100">', '</td>') descriptcontext = DailyFxSpiderUtils.filterSubSetContext( descriptSubSet['targetContext'], '<td style="background-color:#FFFFFF" width="550" class="stitle">', '</td>') descriptdetails = '' if DailyFxSpiderUtils.filterTargetFlag(descriptcontext, 'class="openDiv">'): currentdescriptcontext = DailyFxSpiderUtils.filterSubSetContext( descriptcontext, 'class="openDiv">', '</a>') descriptdetails = DailyFxSpiderUtils.filterSubSetContext( descriptcontext, 'style="display:none;">', '</div>') descriptcontext = currentdescriptcontext return { 'titletime': titletime, 'descriptcontext': descriptcontext, 'descriptdetails': descriptdetails } elif DailyFxSpiderUtils.filterTargetFlag( sectionDict, '<td style="background-color:#F2F2F2" width="100">'): titleTimeSubSet = DailyFxSpiderUtils.divisionTarget( sectionDict, '<td style="background-color:#F2F2F2" width="100">', '</td>') titletime = DailyFxSpiderUtils.filterSubSetContext( titleTimeSubSet['targetContext'], '<td style="background-color:#F2F2F2" width="100">', '</td>') descriptSubSet = DailyFxSpiderUtils.divisionTarget( titleTimeSubSet['nextContext'], '<td style="background-color:#F2F2F2" width="550" class="stitle">', '</td>') descriptcontext = DailyFxSpiderUtils.filterSubSetContext( descriptSubSet['targetContext'], '<td style="background-color:#F2F2F2" width="550" class="stitle">', '</td>') descriptdetails = '' if DailyFxSpiderUtils.filterTargetFlag(descriptcontext, 'class="openDiv">'): currentdescriptcontext = DailyFxSpiderUtils.filterSubSetContext( descriptcontext, 'class="openDiv">', '</a>') descriptdetails = DailyFxSpiderUtils.filterSubSetContext( descriptcontext, 'style="display:none;">', '</div>') descriptcontext = currentdescriptcontext return { 'titletime': titletime, 'descriptcontext': descriptcontext, 'descriptdetails': descriptdetails }