Esempio n. 1
0
    def downloadData(in_queue):
        out_queue = queue.Queue()
        page = 1
        totalpage = in_queue.qsize()
        #start multi-thread fetching
        threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
                                           threadName)
        globals.data[threadName + '_threadlist'] = threadList

        #page downloading
        downloadErrorCnt = 0
        while (globals.data[threadName + '_running'] and (page <= totalpage)):
            try:
                err, u, content = out_queue.get(False)
                if (err):
                    utils.output(err, 'red')
                    downloadErrorCnt += 1
                    continue

                end_time = datetime.datetime.now()
                diffStr = '{}'.format(end_time - start_time)
                tipstr = '{} 下载'.format(threadName)
                outstr = '{}({}线程) 第{}/{}页 ({})'.format(
                    tipstr, threadCount, page, totalpage, diffStr)
                utils.overwrite(outstr, tipstr)

                page = page + 1

            except queue.Empty:
                time.sleep(0.01)
                QCoreApplication.processEvents()

        return downloadErrorCnt
Esempio n. 2
0
    def downloadData(in_queue):
        out_queue = queue.Queue()
        page = 1
        totalpage = in_queue.qsize()
        #start multi-thread fetching
        threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)
        globals.data[threadName+'_threadlist'] = threadList        

        #page downloading
        downloadErrorCnt = 0
        while(globals.data[threadName+'_running'] and (page <= totalpage)):
            try:
                err, u, content = out_queue.get(False)
                if(err):
                    utils.output(err, 'red')
                    downloadErrorCnt +=1
                    continue
                
                end_time = datetime.datetime.now()
                diffStr = '{}'.format(end_time - start_time)
                tipstr = '{} 下载'.format(threadName)
                outstr = '{}({}线程) 第{}/{}页 ({})'.format(tipstr, threadCount, page, totalpage, diffStr)
                utils.overwrite(outstr, tipstr)

                page = page +1

            except queue.Empty:
                time.sleep(0.01)
                QCoreApplication.processEvents()

        return downloadErrorCnt
Esempio n. 3
0
def cancelDownload(threadName):

        def stopThread(threadName):
            threadList = []
            try:
                threadList = globals.data[threadName+'_threadlist']
                globals.data[threadName+'_threadlist'] = None
                globals.data[threadName+'_running'] = False
            except:
                pass

            if(threadList):
                for t in threadList:
                    t.stop()


        threadLeft = 0
        stopThread(threadName)
        for t in threading.enumerate():
            if(t.getName() == threadName):
                threadLeft +=1

        if(threadLeft > 0):
            tipstr = '终止下载中, 目前剩余'
            outstr = '{} {} 线程'.format(tipstr, threadLeft)
            utils.overwrite(outstr, tipstr)
            return False
        else:
            utils.output('所有下载线程已终止')
        return True
Esempio n. 4
0
def run(subcmd, params):
    chart = globals.mainwin.chart
    stocklist = stock.getStockList()

    if(subcmd == 'di'):
        filterResults = []
        di = indicators.di.Di()

        utils.output('开始di条件过滤')
        prefixstr = 'filter di条件过滤:'
        totalstock = len(stocklist)
        cnt = 0
        start_time = datetime.datetime.now()
        for stockcode in stocklist:
            cnt +=1
            stockcode = stock.normalizeStockCode(stockcode)
            end_time = datetime.datetime.now()
            diffStr = '{}'.format(end_time - start_time)
            outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(prefixstr, cnt, totalstock, stockcode, len(filterResults), diffStr)
            utils.overwrite(outstr, prefixstr)
            stockdata = stock.getStockDayDataQianFuQuan(stockcode)
            stockdata = stockdata[-500:]
            didata = di.calculateData(stockdata)
            QCoreApplication.processEvents()
            if(didata[-1] and didata[-1].get('tip')):
                filterResults.append({'code':stockcode, 'date':didata[-1]['date']})

        
        dataArr = []

        for row in filterResults:
            scode = row['code']
            sdate = row['date']
            row=[]
            row.append(scode)
            row.append(stock.getStockName(scode))
            row.append(sdate)
            dataArr.append(row)
    
        #write to file begin
        filename = globals.datapath + '/lastFilterResult.dat'
        try:
            f = open(filename, 'wb')
        except IOError:
            exc_type, exc_value = sys.exc_info()[:2]
            errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
            output(errmsg, 'red')
            return False
        
        pickle.dump(dataArr, f)
        f.close() 
        #write to file end
        
        tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
        tbResults = globals.mainwin.stockfilterresultdlg.tbResults
        tbResults.setModel(tablemodel)
        globals.mainwin.windowStockFilterResult()
Esempio n. 5
0
def run(subcmd, params):
    chart = globals.mainwin.chart
    stocklist = stock.getStockList()

    if (subcmd == 'di'):
        filterResults = []
        di = indicators.di.Di()

        utils.output('开始di条件过滤')
        prefixstr = 'filter di条件过滤:'
        totalstock = len(stocklist)
        cnt = 0
        start_time = datetime.datetime.now()
        for stockcode in stocklist:
            cnt += 1
            stockcode = stock.normalizeStockCode(stockcode)
            end_time = datetime.datetime.now()
            diffStr = '{}'.format(end_time - start_time)
            outstr = '{} {}/{} 当前代码:{} 符合条件:{} ({})'.format(
                prefixstr, cnt, totalstock, stockcode, len(filterResults),
                diffStr)
            utils.overwrite(outstr, prefixstr)
            stockdata = stock.getStockDayDataQianFuQuan(stockcode)
            stockdata = stockdata[-500:]
            didata = di.calculateData(stockdata)
            QCoreApplication.processEvents()
            if (didata[-1] and didata[-1].get('tip')):
                filterResults.append({
                    'code': stockcode,
                    'date': didata[-1]['date']
                })

        dataArr = []

        for row in filterResults:
            scode = row['code']
            sdate = row['date']
            row = []
            row.append(scode)
            row.append(stock.getStockName(scode))
            row.append(sdate)
            dataArr.append(row)

        #write to file begin
        filename = globals.datapath + '/lastFilterResult.dat'
        try:
            f = open(filename, 'wb')
        except IOError:
            exc_type, exc_value = sys.exc_info()[:2]
            errmsg = '{}: {}'.format(exc_type.__name__, exc_value)
            output(errmsg, 'red')
            return False

        pickle.dump(dataArr, f)
        f.close()
        #write to file end

        tablemodel = stockfilterresultdlg.FilterResultModel(dataArr, None)
        tbResults = globals.mainwin.stockfilterresultdlg.tbResults
        tbResults.setModel(tablemodel)
        globals.mainwin.windowStockFilterResult()
Esempio n. 6
0
def _updateStockList(): 
    
    threadName = 'update_stocklist'
    threadCount = 10
    start_time = datetime.datetime.now()
    
    #terminal all thread from threadList
    def stopThread(threadList):
        for t in threadList:
            t.stop()

    partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
    page = 1

    url = '{}{}'.format(partUrl, ((page-1)*50+1))
    err, u, content = utils.getWebContent(url)
    if(err):
        utils.output(err, 'red')
        return False

    #get basic info
    content = content.decode('gbk')
    totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
    if(len(totalpagelist)<1):
        utils.output('取得总页数(totalpage)出错', 'red')
        return False
        
    totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content)
    if(len(totalstocklist)<1):
        err = '取得总股票数(totalstock)出错'
        utils.output(err, 'red')
        return False

    totalpage = int(totalpagelist[0])
    totalstock = int(totalstocklist[0])
    utils.output('共{}页, {}只股票'.format(totalpage, totalstock))
    
    #get real stock list data
    in_queue = queue.Queue()
    out_queue = queue.Queue()
    for i in range(1, totalpage+1):
        url = '{}{}'.format(partUrl, ((i-1)*50+1))
        in_queue.put(url)
    
    #start multi-thread fetching
    threadList = utils.getWebContentMT(in_queue, out_queue, threadCount, threadName)

    pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
    myre = re.compile(pattern, re.DOTALL)

    stocklist = []
    tipstr = 'update stocklist 下载'
    #page downloading
    while(page <= totalpage):
        try:
            err, u, content = out_queue.get(False)
            if(err):
                utils.output(err, 'red')
                stopThread(threadList)
                return False
            
            outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page, totalpage)
            if(utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
                utils.output(outstr)
            else:
                utils.overwrite(outstr)

            page = page +1

            text = content.decode('gbk')
            #find stock list
            results = myre.findall(text)
            if(not results):
                err = '无法提取到股票列表数据'
                utils.output(err, 'red')
                stopThread(threadList)
                return False

            for r in results:
                lst = list(r)
                line = '{},{}'.format(lst[3], lst[6])
                stocklist.append(line)

        except queue.Empty:
            time.sleep(0.01)
            QCoreApplication.processEvents()

    #download finish, stock list ready
    if(len(stocklist) != totalstock):
        err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist), totalstock)
        utils.output(err, 'red')
        return False

    stocklist.sort()
    utils.output('获取股票列表完成: {}只'.format(len(stocklist)))
    
    #save data
    filename = globals.datapath + '/stocklist.txt'
    data = "\r\n".join(stocklist)
    if(not utils.file_put_contents(filename, data)):
        utils.output('数据保存失败', 'red')
        return False

    utils.output('数据保存成功')

    end_time = datetime.datetime.now()
    return '{}'.format(end_time - start_time)
Esempio n. 7
0
def _updateStockList():

    threadName = 'update_stocklist'
    threadCount = 10
    start_time = datetime.datetime.now()

    #terminal all thread from threadList
    def stopThread(threadList):
        for t in threadList:
            t.stop()

    partUrl = 'http://static.sse.com.cn/sseportal/webapp/datapresent/SSEQueryStockInfoAct?reportName=BizCompStockInfoRpt&PRODUCTID=&PRODUCTJP=&PRODUCTNAME=&keyword=&tab_flg=&CURSOR='
    page = 1

    url = '{}{}'.format(partUrl, ((page - 1) * 50 + 1))
    err, u, content = utils.getWebContent(url)
    if (err):
        utils.output(err, 'red')
        return False

    #get basic info
    content = content.decode('gbk')
    totalpagelist = re.findall(r'共<strong>([0-9]+)</strong>页', content)
    if (len(totalpagelist) < 1):
        utils.output('取得总页数(totalpage)出错', 'red')
        return False

    totalstocklist = re.findall(r'第1条到第50条,共([0-9]+)条', content)
    if (len(totalstocklist) < 1):
        err = '取得总股票数(totalstock)出错'
        utils.output(err, 'red')
        return False

    totalpage = int(totalpagelist[0])
    totalstock = int(totalstocklist[0])
    utils.output('共{}页, {}只股票'.format(totalpage, totalstock))

    #get real stock list data
    in_queue = queue.Queue()
    out_queue = queue.Queue()
    for i in range(1, totalpage + 1):
        url = '{}{}'.format(partUrl, ((i - 1) * 50 + 1))
        in_queue.put(url)

    #start multi-thread fetching
    threadList = utils.getWebContentMT(in_queue, out_queue, threadCount,
                                       threadName)

    pattern = '<tr>(.*?)<td class="table3"(.*?)<a href=(.*?)>([0-9]+)</a></td>(.*?)<td class="table3" bgcolor="(.*?)">(.*?)</td>(.*?)</tr>'
    myre = re.compile(pattern, re.DOTALL)

    stocklist = []
    tipstr = 'update stocklist 下载'
    #page downloading
    while (page <= totalpage):
        try:
            err, u, content = out_queue.get(False)
            if (err):
                utils.output(err, 'red')
                stopThread(threadList)
                return False

            outstr = '{}({}线程) 第{}/{}页'.format(tipstr, threadCount, page,
                                               totalpage)
            if (utils.getLastCmdeditLine()[0:len(tipstr)] != tipstr):
                utils.output(outstr)
            else:
                utils.overwrite(outstr)

            page = page + 1

            text = content.decode('gbk')
            #find stock list
            results = myre.findall(text)
            if (not results):
                err = '无法提取到股票列表数据'
                utils.output(err, 'red')
                stopThread(threadList)
                return False

            for r in results:
                lst = list(r)
                line = '{},{}'.format(lst[3], lst[6])
                stocklist.append(line)

        except queue.Empty:
            time.sleep(0.01)
            QCoreApplication.processEvents()

    #download finish, stock list ready
    if (len(stocklist) != totalstock):
        err = '取得的股票列表数目(stocklist)出错 {}!={}'.format(len(stocklist),
                                                     totalstock)
        utils.output(err, 'red')
        return False

    stocklist.sort()
    utils.output('获取股票列表完成: {}只'.format(len(stocklist)))

    #save data
    filename = globals.datapath + '/stocklist.txt'
    data = "\r\n".join(stocklist)
    if (not utils.file_put_contents(filename, data)):
        utils.output('数据保存失败', 'red')
        return False

    utils.output('数据保存成功')

    end_time = datetime.datetime.now()
    return '{}'.format(end_time - start_time)