Ejemplo n.º 1
0
    def getdata(self,quote,datedebut=None,datefin=None):
        if not datefin:
            datefin = date.today()

        if not datedebut:
            datedebut = date.today()
        if isinstance(datedebut,Datation):
            datedebut = datedebut.date()

        if isinstance(datefin,Datation):
            datefin = datefin.date()

        d1 = self.parseDate(datedebut)
        d2 = self.parseDate(datefin)

        debug("Import_yahoo:getdata quote:%s begin:%s end:%s" % (quote,d1,d2))

        sname = yahooTicker(quote.ticker(),quote.market(),quote.place())

        if sname[0]=='^':
            ss = "%5E" + sname[1:]
        else:
            ss = sname
        query = (
            ('s', ss),
            ('a', '%02d' % (int(d1[1])-1)),
            ('b', d1[2]),
            ('c', d1[0]),
            ('d', '%02d' % (int(d2[1])-1)),
            ('e', d2[2]),
            ('f', d2[0]),
            ('y', '0'),
            ('g', 'd'),
            ('ignore', '.csv'),
        )
        query = map(lambda (var, val): '%s=%s' % (var, str(val)), query)
        query = string.join(query, '&')
        url = yahooUrl(quote.market(),live=False) + '?' + query

        debug("Import_yahoo:getdata: url=%s ",url)
        try:
            buf=self.m_connection.getDataFromUrl(url)
        except:
            debug('Import_yahoo:unable to connect :-(')
            return None

        # pull data
        lines = self.splitLines(buf)
        if len(lines)<=0:
            # empty content
            return None
        header = string.split(lines[0],',')
        data = ""

        if (header[0] != "Date"):
            # no valid content
            return None

        for eachLine in lines:
            sdata = string.split (eachLine, ',')
            sdate = sdata[0]
            if (sdate != "Date"):
                if re_p3_1.match(sdate):
                    #print 'already good format ! ',sdate,sdata
                    pass
                else:
                    sdate = dd_mmm_yy2yyyymmdd(sdate)
                open = string.atof(sdata[1])
                high = string.atof(sdata[2])
                low = string.atof(sdata[3])
                value = string.atof(sdata[6])   #   Adj. Close*
                volume = string.atoi(sdata[5])

                if volume >= 0:
                    # encode in EBP format
                    # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME
                    line = (
                       quote.key(),
                       sdate,
                       open,
                       high,
                       low,
                       value,
                       volume
                    )
                    line = map(lambda (val): '%s' % str(val), line)
                    line = string.join(line, ';')
                    # append
                    data = data + line + '\r\n'

        return data
    def getdata(self,quote):
        debug("LiveUpdate_yahoo:getdata quote:%s " % quote)
        self.m_connected = False

        sname = yahooTicker(quote.ticker(),quote.market(),quote.place())

        if sname[0]=='^':
            ss = "%5E" + sname[1:]
        else:
            ss = sname

            
        query = (
            ('s', ss),
            ('f', 'sl1d1t1c1ohgv'),
            ('e', '.csv'),
        )
        query = map(lambda (var, val): '%s=%s' % (var, str(val)), query)
        query = string.join(query, '&')
        url = yahooUrl(quote.market(),live=True) + '?' + query

        debug("LiveUpdate_yahoo:getdata: url=%s",url)
        try:
            data=self.m_connection.getDataFromUrl(url)[:-2] # Get rid of CRLF
        except:
            debug('LiveUpdate_yahoo:unable to connect :-(')
            return None

        # pull data
        s400 = re.search(r"400 Bad Request", data, re.IGNORECASE|re.MULTILINE)
        if s400:
            if itrade_config.verbose:
                info('unknown %s quote (400 Bad Request) from Yahoo' % (quote.ticker()))
            return None


        sdata = string.split (data, ',')
        if len (sdata) < 9:
            if itrade_config.verbose:
                info('invalid data (bad answer length) for %s quote' % (quote.ticker()))
            return None

        #print sdata

        # connexion / clock
        self.m_connected = True

        # store for later use
        key = quote.key()

        sclock = sdata[3][1:-1]
        if sclock=="N/A" or sdata[2]=='"N/A"' or len(sclock)<5:
            if itrade_config.verbose:
                info('invalid datation for %s : %s %s' % (quote.ticker(),sclock,sdata[2]))
                #print sdata
            return None

        # start decoding
        symbol = sdata[0][1:-1]
        if symbol != sname:
            if itrade_config.verbose:
                info('invalid ticker : ask for %s and receive %s' % (sname,symbol))
            return None

        # date
        try:
            date = self.yahooDate(sdata[2])
            self.m_dcmpd[key] = sdata
            self.m_clock[key] = self.convertClock(quote.place(),sclock,date)
            self.m_dateindice[key] = sdata[2].replace('"','')
        except ValueError:
            if itrade_config.verbose:
                info('invalid datation for %s : %s %s' % (quote.ticker(),sclock,sdata[2]))
            return None

        # decode data
        value = string.atof (sdata[1])

        if (sdata[4]=='N/A'):
            debug('invalid change : N/A')
            change = 0.0
            return None
        else:
            change = string.atof (sdata[4])
        if (sdata[5]=='N/A'):
            debug('invalid open : N/A')
            open = 0.0
            return None
        else:
            open = string.atof (sdata[5])
        if (sdata[6]=='N/A'):
            debug('invalid high : N/A')
            high = 0.0
            return None
        else:
            high = string.atof (sdata[6])
        if (sdata[7]=='N/A'):
            debug('invalid low : N/A')
            low = 0.0
            return None
        else:
            low = string.atof (sdata[7])

        volume = string.atoi (sdata[8])
        if volume<0:
            debug('volume : invalid negative %d' % volume)
            return None
        if volume==0 and quote.list()!=QLIST_INDICES:
            debug('volume : invalid zero value %d' % volume)
            return None
        else:
            if value-change <= 0:
                return None
            else:
                percent = (change / (value - change))*100.0

        # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME;PERCENT;PREVIOUS
        data = (
          key,
          date,
          open,
          high,
          low,
          value,
          volume,
          percent,
          (value-change)
        )
        data = map(lambda (val): '%s' % str(val), data)
        data = string.join(data, ';')

        # temp: hunting an issue (SF bug 1848473)
        # if itrade_config.verbose:
        #    print data

        return data
    def getdata(self, quote, datedebut=None, datefin=None):
        # specific numTradeYears
        itrade_config.numTradeYears = 2

        if not datefin:
            datefin = date.today()
        if not datedebut:
            datedebut = date.today()
        if isinstance(datedebut, Datation):
            datedebut = datedebut.date()
        if isinstance(datefin, Datation):
            datefin = datefin.date()
        d1 = self.parseDate(datedebut)
        d2 = self.parseDate(datefin)

        debug("Import_yahoojp:getdata quote:%s begin:%s end:%s" %
              (quote, d1, d2))

        sname = yahooTicker(quote.ticker(), quote.market(), quote.place())

        ss = sname

        ch = '<tr align=right bgcolor="#ffffff">'
        lines = []

        for cursor in range(0, 4650, 50):

            url = yahooUrlJapan(
                quote.market(), live=False
            ) + '?' + 'c=%s&a=%s&b=%s&f=%s&d=%s&e=%s&g=d&s=%s&y=%s&z=%s' % (
                d1[0], d1[1], d1[2], d2[0], d2[1], d2[2], ss, str(cursor), ss)
            #url = 'http://table.yahoo.co.jp/t?s=%s&a=1&b=1&c=2000&d=%s&e=%s&f=%s&g=d&q=t&y=%s&z=/b?p=tjfzqcvy4.ewcf7pt&x=.csv' % (ss,d2[1],d2[2],d2[0],str(cursor))

            debug("Import_yahoojp:getdata: url=%s ", url)
            try:
                buf = self.m_connection.getDataFromUrl(url)
            except:
                debug('Import_yahoojp:unable to connect :-(')
                return None
            # pull data
            linesjp = self.splitLines(buf)
            if len(linesjp) <= 0:
                # empty content
                return None

            #typical lines indices

            #<tr align=right bgcolor="#ffffff">
            #<td><small>2009126</small></td>      (DATE)
            #<td><small>772.59</small></td>            (OPEN)
            #<td><small>777.91</small></td>            (HIGH)
            #<td><small>767.82</small></td>            (LOW)
            #<td><small><b>768.28</b></small></td>     (LAST)

            #typical lines quotes

            #<tr align=right bgcolor="#ffffff">
            #<td><small>2009119</small></td>           (DATE)
            #<td><small>198</small></td>               (OPEN)
            #<td><small>200</small></td>               (HIGH)
            #<td><small>198</small></td>               (LOW)
            #<td><small><b>199</b></small></td>        (LAST)
            #<td><small>92,000</small></td>            (VOLUME)
            #<td><small>199</small></td>               (ADJUSTCLOSE)
            #</tr><tr align=right bgcolor="#ffffff">

            #<td><small>2009116</small></td>
            #<td><small>197</small></td>
            #<td><small>200</small></td>

            n = 0
            i = 0
            q = 0
            #header = 'Date,Open,High,Low,Close,Volume,Adj Close'
            #filedata.write(header+'\n')
            for line in linesjp:

                if ch in line: n = 1
                if n == 1:
                    q = 1
                    if '<td><small>' in line:
                        i = i + 1
                        data = line[(line.find('small>') +
                                     6):(line.find('</'))]
                        if i == 1:
                            date = data
                            date = date.replace('ǯ', ' ')
                            date = date.replace('·î', ' ')
                            date = date.replace('Æü', '')
                            date = date.split()
                            if len(date[1]) == 1: date[1] = '0' + date[1]
                            if len(date[2]) == 1: date[2] = '0' + date[2]
                            date = '-'.join(date)
                        elif i == 2:
                            open = data
                            open = open.replace(',', '')
                        elif i == 3:
                            high = data
                            high = high.replace(',', '')
                        elif i == 4:
                            low = data
                            low = low.replace(',', '')
                        elif i == 5:
                            close = data[3:]
                            close = close.replace(',', '')

                            if ss == '998405' or ss == '998407' or ss == '23337':
                                volume = '0'
                                open = open.replace(',', '')
                                high = high.replace(',', '')
                                low = low.replace(',', '')
                                close = close.replace(',', '')
                                adjustclose = close
                                i = 0
                                n = 0
                                ligne = ','.join([
                                    date, open, high, low, close, volume,
                                    adjustclose
                                ])
                                #print ligne
                                lines.append(ligne)

                        elif i == 6:
                            volume = data
                            volume = volume.replace(',', '')
                        elif i == 7:
                            i = 0
                            n = 0
                            adjustclose = data
                            adjustclose = adjustclose.replace(',', '')
                            ligne = ','.join([
                                date, open, high, low, close, volume,
                                adjustclose
                            ])
                            #print ligne
                            lines.append(ligne)
            if q == 0:
                break
        data = ""
        for eachLine in lines:
            sdata = string.split(eachLine, ',')
            sdate = sdata[0]
            open = string.atof(sdata[1])
            high = string.atof(sdata[2])
            low = string.atof(sdata[3])
            value = string.atof(sdata[6])  #   Adj. Close*
            volume = string.atoi(sdata[5])

            if volume >= 0:
                # encode in EBP format
                # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME
                line = (quote.key(), sdate, open, high, low, value, volume)
                line = map(lambda (val): '%s' % str(val), line)
                line = string.join(line, ';')

                # append
                data = data + line + '\r\n'
        return data
    def getdata(self,quote):

        sname = yahooTicker(quote.ticker(),quote.market(),quote.place())
        ss = sname
        url = yahooUrlJapan(quote.market(),live=True) + '?' +'s=%s&d=v2' % (ss)

        debug("LiveUpdate_yahoojp:getdata: url=%s",url)
        
        try:
            
            data=self.m_connection.getDataFromUrl(url)
            data = data.replace('</td>','\n')
            lines = self.splitLines(data)

            # returns the data
            
            for line in lines:
                if 'uncompressed' in line:
                    year = line[line.find('JST ')+4:line.find(' -->')]
                else : year = '0000'

                   
                
            
            #sdata =[]
            ch = '<td nowrap align=center>'
            i = 0
            n = 0
                #typical lines
                #1 datetime  <td nowrap align=center>1/21       
                #2 last  <td nowrap><b>226</b>     
                #3 changetoday  <td nowrap><font color=ff0020>-4</font>
                                #<td nowrap>---
                #4 change_percent  <td nowrap><font color=ff0020>-1.74%</font>
                                   #<td nowrap>
                #5 volume   <td nowrap>1,705,200
                #6 open  <td nowrap>221      
                #7 high  <td nowrap>230
                #8 low   <td nowrap>221

            for line in lines:


                if '§<strong>' in line:
                    local_time = line[line.find(':')-2:line.find(':')+3]
                    local_date = line[line.find('§<strong>')+9:line.find('Æü ')]
                    local_date = local_date.replace('·î ',' ').split()
                    month = local_date[0]
                    if len(month)== 1 : month = '0' + month
                    day = local_date[1]
                    if len(day)== 1 : day = '0' + day
                    local_date = '"'+month+'/'+day+'/'+year+'"'
                    
                if ch in line:
                    n = 1
                if n == 1:
                    i=i+1
                    if i == 1:
                        date_time = line[len(ch):]
                        if date_time.find(':') != -1:
                            #print "last clock"
                            sclock = '"'+date_time+'"'
                            date = local_date
                        else :
                            #print "last date"
                            if date_time == '---':
                                date = 'N/A'
                            else:
                                date = 'N/A'
                                #date = '"'+date_time+'/'+year+'"'
                            sclock = "N/A"
                        #print 'date,hour',date,sclock
                    if i == 2:
                        last = line[line.find('<b>')+3:line.find('</b>')]
                        if last == '---': last = '0.0'
                        last = last.replace(',','')
                        #print 'last:',last
                    if i == 3:
                        if '<td nowrap><font color=ff0020>' in line:
                            change = line[line.find('ff0020>')+7:line.find('</font>')]
                        else:
                            change = line[line.find('<td nowrap>')+11:]
                        if change == '---': change = 'N/A'
                    if i == 4:
                        if '<td nowrap><font color=ff0020>' in line:
                            change_percent = line[line.find('ff0020>')+7:line.find('</font>')]
                        else:
                            change_percent = line[line.find('<td nowrap>')+11:]
                    if i == 5:
                        volume = line[line.find('<td nowrap>')+11:]
                        volume = volume.replace(',','')
                        if volume == '---': volume = '0'

                    if i == 6:
                        open = line[line.find('<td nowrap>')+11:]
                        if open == '---': open = 'N/A'
                        open = open.replace(',','')

                    if i == 7:
                        high = line[line.find('<td nowrap>')+11:]
                        if high == '---': high = 'N/A'
                        high = high.replace(',','')

                    if i == 8:
                        low = line[line.find('<td nowrap>')+11:]
                        if low == '---': low = 'N/A'
                        low = low.replace(',','')

                        n = 0
                        i = 0
                        
                        colors = ['red', 'blue', 'green', 'yellow']
                        result = ''.join(colors)


                        data = ','.join(['"'+ss+'"',last,date,sclock,change,open,high,low,volume])

                        break



            sdata = string.split (data, ',')

            if len (sdata) < 9:
                if itrade_config.verbose:
                    info('invalid data (bad answer length) for %s quote' % (quote.ticker()))
                return None

            # connexion / clock
            self.m_connected = True

            # store for later use
            key = quote.key() 

            sclock = sdata[3][1:-1]
            if sclock=="N/A" or sdata[2]=='"N/A"' or len(sclock)<5:
                if itrade_config.verbose:
                    info('invalid datation for %s : %s : %s' % (quote.ticker(),sclock,sdata[2]))
                return None

            # start decoding
            symbol = sdata[0][1:-1]
            if symbol != sname:
                if itrade_config.verbose:
                    info('invalid ticker : ask for %s and receive %s' % (sname,symbol))
                return None

            # date
            try:
                date = self.yahooDate(sdata[2])
                self.m_dcmpd[key] = sdata
                self.m_clock[key] = self.convertClock(quote.place(),sclock,date)
                self.m_dateindice[key] = sdata[2].replace('"','')

            except ValueError:
                if itrade_config.verbose:
                    info('invalid datation for %s : %s : %s' % (quote.ticker(),sclock,sdata[2]))
                return None

            # decode data
            value = string.atof (sdata[1])

            if (sdata[4]=='N/A'):
                debug('invalid change : N/A')
                change = 0.0
                return None
            else:
                change = string.atof (sdata[4])
            if (sdata[5]=='N/A'):
                debug('invalid open : N/A')
                open = 0.0
                return None
            else:
                open = string.atof (sdata[5])
            if (sdata[6]=='N/A'):
                debug('invalid high : N/A')
                high = 0.0
                return None
            else:
                high = string.atof (sdata[6])
            if (sdata[7]=='N/A'):
                debug('invalid low : N/A')
                low = 0.0
                return None
            else:
                low = string.atof (sdata[7])

            volume = string.atoi (sdata[8])
            if volume<0:
                debug('volume : invalid negative %d' % volume)
                return None
            if volume==0 and quote.list()!=QLIST_INDICES:
                debug('volume : invalid zero value %d' % volume)
                return None
            else:
                if value-change <= 0:
                    return None
                else:
                    percent = (change / (value - change))*100.0

            # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME;PERCENT;PREVIOUS
            data = (
              key,
              date,
              open,
              high,
              low,
              value,
              volume,
              percent,
              (value-change)
            )
            data = map(lambda (val): '%s' % str(val), data)
            data = string.join(data, ';')

            # temp: hunting an issue (SF bug 1848473)
            #if itrade_config.verbose:
                #print data

            return data
        
        except:
            debug('LiveUpdate_yahoojapan:unable to connect :-(')
            return None
    def getdata(self,quote,datedebut=None,datefin=None):
        if not datefin:
            datefin = date.today()

        if not datedebut:
            datedebut = date.today()

        if isinstance(datedebut,Datation):
            datedebut = datedebut.date()

        if isinstance(datefin,Datation):
            datefin = datefin.date()

        d1 = self.parseDate(datedebut)
        d2 = self.parseDate(datefin)

        debug("Import_yahoo:getdata quote:%s begin:%s end:%s" % (quote,d1,d2))

        sname = yahooTicker(quote.ticker(),quote.market(),quote.place())

        if sname[0]=='^':
            ss = "%5E" + sname[1:]
        else:
            ss = sname

        query = (
            ('s', ss),
            ('a', '%02d' % (int(d1[1])-1)),
            ('b', d1[2]),
            ('c', d1[0]),
            ('d', '%02d' % (int(d2[1])-1)),
            ('e', d2[2]),
            ('f', d2[0]),
            ('y', '0'),
            ('g', 'd'),
            ('ignore', '.csv'),
        )
        query = map(lambda (var, val): '%s=%s' % (var, str(val)), query)
        query = string.join(query, '&')
        url = yahooUrl(quote.market(),live=False) + '?' + query

        debug("Import_yahoo:getdata: url=%s ",url)
        try:
            buf=self.m_connection.getDataFromUrl(url)
        except:
            debug('Import_yahoo:unable to connect :-(')
            return None

        # pull data
        lines = self.splitLines(buf)
        if len(lines)<=0:
            # empty content
            return None
        header = string.split(lines[0],',')
        data = ""

        if (header[0]<>"Date"):
            # no valid content
            return None

        for eachLine in lines:
            sdata = string.split (eachLine, ',')
            sdate = sdata[0]
            if (sdate<>"Date"):
                if re_p3_1.match(sdate):
                    #print 'already good format ! ',sdate,sdata
                    pass
                else:
                    sdate = dd_mmm_yy2yyyymmdd(sdate)
                open = string.atof(sdata[1])
                high = string.atof(sdata[2])
                low = string.atof(sdata[3])
                value = string.atof(sdata[6])   #   Adj. Close*
                volume = string.atoi(sdata[5])

                if volume>=0:
                    # encode in EBP format
                    # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME
                    line = (
                      quote.key(),
                      sdate,
                      open,
                      high,
                      low,
                      value,
                      volume
                    )
                    line = map(lambda (val): '%s' % str(val), line)
                    line = string.join(line, ';')

                    # append
                    data = data + line + '\r\n'
        return data
    def getdata(self, quote):
        debug("LiveUpdate_yahoo:getdata quote:%s " % quote)
        self.m_connected = False

        sname = yahooTicker(quote.ticker(), quote.market(), quote.place())

        if sname[0] == '^':
            ss = "%5E" + sname[1:]
        else:
            ss = sname

        query = (
            ('s', ss),
            ('f', 'sl1d1t1c1ohgv'),
            ('e', '.csv'),
        )
        query = map(lambda (var, val): '%s=%s' % (var, str(val)), query)
        query = string.join(query, '&')
        url = yahooUrl(quote.market(), live=True) + '?' + query

        debug("LiveUpdate_yahoo:getdata: url=%s", url)
        try:
            data = self.m_connection.getDataFromUrl(url)[:
                                                         -2]  # Get rid of CRLF
        except:
            debug('LiveUpdate_yahoo:unable to connect :-(')
            return None

        # pull data
        s400 = re.search(r"400 Bad Request", data,
                         re.IGNORECASE | re.MULTILINE)
        if s400:
            if itrade_config.verbose:
                info('unknown %s quote (400 Bad Request) from Yahoo' %
                     (quote.ticker()))
            return None

        sdata = string.split(data, ',')
        if len(sdata) < 9:
            if itrade_config.verbose:
                info('invalid data (bad answer length) for %s quote' %
                     (quote.ticker()))
            return None

        #print sdata

        # connexion / clock
        self.m_connected = True

        # store for later use
        key = quote.key()

        sclock = sdata[3][1:-1]
        if sclock == "N/A" or sdata[2] == '"N/A"' or len(sclock) < 5:
            if itrade_config.verbose:
                info('invalid datation for %s : %s %s' %
                     (quote.ticker(), sclock, sdata[2]))
                #print sdata
            return None

        # start decoding
        symbol = sdata[0][1:-1]
        if symbol != sname:
            if itrade_config.verbose:
                info('invalid ticker : ask for %s and receive %s' %
                     (sname, symbol))
            return None

        # date
        try:
            date = self.yahooDate(sdata[2])
            self.m_dcmpd[key] = sdata
            self.m_clock[key] = self.convertClock(quote.place(), sclock, date)
            self.m_dateindice[key] = sdata[2].replace('"', '')
        except ValueError:
            if itrade_config.verbose:
                info('invalid datation for %s : %s %s' %
                     (quote.ticker(), sclock, sdata[2]))
            return None

        # decode data
        value = string.atof(sdata[1])

        if (sdata[4] == 'N/A'):
            debug('invalid change : N/A')
            change = 0.0
            return None
        else:
            change = string.atof(sdata[4])
        if (sdata[5] == 'N/A'):
            debug('invalid open : N/A')
            open = 0.0
            return None
        else:
            open = string.atof(sdata[5])
        if (sdata[6] == 'N/A'):
            debug('invalid high : N/A')
            high = 0.0
            return None
        else:
            high = string.atof(sdata[6])
        if (sdata[7] == 'N/A'):
            debug('invalid low : N/A')
            low = 0.0
            return None
        else:
            low = string.atof(sdata[7])

        volume = string.atoi(sdata[8])
        if volume < 0:
            debug('volume : invalid negative %d' % volume)
            return None
        if volume == 0 and quote.list() != QLIST_INDICES:
            debug('volume : invalid zero value %d' % volume)
            return None
        else:
            if value - change <= 0:
                return None
            else:
                percent = (change / (value - change)) * 100.0

        # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME;PERCENT;PREVIOUS
        data = (key, date, open, high, low, value, volume, percent,
                (value - change))
        data = map(lambda (val): '%s' % str(val), data)
        data = string.join(data, ';')

        # temp: hunting an issue (SF bug 1848473)
        # if itrade_config.verbose:
        #    print data

        return data
    def getdata(self,quote,datedebut=None,datefin=None):
        # specific numTradeYears
        itrade_config.numTradeYears = 2
        
        if not datefin:
            datefin = date.today()
        if not datedebut:
            datedebut = date.today()
        if isinstance(datedebut,Datation):
            datedebut = datedebut.date()
        if isinstance(datefin,Datation):
            datefin = datefin.date()
        d1 = self.parseDate(datedebut)
        d2 = self.parseDate(datefin)

        debug("Import_yahoojp:getdata quote:%s begin:%s end:%s" % (quote,d1,d2))

        sname = yahooTicker(quote.ticker(),quote.market(),quote.place())

        ss = sname
            
        ch = '<tr align=right bgcolor="#ffffff">'
        lines = []
        
        for cursor in range(0,4650,50):
            
            url = yahooUrlJapan(quote.market(),live=False) + '?' +'c=%s&a=%s&b=%s&f=%s&d=%s&e=%s&g=d&s=%s&y=%s&z=%s' % (d1[0],d1[1],d1[2],d2[0],d2[1],d2[2],ss,str(cursor),ss)
            #url = 'http://table.yahoo.co.jp/t?s=%s&a=1&b=1&c=2000&d=%s&e=%s&f=%s&g=d&q=t&y=%s&z=/b?p=tjfzqcvy4.ewcf7pt&x=.csv' % (ss,d2[1],d2[2],d2[0],str(cursor))
            
            debug("Import_yahoojp:getdata: url=%s ",url)
            try:
                buf=self.m_connection.getDataFromUrl(url)
            except:
                debug('Import_yahoojp:unable to connect :-(')
                return None
            # pull data
            linesjp = self.splitLines(buf)
            if len(linesjp)<=0:
                # empty content
                return None
            
            #typical lines indices
            
            #<tr align=right bgcolor="#ffffff">
            #<td><small>2009126</small></td>      (DATE)
            #<td><small>772.59</small></td>            (OPEN)
            #<td><small>777.91</small></td>            (HIGH)
            #<td><small>767.82</small></td>            (LOW)
            #<td><small><b>768.28</b></small></td>     (LAST)
            
            #typical lines quotes

            #<tr align=right bgcolor="#ffffff">        
            #<td><small>2009119</small></td>           (DATE)
            #<td><small>198</small></td>               (OPEN)
            #<td><small>200</small></td>               (HIGH)
            #<td><small>198</small></td>               (LOW)
            #<td><small><b>199</b></small></td>        (LAST)
            #<td><small>92,000</small></td>            (VOLUME)
            #<td><small>199</small></td>               (ADJUSTCLOSE)
            #</tr><tr align=right bgcolor="#ffffff">

            #<td><small>2009116</small></td>
            #<td><small>197</small></td>
            #<td><small>200</small></td>

            
            n = 0
            i = 0   
            q = 0
            #header = 'Date,Open,High,Low,Close,Volume,Adj Close'
            #filedata.write(header+'\n')
            for line in linesjp:

                if ch in line : n = 1
                if n == 1 :
                    q = 1
                    if '<td><small>' in line:
                        i = i + 1
                        data = line[(line.find('small>')+6):(line.find ('</'))]
                        if i == 1 :
                            date = data
                            date = date.replace('ǯ',' ')
                            date = date.replace('·î',' ')
                            date = date.replace('Æü','')
                            date = date.split()
                            if len(date[1]) == 1 : date[1] = '0'+date[1]
                            if len(date[2]) == 1 : date[2] = '0'+date[2]
                            date = '-'.join(date)
                        elif i == 2 :
                            open = data
                            open = open.replace(',','')
                        elif i == 3 :
                            high = data
                            high = high.replace(',','')
                        elif i == 4 :
                            low = data
                            low = low.replace(',','')
                        elif i == 5 :
                            close = data[3:]
                            close = close.replace(',','')
                            
                            if ss == '998405' or ss == '998407' or ss =='23337' :
                                volume = '0'
                                open = open.replace(',','')
                                high = high.replace(',','')
                                low = low.replace(',','')
                                close = close.replace(',','')
                                adjustclose = close
                                i = 0
                                n = 0
                                ligne = ','.join([date,open,high,low,close,volume,adjustclose])
                                #print ligne
                                lines.append(ligne)
                                
                        elif i == 6 :
                            volume = data
                            volume = volume.replace(',','')
                        elif i == 7 :
                            i = 0
                            n = 0
                            adjustclose = data
                            adjustclose = adjustclose.replace(',','')
                            ligne = ','.join([date,open,high,low,close,volume,adjustclose])
                            #print ligne
                            lines.append(ligne)
            if q == 0:
                break   
        data = ""
        for eachLine in lines:
            sdata = string.split (eachLine, ',')
            sdate = sdata[0]
            open = string.atof(sdata[1])
            high = string.atof(sdata[2])
            low = string.atof(sdata[3])
            value = string.atof(sdata[6])   #   Adj. Close*
            volume = string.atoi(sdata[5])

            if volume>=0:
                # encode in EBP format
                # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME
                line = (
                    quote.key(),
                    sdate,
                    open,
                    high,
                    low,
                    value,
                    volume
                )
                line = map(lambda (val): '%s' % str(val), line)
                line = string.join(line, ';')

                # append
                data = data + line + '\r\n'
        return data
Ejemplo n.º 8
0
    def getdata(self, quote):

        sname = yahooTicker(quote.ticker(), quote.market(), quote.place())
        ss = sname
        url = yahooUrlJapan(quote.market(),
                            live=True) + '?' + 's=%s&d=v2' % (ss)

        debug("LiveUpdate_yahoojp:getdata: url=%s", url)

        try:

            data = self.m_connection.getDataFromUrl(url)
            data = data.replace('</td>', '\n')
            lines = self.splitLines(data)

            # returns the data

            for line in lines:
                if 'uncompressed' in line:
                    year = line[line.find('JST ') + 4:line.find(' -->')]
                else:
                    year = '0000'

            #sdata =[]
            ch = '<td nowrap align=center>'
            i = 0
            n = 0
            #typical lines
            #1 datetime  <td nowrap align=center>1/21
            #2 last  <td nowrap><b>226</b>
            #3 changetoday  <td nowrap><font color=ff0020>-4</font>
            #<td nowrap>---
            #4 change_percent  <td nowrap><font color=ff0020>-1.74%</font>
            #<td nowrap>
            #5 volume   <td nowrap>1,705,200
            #6 open  <td nowrap>221
            #7 high  <td nowrap>230
            #8 low   <td nowrap>221

            for line in lines:

                if '§<strong>' in line:
                    local_time = line[line.find(':') - 2:line.find(':') + 3]
                    local_date = line[line.find('§<strong>') +
                                      9:line.find('Æü ')]
                    local_date = local_date.replace('·î ', ' ').split()
                    month = local_date[0]
                    if len(month) == 1: month = '0' + month
                    day = local_date[1]
                    if len(day) == 1: day = '0' + day
                    local_date = '"' + month + '/' + day + '/' + year + '"'

                if ch in line:
                    n = 1
                if n == 1:
                    i = i + 1
                    if i == 1:
                        date_time = line[len(ch):]
                        if date_time.find(':') != -1:
                            #print "last clock"
                            sclock = '"' + date_time + '"'
                            date = local_date
                        else:
                            #print "last date"
                            if date_time == '---':
                                date = 'N/A'
                            else:
                                date = 'N/A'
                                #date = '"'+date_time+'/'+year+'"'
                            sclock = "N/A"
                        #print 'date,hour',date,sclock
                    if i == 2:
                        last = line[line.find('<b>') + 3:line.find('</b>')]
                        if last == '---': last = '0.0'
                        last = last.replace(',', '')
                        #print 'last:',last
                    if i == 3:
                        if '<td nowrap><font color=ff0020>' in line:
                            change = line[line.find('ff0020>') +
                                          7:line.find('</font>')]
                        else:
                            change = line[line.find('<td nowrap>') + 11:]
                        if change == '---': change = 'N/A'
                    if i == 4:
                        if '<td nowrap><font color=ff0020>' in line:
                            change_percent = line[line.find('ff0020>') +
                                                  7:line.find('</font>')]
                        else:
                            change_percent = line[line.find('<td nowrap>') +
                                                  11:]
                    if i == 5:
                        volume = line[line.find('<td nowrap>') + 11:]
                        volume = volume.replace(',', '')
                        if volume == '---': volume = '0'

                    if i == 6:
                        open = line[line.find('<td nowrap>') + 11:]
                        if open == '---': open = 'N/A'
                        open = open.replace(',', '')

                    if i == 7:
                        high = line[line.find('<td nowrap>') + 11:]
                        if high == '---': high = 'N/A'
                        high = high.replace(',', '')

                    if i == 8:
                        low = line[line.find('<td nowrap>') + 11:]
                        if low == '---': low = 'N/A'
                        low = low.replace(',', '')

                        n = 0
                        i = 0

                        colors = ['red', 'blue', 'green', 'yellow']
                        result = ''.join(colors)

                        data = ','.join([
                            '"' + ss + '"', last, date, sclock, change, open,
                            high, low, volume
                        ])

                        break

            sdata = string.split(data, ',')

            if len(sdata) < 9:
                if itrade_config.verbose:
                    info('invalid data (bad answer length) for %s quote' %
                         (quote.ticker()))
                return None

            # connexion / clock
            self.m_connected = True

            # store for later use
            key = quote.key()

            sclock = sdata[3][1:-1]
            if sclock == "N/A" or sdata[2] == '"N/A"' or len(sclock) < 5:
                if itrade_config.verbose:
                    info('invalid datation for %s : %s : %s' %
                         (quote.ticker(), sclock, sdata[2]))
                return None

            # start decoding
            symbol = sdata[0][1:-1]
            if symbol != sname:
                if itrade_config.verbose:
                    info('invalid ticker : ask for %s and receive %s' %
                         (sname, symbol))
                return None

            # date
            try:
                date = self.yahooDate(sdata[2])
                self.m_dcmpd[key] = sdata
                self.m_clock[key] = self.convertClock(quote.place(), sclock,
                                                      date)
                self.m_dateindice[key] = sdata[2].replace('"', '')

            except ValueError:
                if itrade_config.verbose:
                    info('invalid datation for %s : %s : %s' %
                         (quote.ticker(), sclock, sdata[2]))
                return None

            # decode data
            value = string.atof(sdata[1])

            if (sdata[4] == 'N/A'):
                debug('invalid change : N/A')
                change = 0.0
                return None
            else:
                change = string.atof(sdata[4])
            if (sdata[5] == 'N/A'):
                debug('invalid open : N/A')
                open = 0.0
                return None
            else:
                open = string.atof(sdata[5])
            if (sdata[6] == 'N/A'):
                debug('invalid high : N/A')
                high = 0.0
                return None
            else:
                high = string.atof(sdata[6])
            if (sdata[7] == 'N/A'):
                debug('invalid low : N/A')
                low = 0.0
                return None
            else:
                low = string.atof(sdata[7])

            volume = string.atoi(sdata[8])
            if volume < 0:
                debug('volume : invalid negative %d' % volume)
                return None
            if volume == 0 and quote.list() != QLIST_INDICES:
                debug('volume : invalid zero value %d' % volume)
                return None
            else:
                if value - change <= 0:
                    return None
                else:
                    percent = (change / (value - change)) * 100.0

            # ISIN;DATE;OPEN;HIGH;LOW;CLOSE;VOLUME;PERCENT;PREVIOUS
            data = (key, date, open, high, low, value, volume, percent,
                    (value - change))
            data = map(lambda (val): '%s' % str(val), data)
            data = string.join(data, ';')

            # temp: hunting an issue (SF bug 1848473)
            #if itrade_config.verbose:
            #print data

            return data

        except:
            debug('LiveUpdate_yahoojapan:unable to connect :-(')
            return None