def main():
    test = MonthData( 2 , 2012 )
    assert len( test.data ) == 30
    test = MonthData( 2 , 2013 )
    assert len( test.data ) == 29
    test = MonthData( 3 , 2014 )
    assert len( test.data ) == 32
    test = MonthData( 4 , 2014 )
    assert len( test.data ) == 31
    
    #test = MonthData( 8 , 2015 )
    #PriceWriter.write_month_data_to_file( test , "test" )
    #test = PriceReader.read_month_data( 8 , 2015 , "test" )
    #print test
    #Mithril ore
    priceData = PriceCrawler.get_price_data_from_html( 447 )
    PriceWriter.save_data( priceData )
    
    #Mithril bar
    priceData = PriceCrawler.get_price_data_from_html( 2359 )
    PriceWriter.save_data( priceData )
    
    priceData = PriceCrawler.get_price_data_from_json( "Mithril bar" , 2359 )
    PriceWriter.save_data( priceData )
    
    #404 Error
    priceData = PriceCrawler.get_price_data_from_html( 21736 )
    assert( priceData == None )
    
    fromCSV = PriceReader.get_price_data_from_csv( 447 )
    fromHTML = PriceCrawler.get_price_data_from_html( 447 )
    assert fromCSV == fromHTML
    
    print "Regression testing for price_data_io.py passed."
Exemplo n.º 2
0
    def print_prices(self):
        for item in self.config['all_info'].keys():
            pc = PriceCrawler(self.config, item)

            interested_in_length = len(item)
            print ('-' * 20) + ('-' * interested_in_length) + ('-' * 20)
            print ('-' * 20) + item + ('-' * 20)
            print ('-' * 20) + ('-' * interested_in_length) + ('-' * 20)

            [self.print_pretty(x) for x in pc.get_all()]
            print ''
Exemplo n.º 3
0
def download_data_by_id( id ):
    priceData = PriceCrawler.get_price_data_from_html( id )
    if ( priceData != None ):
        f = open( "price_data/item_ids" , "a" )
        f.write( priceData.get_name() + "," + str( id ) + "\n" )
        f.close()
        PriceWriter.write_price_data_to_csv( "price_data/master_list/" + str(id) + ".csv" , priceData )
Exemplo n.º 4
0
    def get_prices(self):
        """
        Generator method which produces a response dict for every item
        of interest (key in all_info dict)

        :return: response dict every time it is called
        """

        for item in self.model.get_all_items():
            for store in self.model.get_stores_for_item(item):
                pc = PriceCrawler(self.config, self.model, item)

                response = {
                    'timestamp': time.time(),
                    'item_name': item,
                }

                try:
                    response.update(pc.get_store_price(store))
                except TypeError:
                    continue  # no price for store

                yield response
Exemplo n.º 5
0
    def add_new_item(self, item):
        for store in item['stores']:
            rec = {
                'item_name': item['item_name'],
            }
            rec.update(store)
            self.db.mongodb[self.items_collection_name].insert(rec)

            if store['store_name'] == 'skroutz':
                skroutz_site_url = store['url']
                specs = {
                    'item_name': item['item_name'],
                    'specs': get_specs(skroutz_site_url),
                    'quick_specs': get_quick_specs(skroutz_site_url),
                    'image': get_image_for_item(skroutz_site_url)
                }
                self.db.mongodb[self.specs_collection_name].insert(specs)

            # specs = {
            #     'item_name': item['item_name'],
            #     'specs': get_specs('http://www.gsmarena.com/microsoft_lumia_950-7262.php')
            # }
            # self.db.mongodb['specs'].insert(specs)

            from price_crawler import PriceCrawler
            pc = PriceCrawler(self.config, self, item['item_name'])
            rec = {
                'timestamp': time.time(),
                'item_name': item['item_name'],
            }

            try:
                rec.update(pc.get_store_price(store['store_name']))
                self.add_record(rec)
            except TypeError:
                pass  # no price for store
Exemplo n.º 6
0
    def record_commodity_stats(startId, endId):
        for i in range(startId, endId):
            print "Processing " + str(i)
            testData = PriceCrawler.get_price_data_from_html(i)
            if (not testData is None):
                maxPrice = 0
                minPrice = 999999999
                maxVolume = 0
                minVolume = 999999999
                allPoints = testData.get_all_datapoints()

                #we do not include the last data point because
                #that is today's data, which may be incomplete
                #and so the volume may be much less than what it
                #really is.
                for datapoint in allPoints[1:len(allPoints) - 1]:

                    #prices and volumes of 0 are invalid
                    if (datapoint.get_price != 0):
                        maxPrice = max(datapoint.get_price(), maxPrice)
                        minPrice = min(datapoint.get_price(), minPrice)

                    if (datapoint.get_volume() != 0):
                        maxVolume = max(datapoint.get_volume(), maxVolume)
                        minVolume = min(datapoint.get_volume(), minVolume)

                out = open("price_data/item_stats", "a")
                out.write( str(i) + "," + str(maxPrice) + "," + str(minPrice) + \
                    "," + str(maxVolume) + "," + str(minVolume) + "\n" )
                out.close()
            else:
                print str(i) + " was not a valid id"

            #let's not get blocked for too many requests
            sleepInterval = randint(1, 2)
            sleepInterval = 2
            print "Sleeping " + str(sleepInterval)
            sleep(sleepInterval)
    def record_commodity_stats( startId , endId ):
        for i in range( startId , endId ):
            print "Processing " + str( i )
            testData = PriceCrawler.get_price_data_from_html( i )
            if ( not testData is None ):
                maxPrice = 0
                minPrice = 999999999
                maxVolume = 0
                minVolume = 999999999
                allPoints = testData.get_all_datapoints()
                
                #we do not include the last data point because
                #that is today's data, which may be incomplete
                #and so the volume may be much less than what it
                #really is.
                for datapoint in allPoints[1:len(allPoints)-1]:

                    #prices and volumes of 0 are invalid                    
                    if ( datapoint.get_price != 0 ):
                        maxPrice = max( datapoint.get_price() , maxPrice )
                        minPrice = min( datapoint.get_price() , minPrice )
                    
                    if ( datapoint.get_volume() != 0 ):
                        maxVolume = max( datapoint.get_volume() , maxVolume )
                        minVolume = min( datapoint.get_volume() , minVolume )
                    
                out = open( "price_data/item_stats" , "a" )
                out.write( str(i) + "," + str(maxPrice) + "," + str(minPrice) + \
                    "," + str(maxVolume) + "," + str(minVolume) + "\n" )
                out.close()
            else:
                print str( i ) + " was not a valid id"
                
            #let's not get blocked for too many requests
            sleepInterval = randint( 1 , 2 )
            sleepInterval = 2
            print "Sleeping " + str( sleepInterval )
            sleep( sleepInterval )
Exemplo n.º 8
0
 def download_data_by_name_and_id(name, id):
     data = PriceCrawler.get_price_data_from_json(name, id)
     if (data != None):
         PriceWriter.save_data(data)
     else:
         raise "Invalid commodity name or id."
 def download_data_by_name_and_id( name , id ):
     data = PriceCrawler.get_price_data_from_json( name , id )
     if ( data != None ) :
         PriceWriter.save_data( data )
     else :
         raise "Invalid commodity name or id."