예제 #1
0
파일: vodafone.py 프로젝트: symroe/ukuncut
def parse(handle):
    # req = urllib2.urlopen(url)
    soup = BeautifulSoup.BeautifulSoup(handle)
    result_div = soup.find('div', {'class' : 'tableTopBorder'})
    for result in result_div.findAll('tr', {'class' : re.compile('lightBlue')}):
        try:
            cells = result.findAll('td')

            addresses = ''.join([e for e in cells[1].recursiveChildGenerator() if isinstance(e,unicode)])
            addresses = addresses.strip().split('\n')

            name = addresses.pop(0)
            postcode = addresses.pop(-1)

            addresses_dict = {}
            for i, address in enumerate(addresses):
                i = i+1
                addresses_dict["address%s" % i] = address

            url = cells[-1].a['href']
            parsed_url = urlparse.parse_qs(url)
            lat = parsed_url['lat']
            lon = parsed_url['lon']

            try:
                d = Dodger.objects.get(name=name, company=company_id, postcode=postcode)
            except Exception, e:
                d = Dodger(name=name, company=company_id, postcode=postcode)

            d.address1 = addresses_dict.get('address1')
            d.address2 = addresses_dict.get('address2')
            d.address3 = addresses_dict.get('address3')
            d.address4 = addresses_dict.get('address4')
            d.country = "United Kingdom"
            d.location = Point(float(lat[0]), float(lon[0]))
            d.brand = brand
            d.save()
        except Exception, e:
            print brand_name, e
예제 #2
0
파일: arcadia.py 프로젝트: symroe/ukuncut
def scrape():
    
    company_id = Dodger.C_ARCADIA
    
    brands = ( 
        (12551,'Burton',          "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12551&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12552,'Dorothy Perkins', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12552&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12553,'Evans',           "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12553&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12554,'Miss selfridge',  "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12554&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12555,'Topshop',         "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12555&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12556,'Topman' ,         "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12556&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
        (12557,'Wallis',          "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12557&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
    )

    for brand_id, brand_name, url in brands:
    
        try:
            brand = Brand.objects.get(brand_id=brand_id)
        except Brand.DoesNotExist:
            brand = Brand(brand_id=brand_id, name=brand_name)
            brand.save()
        
        r = urllib2.Request(url, headers={'User-Agent' : "Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"})
        res = urllib2.urlopen(r)
        x = res.read()[19:-2]

        data = json.loads(x)
    
        for store in data['stores']['store']:
            for k,v in store.items():
                store[k] = v.encode('utf8')
            
            try:
                d = Dodger.objects.get(company=company_id, doger_id=store['storeId'], brand=brand)
            except Dodger.DoesNotExist:
                d = Dodger()
            d.name = store['storeName']
            d.company = company_id
            d.brand = brand
            d.doger_id = store['storeId']
            d.address1 = store.get('address1')
            d.address2 = store.get('address2')
            d.address3 = store.get('address3')
            d.address4 = store.get('address4')
            d.postcode = store.get('postcode')
            d.phone = store.get('telephoneNumber')
            # d.location = fromstr('POINT(%s, %s)' % (store['latitude'], store['longitude']))
            # d.location = 'POINT((%s, %s))' % (store['latitude'], store['longitude'])
            if float(store['latitude']) and float(store['longitude']):
                d.location = Point(float(store['latitude']), float(store['longitude']))
            d.country = store['country']
            d.save()
            
            
            # Delete all opeing times for this Dodger
            d.opening_times.all().delete()
            
            def parse_open_time(str_time):
                # 08:00-21:00
                if str_time:
                    open_close = str_time.split('-')
                    if len(open_close) >= 2:
                        open_close = [v.replace('.', ':') for v in open_close]
                        return open_close
                
            try:
                # Monday
                open_close = parse_open_time(store.get('openingMon'))
                if store.get('openingMon') and open_close:
                    o = d.opening_times.create(day_of_week=0, open_time=open_close[0], close_time=open_close[1])
                # Tuesday
                open_close = parse_open_time(store.get('openingTue'))
                if store.get('openingTue') and open_close:
                    o = d.opening_times.create(day_of_week=1, open_time=open_close[0], close_time=open_close[1])
                # Wednesday
                open_close = parse_open_time(store.get('openingWed'))
                if store.get('openingWed') and open_close:
                    o = d.opening_times.create(day_of_week=2, open_time=open_close[0], close_time=open_close[1])
                # Thursday
                open_close = parse_open_time(store.get('openingThu'))
                if store.get('openingThu') and open_close:
                    o = d.opening_times.create(day_of_week=3, open_time=open_close[0], close_time=open_close[1])
                # Friday
                open_close = parse_open_time(store.get('openingFri'))
                if store.get('openingFri') and open_close:
                    o = d.opening_times.create(day_of_week=4, open_time=open_close[0], close_time=open_close[1])
                # Saturday
                open_close = parse_open_time(store.get('openingSat'))
                if store.get('openingSat') and open_close:
                    o = d.opening_times.create(day_of_week=5, open_time=open_close[0], close_time=open_close[1])
                # Sunday
                open_close = parse_open_time(store.get('openingSun'))
                if store.get('openingSun') and open_close:
                    o = d.opening_times.create(day_of_week=6, open_time=open_close[0], close_time=open_close[1])
            except Exception, e:
                print e
                print open_close