from web.ukuncut.models import Brand, Dodger
from openingtimes.models import OpenTime
BASE_URL = "http://www.bhs.co.uk/mall/infopageviewer.cfm/bhsstore/storelocatorbranch?branch_id="
company_id = Dodger.C_BHS
brand_name = Dodger.COMPANIES[company_id-1][1]
brand_id = company_id
print brand_name
try:
brand = Brand.objects.get(brand_id=brand_id)
except Brand.DoesNotExist:
brand = Brand(brand_id=brand_id, name=brand_name)
brand.save()
def populate_ids():
import redis
r = redis.Redis(db=4)
start_id = 0
while start_id < 1000:
req = urllib2.urlopen("%s%s" % (BASE_URL,start_id))
soup = BeautifulSoup.BeautifulSoup(req)
if not soup.findAll('img', {'src': re.compile('/mall/errors/fault.gif')}):
print start_id
r.sadd('bhs_ids', start_id)
def scrape():
company_id = Dodger.C_ARCADIA
brands = (
(12551,'Burton', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12551&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12552,'Dorothy Perkins', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12552&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12553,'Evans', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12553&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12554,'Miss selfridge', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12554&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12555,'Topshop', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12555&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12556,'Topman' , "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12556&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
(12557,'Wallis', "http://cloudservices.arcadiagroup.co.uk/storestock/storestock?brand=12557&jsonp_callback=jsonp1292077523475&lat=51.461752&long=-0.114286&dist=50000&res=10000000&_=1292077544710",),
)
for brand_id, brand_name, url in brands:
try:
brand = Brand.objects.get(brand_id=brand_id)
except Brand.DoesNotExist:
brand = Brand(brand_id=brand_id, name=brand_name)
brand.save()
r = urllib2.Request(url, headers={'User-Agent' : "Mozilla/5.0 (X11; U; Linux i686) Gecko/20071127 Firefox/2.0.0.11"})
res = urllib2.urlopen(r)
x = res.read()[19:-2]
data = json.loads(x)
for store in data['stores']['store']:
for k,v in store.items():
store[k] = v.encode('utf8')
try:
d = Dodger.objects.get(company=company_id, doger_id=store['storeId'], brand=brand)
except Dodger.DoesNotExist:
d = Dodger()
d.name = store['storeName']
d.company = company_id
d.brand = brand
d.doger_id = store['storeId']
d.address1 = store.get('address1')
d.address2 = store.get('address2')
d.address3 = store.get('address3')
d.address4 = store.get('address4')
d.postcode = store.get('postcode')
d.phone = store.get('telephoneNumber')
# d.location = fromstr('POINT(%s, %s)' % (store['latitude'], store['longitude']))
# d.location = 'POINT((%s, %s))' % (store['latitude'], store['longitude'])
if float(store['latitude']) and float(store['longitude']):
d.location = Point(float(store['latitude']), float(store['longitude']))
d.country = store['country']
d.save()
# Delete all opeing times for this Dodger
d.opening_times.all().delete()
def parse_open_time(str_time):
# 08:00-21:00
if str_time:
open_close = str_time.split('-')
if len(open_close) >= 2:
open_close = [v.replace('.', ':') for v in open_close]
return open_close
try:
# Monday
open_close = parse_open_time(store.get('openingMon'))
if store.get('openingMon') and open_close:
o = d.opening_times.create(day_of_week=0, open_time=open_close[0], close_time=open_close[1])
# Tuesday
open_close = parse_open_time(store.get('openingTue'))
if store.get('openingTue') and open_close:
o = d.opening_times.create(day_of_week=1, open_time=open_close[0], close_time=open_close[1])
# Wednesday
open_close = parse_open_time(store.get('openingWed'))
if store.get('openingWed') and open_close:
o = d.opening_times.create(day_of_week=2, open_time=open_close[0], close_time=open_close[1])
# Thursday
open_close = parse_open_time(store.get('openingThu'))
if store.get('openingThu') and open_close:
o = d.opening_times.create(day_of_week=3, open_time=open_close[0], close_time=open_close[1])
# Friday
open_close = parse_open_time(store.get('openingFri'))
if store.get('openingFri') and open_close:
o = d.opening_times.create(day_of_week=4, open_time=open_close[0], close_time=open_close[1])
# Saturday
open_close = parse_open_time(store.get('openingSat'))
if store.get('openingSat') and open_close:
o = d.opening_times.create(day_of_week=5, open_time=open_close[0], close_time=open_close[1])
# Sunday
open_close = parse_open_time(store.get('openingSun'))
if store.get('openingSun') and open_close:
o = d.opening_times.create(day_of_week=6, open_time=open_close[0], close_time=open_close[1])
except Exception, e:
print e
print open_close