def fetch_stores(data): url = data['url'] try: body = cm.post_data(url, {'rsp': 'json', 'country': data['country_code']}) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw = json.loads(body) store_list = [] for s in raw['stores']: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = cm.html2plain(s['name']).strip() addr_list = [] for key in ['address1', 'address2']: if s[key].strip() != '': addr_list.append(cm.reformat_addr(s[key])) entry[cm.addr_e] = ' '.join(addr_list) # r=s['region'].strip().upper() # m = re.search(ur'\b([A-Z]{2})\b', r) # if data[cm.country_e]=='UNITED STATES' and m is not None: # # 美国 # ret = gs.look_up(m.group(1), 2) # if ret is not None: # r = ret['name_e'] # entry[cm.province_e] = r entry[cm.city_e] = cm.extract_city(s['city'])[0] entry[cm.zip_code] = s['zip'].strip() entry[cm.country_e] = data[cm.country_e] entry[cm.lat] = string.atof(s['lat']) entry[cm.lng] = string.atof(s['lng']) entry[cm.tel] = s['phone'].strip() entry[cm.fax] = s['fax'].strip() entry[cm.email] = s['emailaddress'].strip() entry[cm.url] = s['website'].strip() days = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] opening = [] if 'openingHours' in s and s['openingHours'] is not None: for m in re.finditer(ur'i:(\d);s:\d+:\\?"([^\\"]+?)\\?"', s['openingHours']): opening.append('%s: %s' % (days[string.atoi(m.group(1))], m.group(2).strip())) entry[cm.hours] = ', '.join(opening) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores')
def fetch_store_detail(s, data, isOfficial=False): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = cm.html2plain(s['name']).strip() entry[cm.country_e] = data['country'] val = cm.html2plain(s['city']).strip().upper() entry[cm.city_e] = cm.extract_city(val if val and val != '' else data['city'])[0] entry[cm.addr_e] = cm.html2plain(s['address']).strip() entry[cm.email] = s['email'].strip() entry[cm.tel] = s['phone'].strip() entry[cm.fax] = s['fax'].strip() entry[cm.store_class] = 'Official Retailer' if isOfficial else 'Retailer' try: entry[cm.lat] = string.atof(s['lat']) if s['lat'] != '' else '' except (ValueError, KeyError, TypeError) as e: cm.dump('Error in fetching lat: %s' % str(e), log_name) try: entry[cm.lng] = string.atof(s['lng']) if s['lng'] != '' else '' except (ValueError, KeyError, TypeError) as e: cm.dump('Error in fetching lng: %s' % str(e), log_name) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) return entry
def fetch_stores(data): body = data['body'] start = body.find(u'<ul class="storelist storelist_%s' % data['code']) if start == -1: cm.dump('Error in finding stores for %s' % data['code']) return [] body = cm.extract_closure(body[start:], ur'<ul\b', ur'</ul>')[0] store_list = [] for m in re.findall(ur'<li class="sitem">(.+?)</li>', body, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m1 = re.search(ur'<h3>(.+?)</h3>', m) if m1 is not None: entry[cm.name_c] = m1.group(1).strip() m1 = re.search(ur'<div class="addr">(.+?)</div>', m) if m1 is not None: entry[cm.addr_e] = m1.group(1).replace(u'地址:', '').replace(u'地址:', '').strip() m1 = re.search(ur'<div class="tel">(.+?)</div>', m) if m1 is not None: entry[cm.tel] = m1.group(1).replace(u'电话:', '').replace(u'电话:', '').strip() entry[cm.city_c] = data['city'] ret = gs.look_up(data['city'], 3) if ret is not None: entry[cm.city_e] = ret['name_e'] entry[cm.city_c] = ret['name_c'] if ret['province'] != '': entry[cm.province_e] = ret['province']['name_e'] entry[cm.country_e] = u'CHINA' gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), 'canali_log.txt') db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_store_details(data): # http://maps.oasis-stores.com/index-v2.php?coutnryISO=GB&brand=oasis&lat=51.42014&lng=-0.20954 url = data['store_url'] code = data['country_code'] city = data['city_e'] try: html = cm.get_data(url, { 'latitude': data['lat'], 'longitude': data['lng'], 'brand': 'oasis' }) except Exception: print 'Error occured: %s' % url dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] raw = json.loads(html) entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = raw['name'] addr_list = [] for i in xrange(1, 4): tmp = cm.html2plain(raw['address%d' % i]).strip() if tmp != '': addr_list.append(tmp) entry[cm.addr_e] = ', '.join(addr_list) state = raw['countryRegion'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() state = raw['state'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() state = raw['county'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() entry[cm.zip_code] = raw['postcode'] entry[cm.country_e] = data['country_e'] entry[cm.city_e] = cm.extract_city(data['city_e'])[0] entry[cm.lat] = string.atof(data['lat']) entry[cm.lng] = string.atof(data['lng']) entry[cm.tel] = raw['phone'] entry[cm.email] = raw['email'] tmp = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] entry[cm.hours] = ', '.join([raw[d + '_open_times'] for d in tmp]) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return [entry]
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching stores: %s' % url, log_name) return [] store_list = [] for m in re.finditer(ur'<item id="\d+">', body): sub = cm.extract_closure(body[m.start():], ur'<item\b', ur'</item>')[0] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m1 = re.search(ur'<country>([^<>]+)</country>', sub) if m1 is not None: tmp = m1.group(1).split('/') for v in tmp: ret = gs.look_up(v.strip().upper(), 1) if ret is not None: entry[cm.country_e] = ret['name_e'] break m1 = re.search(ur'<city>([^<>]+)</city>', sub) if m1 is not None: val = cm.reformat_addr(m1.group(1)) if entry[cm.country_e] == 'UNITED STATES': tmp_list = tuple(tmp.strip() for tmp in cm.reformat_addr(val).strip(',')) if len(tmp_list) == 2: if re.search('[A-Z]{2}', tmp_list[1]): entry[cm.province_e] = tmp_list[1] entry[cm.city_e] = cm.extract_city(m1.group(1))[0] m1 = re.search(ur'<brands>([^<>]+)</brands>', sub) if m1 is not None: tmp = m1.group(1).split('/') brand_list = [] for v in tmp: if v.strip() != '': brand_list.append(v) entry[cm.store_type] = ', '.join(brand_map[key] for key in brand_list) m1 = re.search(ur'<name>([^<>]+)</name>', sub) if m1 is not None: entry[cm.name_e] = m1.group(1).strip() m1 = re.search(ur'<address>([^<>]+)</address>', sub) if m1 is not None: entry[cm.addr_e] = cm.reformat_addr(m1.group(1)) m1 = re.search(ur'<tel>([^<>]+)</tel>', sub) if m1 is not None: entry[cm.tel] = m1.group(1).strip() m1 = re.search(ur'sll=(-?\d+\.\d+),(-?\d+\.\d+)', sub) if m1 is not None: entry[cm.lat] = string.atof(m1.group(1)) entry[cm.lng] = string.atof(m1.group(2)) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None: entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def get_detailed_store(html, store_cat): store_list = [] start = 0 while True: sub_html, start, end = common.extract_closure(html, ur"<li\b", ur"</li>") if end == 0: break # 得到单个门店的页面代码 html = html[end:] entry = common.init_store_entry(brand_id, brandname_e, brandname_c) m = re.findall(ur'<div class="store-title -h3a">(.+?)</div>', sub_html) if len(m) > 0: entry[common.name_e] = common.reformat_addr(m[0]) m = re.findall(ur'<div class="store-address">(.+?)</div>', sub_html, re.S) if len(m) > 0: addr = common.reformat_addr(m[0]) # 最后一行是否为电话号码? terms = addr.split(", ") tel = common.extract_tel(terms[-1]) if tel != "": addr = ", ".join(terms[:-1]) entry[common.tel] = tel entry[common.addr_e] = addr # 获得门店类型 # store_type = [store_cat] type_html, type_start, type_end = common.extract_closure(sub_html, ur'<ul class="service-list">', ur"</ul>") if type_end != 0: store_type = [m for m in re.findall(ur'<li class="service-item">(.+?)</li>', type_html)] store_type.insert(0, store_cat) entry[common.store_type] = ", ".join(store_type) else: entry[common.store_type] = store_cat # 获得经纬度 m = re.findall(ur'data-latitude="(-?\d+\.\d+)"', sub_html) if len(m) > 0: entry[common.lat] = string.atof(m[0]) m = re.findall(ur'data-longitude="(-?\d+\.\d+)"', sub_html) if len(m) > 0: entry[common.lng] = string.atof(m[0]) entry[common.city_e] = common.extract_city(data[common.city_e])[0] entry[common.country_e] = common.reformat_addr(data[common.country_e]).strip().upper() gs.field_sense(entry) print "%s: Found store: %s, %s (%s, %s, %s)" % ( brandname_e, entry[common.name_e], entry[common.addr_e], entry[common.city_e], entry[common.country_e], entry[common.continent_e], ) db.insert_record(entry, "stores") store_list.append(entry) return store_list
def fetch_stores(data): param = {'action': 'getStoresFromAjax', 'country': data['country_code'], 'region': data['city'], 'collection': ''} url = data['url'] try: body = cm.post_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] store_list = [] for m1 in re.finditer(ur'<div class="shop-type-container">', body): sub = cm.extract_closure(body[m1.start():], ur'<div\b', ur'</div>')[0] store_class = '' m2 = re.search(ur'<div class="shop-type-title">(.+?)</div>', sub, re.S) if m2 is not None: store_class = cm.reformat_addr(m2.group(1)) for m2 in re.finditer(ur'<div class="shop"', sub): store_sub = cm.extract_closure(sub[m2.start():], ur'<div\b', ur'</div>')[0] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.store_class] = store_class entry[cm.country_e] = data['country_code'] entry[cm.city_e] = cm.extract_city(data['city'])[0] m3 = re.search(ur'loadStore\((\d+)\s*,\s*(-?\d+\.\d+)\s*,\s*(-?\d+\.\d+)\)', store_sub) if m3 is not None: data['store_id'] = string.atoi(m3.group(1)) entry[cm.lat] = string.atof(m3.group(2)) entry[cm.lng] = string.atof(m3.group(3)) entry[cm.store_type] = ', '.join(get_detail(data)) m3 = re.search(ur'<div class="shop-name shop-details shop-main-name">([^<>]+)</div>', store_sub) if m3 is not None: entry[cm.name_e] = m3.group(1).strip() addr_list = [] m3 = re.search(ur'<div class="shop-street shop-details">([^<>]+)</div>', store_sub) if m3 is not None: addr_list.append(cm.reformat_addr(m3.group(1))) m3 = re.search(ur'<div class="shop-city shop-details">([^<>]+)</div>', store_sub) if m3 is not None: tmp = cm.reformat_addr(m3.group(1)) m3 = re.search(ur'(\d{4,})', tmp) if m3 is not None: entry[cm.zip_code] = m3.group(1).strip() addr_list.append(tmp) entry[cm.addr_e] = ', '.join(addr_list) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def get_store_details(data): url = data['url'] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': brand_id} cm.dump(dump_data) return [] entry = cm.init_store_entry(brand_id, brandname_e, brandname_c) entry[cm.name_e] = data['name'] entry[cm.url] = data['url'] start = html.find(ur'<div class="storelocator-breadcrumbs">') if start == -1: return [] sub, start, end = cm.extract_closure(html[start:], ur'<ul>', ur'</ul>') if end == 0: return [] # 最后一个<li>...</li> m = re.findall(ur'<li>(.+?)</li>', sub, re.S) if len(m) > 0: entry[cm.addr_e] = cm.reformat_addr(m[-1]) # 经纬度 m = re.findall(ur'position: new google.maps.LatLng\((-?\d+\.\d+).*?(-?\d+\.\d+)\)', html) if len(m) > 0: cm.update_entry(entry, {cm.lat: string.atof(m[0][0]), cm.lng: string.atof(m[0][1])}) m = re.search(ur'<div class="contact right">(.+?)</div>', html, re.S) if m is not None: contact_sub = m.group(1) pat_tel = re.compile(ur'<p class="phone">(.+?)</p>') m1 = re.search(pat_tel, contact_sub) if m1: entry[cm.tel] = cm.extract_tel(m1.group(1)) contact_sub = re.sub(pat_tel, '', contact_sub) hours_list=[tmp.strip() for tmp in cm.reformat_addr(contact_sub).split(',')] if 'opening hours' in hours_list[0].lower(): del hours_list[0] entry[cm.hours] = ', '.join(hours_list) # Geo country = data['country'] city = data['city'] cm.update_entry(entry, {cm.country_e: country, cm.city_e: city}) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( brandname_e, brand_id, entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return entry
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching stores: %s' % url, log_name) return [] store_list = [] for m1 in re.finditer(ur'<lignecountry\s+titre\s*=\s*"([^"]+)"', body): country = m1.group(1).strip().upper() if country == 'U.S.A.': country = 'US' sub_country = cm.extract_closure(body[m1.start():], ur'<lignecountry\b', ur'</lignecountry>')[0] for m2 in re.finditer(ur'<lignecity\s+titre\s*=\s*"([^"]+)"', sub_country): city = m2.group(1).strip().upper() sub_city = cm.extract_closure(sub_country[m2.start():], ur'<lignecity\b', ur'</lignecity>')[0] m3 = re.search(ur'<!\[CDATA\[(.+?)\]\]>', sub_city, re.S) if m3 is None: continue sub_city = m3.group(1) store_subs = re.split(ur'<\s*h2\s*>\s*LANVIN BOUTIQUE\s*<\s*/h2\s*>', sub_city) for s in store_subs: if s.strip() == '': continue m4 = re.search(ur'<p>(.+?)</p>', s, re.S) if m4 is None: continue entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = country entry[cm.city_e] = city s = m4.group(1) m4 = re.search(ur'(.+?)\n\s*\n', s, re.S) if m4 is not None: entry[cm.addr_e] = cm.reformat_addr(m4.group(1)) m4 = re.search(ur'Phone:(.+?)\n\s*\n', s, re.S) if m4 is not None: entry[cm.tel] = cm.reformat_addr(m4.group(1).strip()) m4 = re.search(ur'Boutique Hours:(.+?)\n\s*\n', s, re.S) if m4 is not None: entry[cm.hours] = cm.reformat_addr(m4.group(1).strip()) m4 = re.search(ur'Products available:(.+?)\n\s*\n', s, re.S) if m4 is not None: entry[cm.store_type] = m4.group(1).strip() m4 = re.search(ur'Email:\s*<a href="mailto:([^"]+)">', s) if m4 is not None: entry[cm.email] = m4.group(1).strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): url = data["host"] + data["country_url"] % data["country_id"] try: body = cm.get_data(url) except Exception: cm.dump("Error in fetching countries: %s" % url, log_name) return [] raw = json.loads(body)["rawPos"] store_list = [] for s in raw: entry = cm.init_store_entry(data["brand_id"], data["brandname_e"], data["brandname_c"]) addr_list = [] for tmp2 in [cm.html2plain(s[tmp1]).strip() for tmp1 in ["address%d" % v for v in xrange(1, 5)]]: if tmp2 != "": addr_list.append(tmp2) entry[cm.addr_e] = ", ".join(addr_list) entry[cm.city_e] = cm.extract_city(s["city"]["name"])[0] entry[cm.country_e] = s["country"]["countryCode"] entry[cm.email] = s["email"] entry[cm.fax] = s["fax"] if s["latitude"] != "": entry[cm.lat] = string.atof(s["latitude"]) if s["longitude"] != "": entry[cm.lng] = string.atof(s["longitude"]) entry[cm.hours] = cm.reformat_addr(s["openingSchedule"]) phone_list = [] for key in ["phone1", "phone2"]: if s[key].strip() != "": phone_list.append(s[key].strip()) entry[cm.tel] = ", ".join(phone_list) entry[cm.zip_code] = s["postalCode"] entry[cm.name_e] = s["shopName"] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == "": entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == "": entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( "(%s / %d) Found store: %s, %s (%s, %s)" % ( data["brandname_e"], data["brand_id"], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e], ), log_name, ) db.insert_record(entry, "stores") store_list.append(entry) return store_list
def fetch_uk(body, data): start = body.find(u'<div class="fableft">') if start == -1: print "Error in finding %s stores" % data["name"] return [] body, start, end = cm.extract_closure(body[start:], ur"<div\b", ur"</div>") if end == 0: print "Error in finding %s stores" % data["name"] return [] store_list = [] for m in re.findall(ur"<div>\s*(.+?)\s*</div>", body, re.S): entry = cm.init_store_entry(data["brand_id"], data["brandname_e"], data["brandname_c"]) entry[cm.country_e] = data["name"] addr_list = re.findall(ur"<p>\s*(.+?)\s*</p>", m) tel = cm.extract_tel(addr_list[-1]) if tel != "": entry[cm.tel] = tel del addr_list[-1] if data["name"] == "AUSTRALIA": country, province, city = gs.addr_sense(", ".join(addr_list), data["name"]) if city is not None: entry[cm.city_e] = city if province is not None: entry[cm.province_e] = province else: city = addr_list[-2].strip().upper() entry[cm.city_e] = city ret = gs.look_up(city, 3) if ret is not None and ret["country"]["name_e"] == gs.look_up("UK", 1)["name_e"]: entry[cm.city_e] = ret["name_e"] entry[cm.zip_code] = addr_list[-1].strip().upper() entry[cm.addr_e] = ", ".join(addr_list) entry[cm.city_e] = cm.extract_city(entry[cm.city_e])[0] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == "": entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == "": entry[cm.city_e] = ret[2] gs.field_sense(entry) print "(%s / %d) Found store: %s, %s (%s, %s, %s)" % ( data["brandname_e"], data["brand_id"], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e], ) db.insert_record(entry, "stores") store_list.append(entry)
def get_detailed_store(html, store_cat): store_list = [] start = 0 while True: sub_html, start, end = common.extract_closure(html, ur'<li\b', ur'</li>') if end == 0: break # 得到单个门店的页面代码 html = html[end:] entry = common.init_store_entry(brand_id, brandname_e, brandname_c) m = re.findall(ur'<div class="store-title -h3a">(.+?)</div>', sub_html) if len(m) > 0: entry[common.name_e] = common.reformat_addr(m[0]) m = re.findall(ur'<div class="store-address">(.+?)</div>', sub_html, re.S) if len(m) > 0: addr = common.reformat_addr(m[0]) # 最后一行是否为电话号码? terms = addr.split(', ') tel = common.extract_tel(terms[-1]) if tel != '': addr = ', '.join(terms[:-1]) entry[common.tel] = tel entry[common.addr_e] = addr # 获得门店类型 # store_type = [store_cat] type_html, type_start, type_end = common.extract_closure(sub_html, ur'<ul class="service-list">', ur'</ul>') if type_end != 0: store_type = [m for m in re.findall(ur'<li class="service-item">(.+?)</li>', type_html)] store_type.insert(0, store_cat) entry[common.store_type] = ', '.join(store_type) else: entry[common.store_type] = store_cat # 获得经纬度 m = re.findall(ur'data-latitude="(-?\d+\.\d+)"', sub_html) if len(m) > 0: entry[common.lat] = string.atof(m[0]) m = re.findall(ur'data-longitude="(-?\d+\.\d+)"', sub_html) if len(m) > 0: entry[common.lng] = string.atof(m[0]) entry[common.city_e] = common.extract_city(data[common.city_e])[0] entry[common.country_e] = common.reformat_addr(data[common.country_e]).strip().upper() gs.field_sense(entry) print '%s: Found store: %s, %s (%s, %s, %s)' % ( brandname_e, entry[common.name_e], entry[common.addr_e], entry[common.city_e], entry[common.country_e], entry[common.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['url'] param = {'country_id': data['country_code'], 'city': '', 'label_id': '', 'lang': 'en'} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] start = body.find(ur'<stores>') if start == -1: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] body = cm.extract_closure(body[start:], ur'<stores>', ur'</stores>')[0] store_list=[] for m in re.findall(ur'<store\b[^<>]+>(.+?)</store>', body): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['country_code'] m1 = re.search(ur'<name>(.+?)</name>', m) if m1 is not None: entry[cm.name_e] = cm.reformat_addr(m1.group(1).strip()) m1 = re.search(ur'<address>(.+?)</address>', m) if m1 is not None: entry[cm.addr_e] = cm.reformat_addr(m1.group(1).strip()) m1 = re.search(ur'<city>(.+)</city>', m) if m1 is not None: entry[cm.city_e] = cm.extract_city(m1.group(1))[0] m1 = re.search(ur'<zip>(.+?)</zip>', m) if m1 is not None: entry[cm.zip_code] = m1.group(1).strip() m1 = re.search(ur'<tel>(.+?)</tel>', m) if m1 is not None: entry[cm.tel] = m1.group(1).strip() m1 = re.search(ur'<fax>(.+?)</fax>', m) if m1 is not None: entry[cm.fax] = m1.group(1).strip() m1 = re.search(ur'<email>(.+?)</email>', m) if m1 is not None: entry[cm.email] = m1.group(1).strip() m1 = re.search(ur'<link>(.+?)</link>', m) if m1 is not None: entry[cm.url] = m1.group(1).strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): """ 获得商店信息 :param data: :return: """ url = data["post_url"] try: js = json.loads( cm.post_data( url, {"country_id": data["country_id"], "retail_city": "", "retail_type": data["retail_type"]} ).decode("unicode_escape") ) except Exception: print "Error occured in getting country list: %s" % url dump_data = {"level": 1, "time": cm.format_time(), "data": {"url": url}, "brand_id": data["brand_id"]} cm.dump(dump_data) return [] # country_id=108&retail_city=&retail_type=retail # country_id=99&retail_city=&retail_type=service store_list = [] for s in js: entry = cm.init_store_entry(data["brand_id"], data["brandname_e"], data["brandname_c"]) tmp = s["retail_name"].strip() if cm.is_chinese(tmp): entry[cm.name_c] = tmp else: entry[cm.name_e] = tmp entry[cm.addr_e] = s["retail_gmap"].strip() entry[cm.zip_code] = s["retail_zipcode"].strip() entry[cm.city_e] = s["retail_city"].strip().upper() if s["retail_email"] is not None: entry[cm.email] = s["retail_email"].strip() if s["retail_website"] is not None: entry[cm.url] = s["retail_website"].strip() if data["retail_type"] == "retail": entry[cm.store_class] = "Retail" else: entry[cm.store_class] = "Service Center" entry[cm.country_e] = s["country_name"].strip().upper() entry[cm.continent_e] = s["continent_name"].strip().upper() gs.field_sense(entry) print "(%s / %d) Found store: %s, %s (%s, %s)" % ( data["brandname_e"], data["brand_id"], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e], ) store_list.append(entry) db.insert_record(entry, "stores") return store_list
def fetch(level=1, data=None, user='******', passwd=''): db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', brand_id)) try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = { 'level': 1, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': brand_id } cm.dump(dump_data) return [] js = json.loads(html) store_list = [] for s in js['data']['list']: entry = cm.init_store_entry(brand_id, brandname_e, brandname_c) cm.update_entry( entry, { cm.lat: string.atof(s['geo']['lat']), cm.lng: string.atof(s['geo']['lng']) }) entry[cm.name_e] = s['contact']['title'] entry[cm.addr_e] = cm.reformat_addr(s['contact']['address']) entry[cm.tel] = s['contact']['phone'] entry[cm.fax] = s['contact']['fax'] entry[cm.hours] = cm.reformat_addr(s['contact']['hours']) entry[cm.store_type] = s['contact']['selling'] entry[cm.url] = host + s['link'] gs.update_city_map(s['city'], s['country'], s['continent']) cm.update_entry( entry, { cm.continent_e: s['continent'], cm.country_e: s['country'], cm.city_e: s['city'] }) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( brandname_e, brand_id, entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) db.disconnect_db() gs.commit_maps(1) gs.commit_maps(3) return store_list
def fetch_stores(data): """ 获得商店信息 :param data: """ url = data['post_url'] try: html = cm.post_data(url, { 'pid': data['city_id'], 'lang': 'en', 'action': 'popola_box_DX' }) except Exception: print 'Error occured in getting city list: %s' % url dump_data = { 'level': 2, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] store_list = [] for m in re.finditer(ur'<a href="(.+?)".*?>', html): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.url] = m.group(1) store_html, start, end = cm.extract_closure(html[m.start():], ur'<a href', ur'</a>') if end == 0: continue m1 = re.findall(ur'<h3 class="titleShop">(.+?)</h3>', store_html, re.S) if len(m1) > 0: entry[cm.name_e] = m1[0].strip() m1 = re.findall(ur'<p\b.*?>(.+?)(?:</p>|</div>)', store_html, re.S) if len(m1) > 0: terms = cm.reformat_addr(m1[0]).split(',') tel = cm.extract_tel(terms[-1]) if tel != '': terms = terms[:-1] entry[cm.tel] = tel entry[cm.addr_e] = ', '.join([v.strip() for v in terms]) entry['country_e'] = data['country_e'] entry['city_e'] = data['city_e'] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores')
def f(m): store_name = m[0].strip() addr_str = m[1].strip() spl = addr_str.split('<br/>') store_type = cm.html2plain(spl[0].strip()) store_addr = spl[1].strip() hour_idx = 2 store_tel = '' for i in xrange(2, len(spl)): # If this is not a phone number: tel = cm.extract_tel(spl[i]) if tel == '': store_addr += ', ' + spl[i] hour_idx = i + 1 else: store_tel = spl[i].strip() hour_idx = i + 1 break if hour_idx < len(spl): store_hour = cm.html2plain(', '.join(spl[hour_idx:])).strip() else: store_hour = '' # store_addr = cm.reformat_addr('\r\n'.join([val.strip() for val in spl[1:-3]])) store_addr = cm.reformat_addr(store_addr) store_entry = cm.init_store_entry(brand_id, brandname_e, brandname_c) cm.update_entry(store_entry, {cm.continent_e: opt[cm.continent_e].strip().upper(), cm.city_e: opt[cm.city_e].strip().upper(), cm.country_e: opt[cm.country_e].strip().upper(), cm.name_e: cm.name_e, cm.addr_e: store_addr, cm.store_type: store_type, cm.hours: store_hour, cm.tel: store_tel}) if opt.has_key(cm.province_e): store_entry[cm.province_e] = opt[cm.province_e] else: store_entry[cm.province_e] = '' store_entry[cm.city_e] = cm.extract_city(store_entry[cm.city_e])[0] gs.field_sense(store_entry) ret = gs.addr_sense(store_entry[cm.addr_e], store_entry[cm.country_e]) if ret[1] is not None and store_entry[cm.province_e] == '': store_entry[cm.province_e] = ret[1] if ret[2] is not None and store_entry[cm.city_e] == '': store_entry[cm.city_e] = ret[2] gs.field_sense(store_entry) print '%s Found store: %s, %s (%s, %s)' % ( brandname_e, store_entry[cm.name_e], store_entry[cm.addr_e], store_entry[cm.country_e], store_entry[cm.continent_e]) db.insert_record(store_entry, 'stores') return store_entry
def fetch_stores(data): url = data['store_url'] param = {'store_country': data['country_code'], 'store_city': data['city_code']} try: body = cm.post_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] store_list = [] for s in re.findall(ur'<marker\b([^<>]+)/\s*>', body): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m = re.search(ur'store_name="([^"]+)"', s) if m is not None: entry[cm.name_e] = cm.reformat_addr(m.group(1)) entry[cm.country_e] = data['country_code'] entry[cm.city_e] = data['city'] addr_list = [] for key in ['store_mall_name', 'store_address', 'store_zip_code']: m = re.search(ur'%s="([^"]+)"' % key, s) if m is not None: tmp = cm.reformat_addr(m.group(1)) if tmp != '': addr_list.append(tmp) entry[cm.addr_e] = ', '.join(addr_list) m = re.search(ur'store_zip_code="([^"]+)"', s) if m is not None: entry[cm.zip_code] = m.group(1).strip() m = re.search(ur'store_telephone="([^"]+)"', s) if m is not None: entry[cm.tel] = m.group(1).strip() m = re.search(ur'store_fax="([^"]+)"', s) if m is not None: entry[cm.fax] = m.group(1).strip() m = re.search(ur'store_email="([^"]+)"', s) if m is not None: entry[cm.email] = m.group(1).strip() m = re.search(ur'store_latitude="([^"]+)"', s) if m is not None: entry[cm.lat] = string.atof(m.group(1).strip()) m = re.search(ur'store_longitude="([^"]+)"', s) if m is not None: entry[cm.lng] = string.atof(m.group(1).strip()) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_store_details(data): url = data['host'] + data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching store details: %s' % url, log_name) return [] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) start = body.find(ur'<h3>available in store</h3>') if start != -1: type_sub = cm.extract_closure(body[start:], ur'<ul\b', ur'</ul>')[0] entry[cm.store_type] = ', '.join( cm.html2plain(tmp).strip() for tmp in re.findall(ur'<li[^<>]*>(.+?)</li>', type_sub, re.S)) start = body.find(ur"<div class='gmap_info_box'") if start == -1: cm.dump('Error in fetching store details: %s' % url, log_name) return [] body = cm.extract_closure(body[start:], ur'<div\b', ur'</div>')[0] raw = json.loads(cm.extract_closure(body, ur'\{', ur'\}')[0])['table'] entry[cm.name_e] = cm.html2plain(raw['name']) entry[cm.city_e] = data['city'].strip().upper() entry[cm.country_e] = data['country'].strip().upper() # entry[cm.store_type] = data['store_type'] entry[cm.addr_e] = cm.reformat_addr(raw['address']) m = re.search(re.compile(ur'phone:(.*?)fax:(.*?)', re.I | re.S), raw['phone']) if m is not None: entry[cm.tel] = m.group(1).strip() entry[cm.fax] = m.group(2).strip() else: m = re.search(re.compile(ur'phone:(.*?)', re.I | re.S), raw['phone']) if m is not None: entry[cm.tel] = m.group(1).strip() m = re.search(re.compile(ur'fax:(.*?)', re.I | re.S), raw['phone']) if m is not None: entry[cm.fax] = m.group(1).strip() entry[cm.hours] = raw['hours'] if raw['lat'] is not None and raw['lat'] != '': entry[cm.lat] = string.atof(raw['lat']) if raw['lng'] is not None and raw['lng'] != '': entry[cm.lat] = string.atof(raw['lng']) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None: entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') return [entry]
def fetch_stores(data): """ 获得门店的详细信息 :rtype : [entries] :param data: """ try: html = cm.get_data(data['url']) except Exception: print 'Error occured: %s' % url dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': brand_id} cm.dump(dump_data) return [] entries = [] start = html.find(u'<ul class="store-list">') if start == -1: return entries start += len(u'<ul class="store-list">') end = html.find(u'</ul>', start) html = html[start:end] for m1 in re.findall(ur'<li class="(.*?)">(.*?)</li>', html, re.S): store = cm.init_store_entry(brand_id, brandname_e, brandname_c) store[cm.store_type] = m1[0] sub_html = m1[1] m2 = re.findall(ur'<h3 class="store-name">(.*?)</h3>', sub_html) if len(m2) > 0: store[cm.name_e] = cm.reformat_addr(m2[0]) m2 = re.findall(ur'<p class="store-address">(.*?)</p>', sub_html, re.S) if len(m2) > 0: store[cm.addr_e] = cm.reformat_addr(m2[0]) cm.update_entry(store, {cm.continent_e: data[cm.continent_e].strip().upper(), cm.country_e: data[cm.country_e].strip().upper(), cm.city_e: data[cm.city_e].strip().upper()}) entry = store gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) entry[cm.city_e] = cm.extract_city(entry[cm.city_e])[0] print '%s: Found store: %s, %s (%s, %s)' % ( brandname_e, store[cm.name_e], store[cm.addr_e], store[cm.country_e], store[cm.continent_e]) db.insert_record(store, 'stores') entries.append(store)
def fetch_hk(data): loc_list = ('Hong Kong', 'Kowloon', 'Macau', 'New Territories') url = 'http://levi.com.hk/hk/storelocator' store_list = [] for loc in loc_list: param = {'loc': loc} try: body = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching stores: %s' % param, log_name) continue start = body.find(ur'<div id="addWrapper">') if start == -1: cm.dump('Error in fetching stores: %s' % param, log_name) continue sub = cm.extract_closure(body[start:], ur'<ul>', ur'</ul>')[0] for s in re.findall(ur'<li>(.+?)</li>', sub, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = 'MACAU' if loc == 'Macau' else 'HONG KONG' entry[cm.city_e] = entry[cm.country_e] m = re.search(ur'<div id="addStore">([^<>]+)', s) entry[cm.addr_e] = cm.html2plain(m.group(1)) if m else '' m = re.search(ur'<div id="addAddress">([^<>]+)', s) tmp = cm.html2plain(m.group(1)) pat = re.compile(ur'business hours?\s*[:\.]?\s*', re.I) if re.search(pat, tmp): entry[cm.hours] = re.sub(pat, '', tmp).strip() m = re.search(ur'<div id="addPhone">([^<>]+)', s) tmp = cm.html2plain(m.group(1)) pat = re.compile(ur'(tel|phone|telephone)?\s*[:\.]?\s*', re.I) if re.search(pat, tmp): entry[cm.tel] = re.sub(pat, '', tmp).strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): """ 获得商店信息 :param data: """ url = data['url'] try: info = json.loads(cm.get_data(url, {'tskay': data['key_term']})) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw_list = info['shops'] store_list = [] for s in raw_list: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.city_e] = s['city'].strip().upper() entry[cm.country_e] = data['country_e'].strip().upper() entry[cm.name_e] = s['name'].strip() addr = s['address'] entry[cm.addr_e] = addr terms = addr.split(',') if len(terms) > 1 and entry[cm.city_e] in terms[-1].strip().upper(): country = entry['country_e'] tmp = gs.look_up(country, 1) if tmp is not None: country = tmp['name_e'] if country == 'JAPAN': # 日本邮编 m = re.search(ur'\d{3,}[ -\.]+?\d{3,}', terms[-1]) if m is not None: entry[cm.zip_code] = m.group(0) else: m = re.search(ur'\d{4,}', terms[-1]) if m is not None: entry[cm.zip_code] = m.group(0) entry[cm.tel] = s['tel'] entry[cm.fax] = s['fax'] entry[cm.email] = s['email'] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['url'] param = { 'action': 'getStoresByCity', 'idCity': data['city_id'], 'filter': 'clothing;lacoste%20l!ve' } try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] raw = json.loads(body)['root']['DATA']['stores'] store_list = [] for s in [tmp['store'] for tmp in raw]: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = s['name'].strip() entry[cm.country_e] = data['country_code'] entry[cm.addr_e] = cm.html2plain(s['address']).strip() entry[cm.store_type] = s['category'].strip() entry[cm.city_e] = cm.extract_city(s['city'])[0] if s['email'] is not None: entry[cm.email] = s['email'].strip() if s['fax'] is not None: entry[cm.fax] = s['fax'].strip() if s['infoHours'] is not None: entry[cm.hours] = s['infoHours'].strip() if s['latitude'] is not None and s['latitude'].strip() != '': entry[cm.lat] = string.atof(s['latitude']) if s['longitude'] is not None and s['longitude'].strip() != '': entry[cm.lat] = string.atof(s['longitude']) if s['phone'] is not None: entry[cm.tel] = s['phone'].strip() if s['postCode'] is not None: entry[cm.zip_code] = s['postCode'].strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = '%s/en/shops/%s.json' % ( data['host'], urllib.quote(data['country_id'].encode('utf-8'))) try: body = cm.get_data(url) raw = json.loads(body) except Exception: cm.dump('Error in fetching stores: %s' % url, 'unode50_log.txt') dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] store_items = [] for s in raw['distributors']: s['store_class'] = 'distributor' store_items.append(s) for s in raw['shops']: s['store_class'] = 'shop' store_items.append(s) store_list = [] for s in store_items: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.store_class] = s['store_class'] entry[cm.country_e] = data['country'] entry[cm.name_e] = s['title'] if s['lat'] is not None: entry[cm.lat] = string.atof(str(s['lat'])) if s['lng'] is not None: entry[cm.lng] = string.atof(str(s['lng'])) entry[cm.addr_e] = s['address'] entry[cm.city_e] = cm.extract_city(s['city'])[0] entry[cm.tel] = s['phone'] entry[cm.zip_code] = s['postal_code'] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]), 'unode50_log.txt') db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['host'] + data['data_url'] if data['province_code'] != '': param = { 'region': data['continent'], 'country': data['country_code'], 'state': data['province_code'] } else: param = {'region': data['continent'], 'country': data['country_code']} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] raw = json.loads(body)['stores'] store_list = [] for s in raw: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = s['name'] entry[cm.province_e] = data['province_code'].strip().upper() entry[cm.country_e] = data['country_code'].strip().upper() entry[cm.city_e] = s['city'].strip().upper() addr_list = [] for tmp in ['address%d' % idx for idx in xrange(1, 4)]: if tmp in s and s[tmp] is not None and s[tmp].strip() != '': addr_list.append(s[tmp].strip()) entry[cm.addr_e] = ', '.join(addr_list) entry[cm.tel] = s['phone'] entry[cm.zip_code] = s['postalCode'] hours = '' for item in s['openingHours']: hours += '%s %s' % (item['days'], item['openingHours']) entry[cm.hours] = hours entry[cm.url] = s['storePageUrl'] coords = s['coords'] entry[cm.lat] = string.atof(coords['lat']) entry[cm.lng] = string.atof(coords['lng']) gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['post_shops'] param = {'city': data['city_e'], 'paulandjoe_women': 0, 'paulandjoe_man': 0, 'paulandjoe_sister': 0, 'paulandjoe_little': 0, 'paulandjoe_beauty': 0} try: html = cm.post_data(url, param) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] store_list = [] try: for store in (pq(tmp) for tmp in pq(html)('ul')): try: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = cm.html2plain(store('li.first')[0].text).strip() entry[cm.country_e] = data[cm.country_e] entry[cm.city_e] = data[cm.city_e] addr_list = [] for term in (cm.reformat_addr(unicode(pq(tmp))) for tmp in store('li[class!="first"]')): if term != '': addr_list.append(term) tel = cm.extract_tel(addr_list[-1]) if tel != '': entry[cm.tel] = tel del addr_list[-1] entry[cm.addr_e] = ', '.join(addr_list) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s/%d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores') except (IndexError, TypeError) as e: cm.dump(u'Error in parsing %s, %s' % (url, param), log_name) print traceback.format_exc() continue except Exception, e: print traceback.format_exc()
def fetch_stores(data): url = data['data_url'] param = {'output': 'json', 'country': data['country_code'], 'brand': 'dkny'} page = 0 tot_page = -1 store_list = [] while True: page += 1 if tot_page != -1 and page > tot_page: break param['p'] = page try: body = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return () raw = json.loads(body) tot_page = raw['Stores']['TotalPages'] if data['country_code'] not in region_map: # 构造州列表 region_map[data['country_code']] = dict((item['RegionId'], item['Name']) for item in raw['Regions']) for s in raw['Stores']['Items']: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['country_code'].upper() entry[cm.city_e] = cm.extract_city(s['City'])[0] entry[cm.name_e] = cm.html2plain(s['Name']).strip() entry[cm.addr_e] = cm.reformat_addr(s['Address']) entry[cm.tel] = s['Phone'].strip() if s['Phone'] else '' entry[cm.fax] = s['Fax'].strip() if s['Fax'] else '' entry[cm.email] = s['Email'].strip() if s['Email'] else '' entry[cm.lat] = s['Latitude'] if s['Latitude'] else '' entry[cm.lng] = s['Longitude'] if s['Longitude'] else '' region_id = s['RegionId'] if region_id in region_map[data['country_code']]: entry[cm.province_e] = cm.html2plain(region_map[data['country_code']][region_id]).strip().upper() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_uk(body, data): start = body.find(u'<div class="fableft">') if start == -1: print 'Error in finding %s stores' % data['name'] return [] body, start, end = cm.extract_closure(body[start:], ur'<div\b', ur'</div>') if end == 0: print 'Error in finding %s stores' % data['name'] return [] store_list = [] for m in re.findall(ur'<div>\s*(.+?)\s*</div>', body, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['name'] addr_list = re.findall(ur'<p>\s*(.+?)\s*</p>', m) tel = cm.extract_tel(addr_list[-1]) if tel != '': entry[cm.tel] = tel del addr_list[-1] if data['name'] == 'AUSTRALIA': country, province, city = gs.addr_sense(', '.join(addr_list), data['name']) if city is not None: entry[cm.city_e] = city if province is not None: entry[cm.province_e] = province else: city = addr_list[-2].strip().upper() entry[cm.city_e] = city ret = gs.look_up(city, 3) if ret is not None and ret['country']['name_e'] == gs.look_up('UK', 1)['name_e']: entry[cm.city_e] = ret['name_e'] entry[cm.zip_code] = addr_list[-1].strip().upper() entry[cm.addr_e] = ', '.join(addr_list) entry[cm.city_e] = cm.extract_city(entry[cm.city_e])[0] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): url = data['store_url'] param = {'myid': data['key'], 'idioma': 'in'} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching countries: %s, %s' % (url, param), log_name) return [] store_list = [] for s in json.loads(body): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['country'] entry[cm.city_e] = cm.extract_city(data['city'])[0] entry[cm.name_e] = cm.reformat_addr(s['title']) m = re.search(ur'(.+?)-\s*<', s['key']) addr_list = [entry[cm.name_e]] if m is not None: m1 = re.search(ur'-+', m.group(1)) if m1 is not None: tmp = [m.group(1)[:m1.start()], m.group(1)[m1.end():]] else: tmp = [m.group(1)] if len(tmp) > 1: entry[cm.tel] = cm.extract_tel(tmp[1]) m1 = re.search(ur'\d{4,}', tmp[0]) if m1 is not None: entry[cm.zip_code] = m1.group() addr_list.append(tmp[0].strip()) entry[cm.addr_e] = ', '.join(addr_list) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) code = data['country_code'] if gs.look_up(code, 1) is None: entry[cm.country_e] = cm.html2plain(data['country']).strip().upper() else: entry[cm.country_e] = code entry[cm.name_e] = data['store_name'] entry[cm.city_e] = cm.extract_city(data['city'])[0] entry[cm.lat] = data['lat'] if data['lat'] is not None else '' entry[cm.lng] = data['lng'] if data['lng'] is not None else '' m = re.search(ur'data-boutique\s*=\s*"%s"' % data['store_id'], data['content']) sub = data['content'][m.end():] m1 = re.search(ur'<li class="isDistributeur[^<>]+>(.+?)</li>', sub) if m1 is not None: entry[cm.store_class] = cm.reformat_addr(m1.group(1)) m1 = re.search(ur'<li class="place-title[^<>]+>(.+?)</li>', sub, re.S) if m1 is not None: entry[cm.addr_e] = cm.reformat_addr(m1.group(1)) m1 = re.search(ur'<li class="contacts[^<>]+>(.+?)</li>', sub, re.S) if m1 is not None: m2 = re.search(ur'<a class="popupLaunch" href="([^"]+)"', m1.group(1)) if m2: entry = fetch_details(data, m2.group(1), entry) m2 = re.search(ur'<p>(.+?)</p>', m1.group(1), re.S) if m2: ct_list = tuple(tmp.strip() for tmp in cm.reformat_addr(m2.group(1)).split(',')) entry[cm.tel] = cm.extract_tel(ct_list[0]) if len(ct_list) > 1: entry[cm.email] = ct_list[1].strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') return tuple(entry)
def fetch_hk(data): loc_list = ('Hong Kong', 'Kowloon', 'Macau', 'New Territories') url = 'http://levi.com.hk/hk/storelocator' store_list = [] for loc in loc_list: param = {'loc': loc} try: body = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching stores: %s' % param, log_name) continue start = body.find(ur'<div id="addWrapper">') if start == -1: cm.dump('Error in fetching stores: %s' % param, log_name) continue sub = cm.extract_closure(body[start:], ur'<ul>', ur'</ul>')[0] for s in re.findall(ur'<li>(.+?)</li>', sub, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = 'MACAU' if loc == 'Macau' else 'HONG KONG' entry[cm.city_e] = entry[cm.country_e] m = re.search(ur'<div id="addStore">([^<>]+)', s) entry[cm.addr_e] = cm.html2plain(m.group(1)) if m else '' m = re.search(ur'<div id="addAddress">([^<>]+)', s) tmp = cm.html2plain(m.group(1)) pat = re.compile(ur'business hours?\s*[:\.]?\s*', re.I) if re.search(pat, tmp): entry[cm.hours] = re.sub(pat, '', tmp).strip() m = re.search(ur'<div id="addPhone">([^<>]+)', s) tmp = cm.html2plain(m.group(1)) pat = re.compile(ur'(tel|phone|telephone)?\s*[:\.]?\s*', re.I) if re.search(pat, tmp): entry[cm.tel] = re.sub(pat, '', tmp).strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): """ 获得商店信息 :param data: :return: """ url = data['post_url'] try: js = json.loads(cm.post_data(url, {'country_id': data['country_id'], 'retail_city': '', 'retail_type': data['retail_type']}).decode('unicode_escape')) except Exception: print 'Error occured in getting country list: %s' % url dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] # country_id=108&retail_city=&retail_type=retail # country_id=99&retail_city=&retail_type=service store_list = [] for s in js: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) tmp = s['retail_name'].strip() if cm.is_chinese(tmp): entry[cm.name_c] = tmp else: entry[cm.name_e] = tmp entry[cm.addr_e] = s['retail_gmap'].strip() entry[cm.zip_code] = s['retail_zipcode'].strip() entry[cm.city_e] = s['retail_city'].strip().upper() if s['retail_email'] is not None: entry[cm.email] = s['retail_email'].strip() if s['retail_website'] is not None: entry[cm.url] = s['retail_website'].strip() if data['retail_type'] == 'retail': entry[cm.store_class] = 'Retail' else: entry[cm.store_class] = 'Service Center' entry[cm.country_e] = s['country_name'].strip().upper() entry[cm.continent_e] = s['continent_name'].strip().upper() gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores') return store_list
def fetch_details(data): url = data[cm.url] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = data[cm.name_e] start = html.find(ur'<div class="field-address">') if start == -1: return [] sub, start, end = cm.extract_closure(html[start:], ur'<div\b', ur'</div>') if end == 0: return [] m1 = re.search(ur'<div class="locality">(.+?)</div>', sub) if m1 is not None: entry[cm.city_e] = cm.extract_city(m1.group(1))[0] m1 = re.search(ur'<div class="postal-code">(.+?)</div>', sub) if m1 is not None: entry[cm.zip_code] = m1.group(1).strip() entry[cm.country_e] = data[cm.country_e] pat = re.compile(ur'<[^<>]+?>', re.S) entry[cm.addr_e] = cm.reformat_addr(re.sub(pat, u'\r\n', sub)) m1 = re.search(ur'<div class="field-telephone"><a href=".+?" class="tel">(.+?)</a></div>', html) if m1 is not None: entry[cm.tel] = m1.group(1).strip() m1 = re.search(ur'<div class="field-opening-hours">\s*<p>(.+?)</p>\s*</div>', html, re.S) if m1 is not None: entry[cm.hours] = cm.reformat_addr(m1.group(1)) m1 = re.search(ur'"coordinates":\[(-?\d+\.\d{4,})\s*,\s*(-?\d+\.\d{4,})\]', html) if m1 is not None: lat = string.atof(m1.group(1)) lng = string.atof(m1.group(2)) cm.update_entry(entry, {cm.lat: lat, cm.lng: lng}) entry[cm.continent_e] = data[cm.continent_e] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return [entry]
def fetch_stores(data): url = data['store_url'] param = {'myid': data['key'], 'idioma': 'in'} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching countries: %s, %s' % (url, param), log_name) return [] store_list = [] for s in json.loads(body): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['country'] entry[cm.city_e] = cm.extract_city(data['city'])[0] entry[cm.name_e] = cm.reformat_addr(s['title']) m = re.search(ur'(.+?)-\s*<', s['key']) addr_list = [entry[cm.name_e]] if m is not None: m1 = re.search(ur'-+', m.group(1)) if m1 is not None: tmp = [m.group(1)[:m1.start()], m.group(1)[m1.end():]] else: tmp = [m.group(1)] if len(tmp) > 1: entry[cm.tel] = cm.extract_tel(tmp[1]) m1 = re.search(ur'\d{4,}', tmp[0]) if m1 is not None: entry[cm.zip_code] = m1.group() addr_list.append(tmp[0].strip()) entry[cm.addr_e] = ', '.join(addr_list) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['host'] + data['country_url'] % data['country_id'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching countries: %s' % url, log_name) return [] raw = json.loads(body)['rawPos'] store_list = [] for s in raw: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) addr_list = [] for tmp2 in [cm.html2plain(s[tmp1]).strip() for tmp1 in ['address%d' % v for v in xrange(1, 5)]]: if tmp2 != '': addr_list.append(tmp2) entry[cm.addr_e] = ', '.join(addr_list) entry[cm.city_e] = cm.extract_city(s['city']['name'])[0] entry[cm.country_e] = s['country']['countryCode'] entry[cm.email] = s['email'] entry[cm.fax] = s['fax'] if s['latitude'] != '': entry[cm.lat] = string.atof(s['latitude']) if s['longitude'] != '': entry[cm.lng] = string.atof(s['longitude']) entry[cm.hours] = cm.reformat_addr(s['openingSchedule']) phone_list = [] for key in ['phone1', 'phone2']: if s[key].strip() != '': phone_list.append(s[key].strip()) entry[cm.tel] = ', '.join(phone_list) entry[cm.zip_code] = s['postalCode'] entry[cm.name_e] = s['shopName'] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def get_store_details(data): url = data['url'] try: html = cm.post_data(url, {'country': data['country_id'], 'city': data['city_id'], 'recordid': data['store_id']}) except Exception: print 'Error occured: %s' % url dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': brand_id} cm.dump(dump_data) return [] entry = cm.init_store_entry(brand_id, brandname_e, brandname_c) info = json.loads(html)['elements'] addr = cm.reformat_addr(info['address'].replace('\\', '').replace('<p>', ',').replace('</p>', ',')) # 第一行为商店名称 terms = addr.split(',') if len(terms) > 0: entry[cm.name_e] = cm.reformat_addr(terms[0]) entry[cm.addr_e] = addr gmap_url = info['gmap'] m = re.findall(ur'(-?\d+\.\d+),(-?\d+\.\d+)', gmap_url) if len(m) > 0: cm.update_entry(entry, {cm.lat: string.atof(m[0][0]), cm.lng: string.atof(m[0][1])}) entry[cm.url] = info['shareurl'].replace('\\', '') entry[cm.hours] = info['openingtimes'] entry[cm.comments] = info['other'] # Geo country = data['country'] city = data['city'] cm.update_entry(entry, {cm.country_e: country, cm.city_e: city}) entry[cm.city_e] = cm.extract_city(entry[cm.city_e])[0] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( brandname_e, brand_id, entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return entry
def fetch_store_details(data): # http://maps.oasis-stores.com/index-v2.php?coutnryISO=GB&brand=oasis&lat=51.42014&lng=-0.20954 url = data['store_url'] code = data['country_code'] city = data['city_e'] try: html = cm.get_data(url, {'latitude': data['lat'], 'longitude': data['lng'], 'brand': 'oasis'}) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw = json.loads(html) entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = raw['name'] addr_list = [] for i in xrange(1, 4): tmp = cm.html2plain(raw['address%d' % i]).strip() if tmp!='': addr_list.append(tmp) entry[cm.addr_e] = ', '.join(addr_list) state = raw['countryRegion'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() state = raw['state'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() state = raw['county'] if state is not None and state.strip() != '': entry[cm.province_e] = state.strip().upper() entry[cm.zip_code] = raw['postcode'] entry[cm.country_e] = data['country_e'] entry[cm.city_e] = cm.extract_city(data['city_e'])[0] entry[cm.lat] = string.atof(data['lat']) entry[cm.lng] = string.atof(data['lng']) entry[cm.tel] = raw['phone'] entry[cm.email] = raw['email'] tmp = ['mon', 'tue', 'wed', 'thu', 'fri', 'sat', 'sun'] entry[cm.hours] = ', '.join([raw[d + '_open_times'] for d in tmp]) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return [entry]
def fetch_stores(data): url = data['home_url'] try: body = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw = json.loads(body)['results'] store_list = [] for key in raw: store = raw[key] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = store['post_title'] entry[cm.url] = store['post_permalink'].replace(u'\\', '') entry[cm.country_e] = store['country'].strip().upper() entry[cm.city_e] = store['city'].strip().upper() if '_yoox_store_latlong' in store: m = re.findall(ur'-?\d+\.\d+', store['_yoox_store_latlong']) if len(m) == 2: entry[cm.lat] = string.atof(m[0]) entry[cm.lng] = string.atof(m[1]) if 'store_phone' in store: entry[cm.tel] = store['store_phone'].replace('P:', '').replace('T:', '') \ .replace('P', '').replace('T', '').strip() if 'store_email' in store: entry[cm.email] = store['store_email'] if 'store_fax' in store: entry[cm.fax] = store['store_fax'].replace('F:', '').replace('F', '').strip() if 'store_hours' in store: entry[cm.hours] = cm.reformat_addr(store['store_hours']) if 'store_address' in store: entry[cm.addr_e] = cm.reformat_addr(store['store_address']) if 'women' in store and 'men' in store: entry[cm.store_type] = 'Women: %s, men: %s' % (', '.join(store['women']), ', '.join(store['men'])) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['home_url'] try: body = cm.post_data(url, {'lz_sf': data['province'], 'lz_sx': data['city']}) except Exception: cm.dump('Error in fetching stores: %s, %s, %s' % (url, data['province'], data['city']), 'samsonite_log.txt') dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] start = body.find(u'搜索结果') if start == -1: cm.dump('Error in fetching stores: %s, %s, %s' % (url, data['province'], data['city']), 'samsonite_log.txt') return [] body = body[start + 4:] store_list = [] for m in re.findall(ur'</script>\s*(\S+)\s*</span>', body, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = m.strip() entry[cm.addr_e] = m.strip() entry[cm.city_c] = data['city'] ret = gs.look_up(data['city'], 3) if ret is not None: entry[cm.city_e] = cm.extract_city(ret['name_e'])[0] if ret['province'] != '': entry[cm.province_e] = ret['province']['name_e'] entry[cm.province_c] = data['province'] ret = gs.look_up(data['province'], 2) if ret is not None: entry[cm.province_e] = ret['name_e'] entry[cm.country_e] = u'CHINA' gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), 'benetton_log.txt', False) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_store_details(data): url = data['url'] try: body = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) ret = gs.look_up(data['country'], 1) if ret is not None: entry[cm.country_e] = ret['name_e'] m = re.search(ur'<span class="type">Address</span>\s*<p>(.+?)</p>', body, re.S) if m is not None: addr = cm.reformat_addr(m.group(1)) country, province, city = gs.addr_sense(addr) if country is not None and entry[cm.country_e] == '': entry[cm.country_e] = country if province is not None: entry[cm.province_e] = province if city is not None: entry[cm.city_e] = city entry[cm.addr_e] = addr m = re.search(ur'<span class="type">Phone</span>\s*<p>(.+?)</p>', body, re.S) if m is not None: entry[cm.tel] = m.group(1) m = re.search(ur'<span class="type">Opening hours</span>\s*<p>(.+?)</p>', body, re.S) if m is not None: entry[cm.hours] = cm.reformat_addr(m.group(1)) m = re.search(ur'<span class="type">You can find</span>\s*<p>(.+?)</p>', body, re.S) if m is not None: entry[cm.store_type] = cm.reformat_addr(m.group(1)) m = re.search(ur'google.maps.LatLng\(\s*(-?\d+\.\d+)\s*,\s*(-?\d+\.\d+)', body, re.S) entry[cm.lat]=string.atof(m.group(1)) entry[cm.lng]=string.atof(m.group(2)) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') return [entry]
def fetch_stores(data): url = data['url'] param = {'action': 'getStoresByCity', 'idCity': data['city_id'], 'filter': 'clothing;lacoste%20l!ve'} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] raw = json.loads(body)['root']['DATA']['stores'] store_list = [] for s in [tmp['store'] for tmp in raw]: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e] = s['name'].strip() entry[cm.country_e] = data['country_code'] entry[cm.addr_e] = cm.html2plain(s['address']).strip() entry[cm.store_type] = s['category'].strip() entry[cm.city_e] = cm.extract_city(s['city'])[0] if s['email'] is not None: entry[cm.email] = s['email'].strip() if s['fax'] is not None: entry[cm.fax] = s['fax'].strip() if s['infoHours'] is not None: entry[cm.hours] = s['infoHours'].strip() if s['latitude'] is not None and s['latitude'].strip() != '': entry[cm.lat] = string.atof(s['latitude']) if s['longitude'] is not None and s['longitude'].strip() != '': entry[cm.lat] = string.atof(s['longitude']) if s['phone'] is not None: entry[cm.tel] = s['phone'].strip() if s['postCode'] is not None: entry[cm.zip_code] = s['postCode'].strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): """ 获得商店信息 :param data: """ url = data["post_url"] try: html = cm.post_data(url, {"pid": data["city_id"], "lang": "en", "action": "popola_box_DX"}) except Exception: print "Error occured in getting city list: %s" % url dump_data = {"level": 2, "time": cm.format_time(), "data": {"url": url}, "brand_id": data["brand_id"]} cm.dump(dump_data) return [] store_list = [] for m in re.finditer(ur'<a href="(.+?)".*?>', html): entry = cm.init_store_entry(data["brand_id"], data["brandname_e"], data["brandname_c"]) entry[cm.url] = m.group(1) store_html, start, end = cm.extract_closure(html[m.start() :], ur"<a href", ur"</a>") if end == 0: continue m1 = re.findall(ur'<h3 class="titleShop">(.+?)</h3>', store_html, re.S) if len(m1) > 0: entry[cm.name_e] = m1[0].strip() m1 = re.findall(ur"<p\b.*?>(.+?)(?:</p>|</div>)", store_html, re.S) if len(m1) > 0: terms = cm.reformat_addr(m1[0]).split(",") tel = cm.extract_tel(terms[-1]) if tel != "": terms = terms[:-1] entry[cm.tel] = tel entry[cm.addr_e] = ", ".join([v.strip() for v in terms]) entry["country_e"] = data["country_e"] entry["city_e"] = data["city_e"] gs.field_sense(entry) print "(%s / %d) Found store: %s, %s (%s, %s)" % ( data["brandname_e"], data["brand_id"], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e], ) store_list.append(entry) db.insert_record(entry, "stores")
def fetch_stores_cn(data): vals = ['Storenew%d.html' % idx for idx in xrange(1, 10)] vals.append('Store.html') store_list = [] for url in ('http://www.hushpuppies.com.cn/%s' % term for term in vals): try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching stores: %s' % url, log_name) return () city_map = dict((m[0].strip(), m[1].strip()) for m in re.findall( ur'<a rel="([^"]+)" href="#"[^<>]*>([^<>]+)', body)) start = body.find(ur'<div id="all-list-wrap" style="float:left">') if start == -1: cm.dump('Error in fetching stores: %s' % url, log_name) return () sub = cm.extract_closure(body[start:], ur'<div\b', ur'</div>')[0] for m in re.findall(ur'<ul id="([^"]+)"[^<>]*>(.+?)</ul>', sub, re.S): city = city_map[m[0].strip()] for store in re.findall(ur'<li><a>([^<>]+)', m[1]): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = u'CHINA' entry[cm.city_e] = city entry[cm.addr_e] = store.strip() gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): # country=Greece&city=ATHENS&adutl=+01&kids=+02&undercolor=+06&togetmap=mapdata url = data['data_url'] param = {'country': data['country'], 'city': data['city'], 'adutl': ' 01', 'kids': ' 02', 'undercolor': ' 06', 'togetmap': 'mapdata'} try: body = cm.get_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), 'benetton_log.txt', False) dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] store_list = [] for m in re.findall(ur'<marker (.+?)>', body): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m1 = re.search(ur'name=\\"(.+?)\\"', m) if m1 is not None: entry[cm.name_e] = cm.html2plain(m1.group(1).strip().replace(u'\\', '')) m1 = re.search(ur'address=\\"(.+?)\\"', m) if m1 is not None: addr = cm.reformat_addr(cm.html2plain(m1.group(1)).replace(u'\\', '')) tel = cm.extract_tel(addr) if tel != '': entry[cm.tel] = tel addr = addr.replace(tel, '') entry[cm.addr_e] = cm.reformat_addr(addr) m1 = re.search(ur'lat=\\"(.+?)\\"', m) if m1 is not None: entry[cm.lat] = string.atof(m1.group(1)) m1 = re.search(ur'lng=\\"(.+?)\\"', m) if m1 is not None: entry[cm.lng] = string.atof(m1.group(1)) entry[cm.country_e] = data['country'].strip().upper() entry[cm.city_e] = cm.extract_city(data['city'])[0] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), 'benetton_log.txt', False) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error occured in fetching stores: %s' % url, 'canali_log.txt') dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] store_list = [] for m in re.finditer(ur'<div class="storeInfo">', body): sub = cm.extract_closure(body[m.start():], ur'<div\b', ur'</div>')[0] m1 = re.search(ur'<span itemprop="streetAddress">(.+?)</span>', sub) if m1 is None: cm.dump('Error: failed to find the address: %s' % url, 'canali_log.txt') continue entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.addr_e] = cm.reformat_addr(m1.group(1)) entry[cm.country_e] = data['country'] entry[cm.city_e] = data['city'] ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None: entry[cm.province_e] = ret[1] m1 = re.search(ur'<span itemprop="telephone">(.+?)</span>', sub) if m1 is not None: entry[cm.tel] = m1.group(1).strip() m1 = re.search(ur'data-latitude="(.+?)"', sub) if m1 is not None: entry[cm.lat] = string.atof(m1.group(1)) m1 = re.search(ur'data-longitude="(.+?)"', sub) if m1 is not None: entry[cm.lng] = string.atof(m1.group(1)) gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), 'canali_log.txt') db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): body = data['body'] start = body.find(u'<ul class="storelist storelist_%s' % data['code']) if start == -1: cm.dump('Error in finding stores for %s' % data['code']) return [] body = cm.extract_closure(body[start:], ur'<ul\b', ur'</ul>')[0] store_list = [] for m in re.findall(ur'<li class="sitem">(.+?)</li>', body, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m1 = re.search(ur'<h3>(.+?)</h3>', m) if m1 is not None: entry[cm.name_c] = m1.group(1).strip() m1 = re.search(ur'<div class="addr">(.+?)</div>', m) if m1 is not None: entry[cm.addr_e] = m1.group(1).replace(u'地址:', '').replace(u'地址:', '').strip() m1 = re.search(ur'<div class="tel">(.+?)</div>', m) if m1 is not None: entry[cm.tel] = m1.group(1).replace(u'电话:', '').replace(u'电话:', '').strip() entry[cm.city_c] = data['city'] ret = gs.look_up(data['city'], 3) if ret is not None: entry[cm.city_e] = ret['name_e'] entry[cm.city_c] = ret['name_c'] if ret['province'] != '': entry[cm.province_e] = ret['province']['name_e'] entry[cm.country_e] = u'CHINA' gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), 'canali_log.txt') db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url, {'countryCode':data['country_code']}) except Exception: print 'Error occured in fetching stores: %s, %s' % (url, data['country_code']) dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw=json.loads(body)['stores'] store_list=[] for key in raw: store=raw[key] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e]=store['name'] addr = store['address1'] if store['address2']!='': addr+=', '+store['address2'] entry[cm.addr_e]=addr entry[cm.zip_code]=store['postalCode'] entry[cm.city_e]=store['city'].strip().upper() entry[cm.province_e]=store['stateCode'].strip().upper() entry[cm.country_e]=data['country_code'] entry[cm.tel]=store['phone'] entry[cm.fax]=store['fax'] entry[cm.email]=store['email'] entry[cm.hours]=store['storeHours'] if store['latitude']!='': entry[cm.lat]=string.atof(store['latitude']) if store['longitude']!='': entry[cm.lng]=string.atof(store['longitude']) gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): url = data['store_url'] try: body = cm.get_data(url, { 'country': data['country'], 'city': data['city'] }) except Exception: print 'Error occured: %s' % url dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] raw = json.loads(body) store_list = [] for item in raw['items']: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.country_e] = data['country'].strip().upper() tmp = cm.extract_city(data['city'])[0] if entry[cm.country_e] == 'USA': entry[cm.province_e] = tmp else: entry[cm.city_e] = tmp gs.field_sense(entry) addr = cm.reformat_addr(item['address'].replace(u'\\', '')) addr_list = [tmp.strip() for tmp in addr.split(',')] tel = cm.extract_tel(addr_list[-1]) if tel != '': entry[cm.tel] = tel del addr_list[-1] entry[cm.addr_e] = ', '.join(addr_list) entry[cm.store_type] = item['shop_type'] gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % ( data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_cn(data): url = 'http://www.lee.com.cn/xml/storefinder.xml' store_list = [] for store in (pq(tmp) for tmp in pq(url=url)('NewDataSet Table')): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) tmp = store('shop_name')[0] entry[cm.name_e] = tmp.text if tmp.text else '' entry[cm.country_e] = 'CHINA' tmp = store('city')[0] entry[cm.city_e] = tmp.text if tmp.text else '' tmp = store('province')[0] entry[cm.province_e] = tmp.text if tmp.text else '' tmp = store('district')[0] entry[cm.district_e] = tmp.text if tmp.text else '' tmp = store('address')[0] entry[cm.addr_e] = tmp.text if tmp.text else '' tmp = store('tel')[0] entry[cm.tel] = tmp.text if tmp.text else '' gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e]) if ret[0] is not None and entry[cm.country_e] == '': entry[cm.country_e] = ret[0] if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry) return tuple(store_list)
def fetch_stores(data): url = data['host']+'/en/jil_store_stores.json' try: body = cm.get_data(url, {'city':data['city']}) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] raw=json.loads(body) store_list=[] for store in raw: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.name_e]=store['title'] entry[cm.country_e]=store['country'].strip().upper() entry[cm.city_e]=store['city'].strip().upper() entry[cm.lat]=string.atof(store['lat']) entry[cm.lng]=string.atof(store['lng']) entry[cm.tel]=store['telephone'] addr=store['address'] terms=addr.split(',') m = re.search(ur'\d{5,}', terms[-1]) if m is not None: entry[cm.zip_code] = m.group(0) entry[cm.addr_e]=addr gs.field_sense(entry) print '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) db.insert_record(entry, 'stores') store_list.append(entry) return store_list
def fetch_stores(data): param = { 'action': 'getStoresFromAjax', 'country': data['country_code'], 'region': data['city'], 'collection': '' } url = data['url'] try: body = cm.post_data(url, param) except Exception: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return [] store_list = [] for m1 in re.finditer(ur'<div class="shop-type-container">', body): sub = cm.extract_closure(body[m1.start():], ur'<div\b', ur'</div>')[0] store_class = '' m2 = re.search(ur'<div class="shop-type-title">(.+?)</div>', sub, re.S) if m2 is not None: store_class = cm.reformat_addr(m2.group(1)) for m2 in re.finditer(ur'<div class="shop"', sub): store_sub = cm.extract_closure(sub[m2.start():], ur'<div\b', ur'</div>')[0] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) entry[cm.store_class] = store_class entry[cm.country_e] = data['country_code'] entry[cm.city_e] = cm.extract_city(data['city'])[0] m3 = re.search( ur'loadStore\((\d+)\s*,\s*(-?\d+\.\d+)\s*,\s*(-?\d+\.\d+)\)', store_sub) if m3 is not None: data['store_id'] = string.atoi(m3.group(1)) entry[cm.lat] = string.atof(m3.group(2)) entry[cm.lng] = string.atof(m3.group(3)) entry[cm.store_type] = ', '.join(get_detail(data)) m3 = re.search( ur'<div class="shop-name shop-details shop-main-name">([^<>]+)</div>', store_sub) if m3 is not None: entry[cm.name_e] = m3.group(1).strip() addr_list = [] m3 = re.search( ur'<div class="shop-street shop-details">([^<>]+)</div>', store_sub) if m3 is not None: addr_list.append(cm.reformat_addr(m3.group(1))) m3 = re.search( ur'<div class="shop-city shop-details">([^<>]+)</div>', store_sub) if m3 is not None: tmp = cm.reformat_addr(m3.group(1)) m3 = re.search(ur'(\d{4,})', tmp) if m3 is not None: entry[cm.zip_code] = m3.group(1).strip() addr_list.append(tmp) entry[cm.addr_e] = ', '.join(addr_list) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): print '(%s/%d) Found city: %s' % (data['brandname_e'], data['brand_id'], data['city_e']) url = data['url'] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] # 处理重定向 m = re.search('<h2>Object moved to <a href="(.+?)">', html) if m is not None: data['url'] = data['host'] + m.group(1) return fetch_countries(data) m = re.search('var\s+data\s*=\s*', html) if m is None: return [] sub, start, end = cm.extract_closure(html[m.end():], r'\[', r'\]') if end == 0: return [] store_list = [] for s in json.loads(sub): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) name = s['Name'] if cm.is_chinese(name): entry[cm.name_c] = name else: entry[cm.name_e] = name entry[cm.addr_e] = cm.html2plain(s['Street']) entry[cm.city_e] = cm.extract_city(data['city_e'])[0] entry[cm.country_e] = data['country_e'] entry[cm.province_e] = data['province_e'] pat = re.compile(ur'tel[\.: ]*', re.I) entry[cm.tel] = re.sub(pat, '', s['Phone']).strip() pat = re.compile(ur'fax[\.: ]*', re.I) entry[cm.fax] = re.sub(pat, '', s['Fax']).strip() entry[cm.email] = s['Email'].strip() entry[cm.url] = s['Website'].strip() coord = s['LatLng'] if coord is not None and len(coord) >= 2: if coord[0] is not None: entry[cm.lat] = string.atof(coord[0]) if coord[1] is not None: entry[cm.lng] = string.atof(coord[1]) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) print '(%s/%d) Found store: %s, %s (%s, %s)' % ( data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores') return store_list
entry[cm.tel] = m1.group(1).strip() if m1 else '' m1 = re.search(ur'<div class="hours_wrapper">', sub) if m1: hours_sub = cm.extract_closure(sub[m1.start():], ur'<div\b', ur'</div>')[0] hours_list = [] for term in (tmp.strip() for tmp in cm.reformat_addr(hours_sub).split(',')): term = re.sub(re.compile(ur'store hours\s*:', re.I), '', term).strip() if term != '': hours_list.append(term) entry[cm.hours] = ', '.join(hours_list) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None and entry[cm.province_e] == '': entry[cm.province_e] = ret[1] if ret[2] is not None and entry[cm.city_e] == '': entry[cm.city_e] = ret[2] gs.field_sense(entry) cm.dump( '(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): url = data['url'] if 'page' in data: page = data['page'] else: page = 1 tot = -1 tot_page = -1 store_ids = set([]) store_list = [] page_size = 400 f = open('err_log_%s.log' % data['brandname_e'], 'w') while True: msg = 'Fetching page %d...' % page print msg f.write('%s\n' % msg) try: # html = cm.get_data(url, {'brand': 'adidas', 'geoengine': 'google', 'method': 'get', # 'category': 'store', 'latlng': '51.729663,5.310298,100', # 'page': '%d' % page, 'pagesize': page_size, # 'fields': 'name,street1,street2,addressline,buildingname,postal_code,city,' # 'state,store_owner,country,storetype,longitude_google,' # 'latitude_google,store_owner,state,performance,brand_store,' # 'factory_outlet,originals,neo_label,y3,slvr,children,woman,' # 'footwear,football,basketball,outdoor,porsche_design,miadidas,' # 'miteam,stella_mccartney,eyewear,micoach,opening_ceremony', # 'format': 'json', 'storetype': ''}) html = cm.get_data( url, { 'brand': 'adidas', 'geoengine': 'google', 'method': 'get', 'category': 'store', 'latlng': '31.22434895,121.47675279999999, 10000', 'page': '%d' % page, 'pagesize': page_size, 'fields': 'name,street1,street2,addressline,buildingname,postal_code,city,' 'state,store_owner,country,storetype,longitude_google,' 'latitude_google,store_owner,state,performance,brand_store,' 'factory_outlet,originals,neo_label,y3,slvr,children,woman,' 'footwear,football,basketball,outdoor,porsche_design,miadidas,' 'miteam,stella_mccartney,eyewear,micoach,opening_ceremony', 'format': 'json', 'storetype': '' }) except Exception: msg = 'Error occured: %s' % url print msg f.write('%s\n' % msg) dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) page += 1 if 'page' in data: break else: if page > tot_page: break else: continue try: start = html.find('{') if start != -1: html = html[start:] # 去掉控制字符 pat = re.compile(u'[\r\n]') html = re.sub(pat, ' ', html) pat = re.compile(ur'\\.') html = re.sub(pat, ' ', html) raw = json.loads(html)['wsResponse'] if tot == -1: tot = string.atoi(raw['results']) tot_page = (tot - 1) / page_size + 1 raw = raw['result'] def addr_func(addr_list, addr_map, key): if key in addr_map: addr_list.append(addr_map[key].strip()) for s in raw: try: entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) if s['id'] in store_ids: continue store_ids.add(s['id']) entry[cm.name_e] = s['name'] addr_list = [] map(lambda key: addr_func(addr_list, s, key), ['addressline', 'buildingname', 'street1', 'street2']) entry[cm.addr_e] = ', '.join(addr_list) entry[cm.city_e] = cm.extract_city(s['city'])[0] entry[cm.country_e] = s['country'].strip().upper() if 'storetype' in s: entry[cm.store_type] = s['storetype'] entry[cm.lat] = string.atof(s['latitude_google']) entry[cm.lng] = string.atof(s['longitude_google']) entry[cm.store_class] = 'adidas' gs.field_sense(entry) msg = '(%s / %d) Found store: %s, %s (%s, %s, %s)' % ( data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.city_e], entry[cm.country_e], entry[cm.continent_e]) print msg f.write('%s\n' % msg.encode('utf-8')) store_list.append(entry) db.insert_record(entry, 'stores') except Exception, e: msg = 'Error processing. Reason: %s, content: %s' % ( str(e), s) print msg f.write('%s\n' % msg.encode('utf-8')) except Exception, e: msg = 'Error processing page %d, reason: %s' % (page, str(e)) print msg f.write('%s\n' % msg) finally: