def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries_eu(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_stores_eu(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'data_url_eu': 'http://eu.lee.com/ajax/getMapIcons.php', 'url_eu': 'http://eu.lee.com/stores', 'brand_id': 10213, 'brandname_e': u'LEE', 'brandname_c': u'LEE'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) # db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) # results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) results = fetch_cn(data) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)] if level == 1: # 城市列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)] if level == 2: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'url': 'http://www.levi.com/GB/en_GB/findAStore', 'brand_id': 10215, 'brandname_e': u"Levi's", 'brandname_c': u"Levi's", 'city_map': gen_city_map()} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) # db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'data_url': 'xxxxxxxxxx', 'url': 'http://us.levi.com/storeLocServ', 'brand_id': 10215, 'brandname_e': u"Levi's", 'brandname_c': u"Levi's"} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) # db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) # results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) # results.extend(fetch_hk(data)) results = fetch_jp(data) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch_continents(data): url = data['url'] try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching continents: %s' % url, log_name) return []
def memberMessages(opts): try: api = Trading(debug=opts.debug, config_file=opts.yaml, appid=opts.appid, certid=opts.certid, devid=opts.devid, warnings=True) now = datetime.datetime.now() memberData = { "WarningLevel": "High", "MailMessageType": "All", # "MessageStatus": "Unanswered", "StartCreationTime": now - datetime.timedelta(days=60), "EndCreationTime": now, "Pagination": { "EntriesPerPage": "5", "PageNumber": "1" } } api.execute('GetMemberMessages', memberData) dump(api) if api.response_dict().MemberMessage: messages = api.response_dict().MemberMessage.MemberMessageExchange if type(messages) != list: messages = [ messages ] for m in messages: print("%s: %s" % (m.CreationDate, m.Question.Subject[:50])) except ConnectionError as e: print e
def get_frag_countries(url): # 获得国家代码 """ 获得国家的名字和代码 :rtype : [{'id':**, 'country':**}, ...] :param url: :return: """ try: html = common.get_data(url) except Exception: print 'Error occured: %s' % url_fragrance dump_data = {'level': 1, 'time': common.format_time(), 'data': {'url': url_fragrance}, 'brand_id': brand_id} common.dump(dump_data) return [], False start = html.find('<select name="country" id="id_country">') if start == -1: return [], False sub, s, e = common.extract_closure(html[start:], ur'<select\b', ur'</select>') if e == 0: return [], False return [{'id': string.atoi(m[0]), 'country': m[1].strip().upper()} for m in re.findall(ur'<option value="(\d+)".*?>(.+?)</option>', sub)]
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return [] # Walk from the root node, where level == 1. if data is None: data = {'url': 'http://www.tommy.com.cn/storelocator/index/getmapajax', 'brand_id': 10355, 'brandname_e': u'Tommy Hilfiger', 'brandname_c': u'汤米·希尔费格'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def run(opts): try: shopping = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=False) response = shopping.execute('FindPopularItems', {'QueryKeywords': 'Python'}) nodes = response.dom().xpath('//ItemID') itemIds = [n.text for n in nodes] api = FindItem(debug=opts.debug, consumer_id=opts.consumer_id, config_file=opts.yaml) records = api.find_items_by_ids([itemIds[0]]) for r in records: print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35])) dump(api) records = api.find_items_by_ids(itemIds) for r in records: print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35])) dump(api) except ConnectionError as e: print(e) print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'url': 'http://viviennetam.com/contact/store-locator', 'brand_id': 10400, 'brandname_e': u'Vivienne Tam', 'brandname_c': u'Vivienne Tam'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)] if level == 1: # 城市列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)] if level == 2: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'data_url': 'http://www.breguet.com/en/content/action', 'url': 'http://www.breguet.com/en/Worldwide-retailers', 'brand_id': 10053, 'brandname_e': u'Breguet', 'brandname_c': u'宝玑'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def run(opts): try: p = Parallel() apis = [] api1 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml) api1.execute('findItemsAdvanced', {'keywords': 'python'}) apis.append(api1) api4 = html(parallel=p) api4.execute('http://www.ebay.com/sch/i.html?_nkw=Shirt&_rss=1') apis.append(api4) api2 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml) api2.execute('findItemsAdvanced', {'keywords': 'perl'}) apis.append(api2) api3 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml) api3.execute('findItemsAdvanced', {'keywords': 'php'}) apis.append(api3) p.wait() if p.error(): print(p.error()) for api in apis: dump(api) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_states(data): global national_added url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching states: %s' % url, log_name) return [] national_added = False m = re.search(ur'Choose a (state|region|province)', body) if m is None: d = data.copy() d['state'] = '' return [d] body = cm.extract_closure(body[m.start():], ur'<ul>', ur'</ul>')[0] results = [] for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body): d = data.copy() d['url'] = data['host'] + m[0] d['state'] = cm.html2plain(m[1]).strip().upper() results.append(d)
def fetch_countries(data): url = data['host'] + 'boutique' try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching countries: %s' % url, log_name) return []
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = { 'data_url': 'http://www.c-and-a.com/uk/en/corporate/company/stores/storefinder/?no_cache=1&tx_iostorefinder_pi1%5Baction%5D=search&tx_iostorefinder_pi1%5Bcontroller%5D=store', 'url': 'http://www.c-and-a.com/uk/en/corporate/fashion/stores/', 'brand_id': 10059, 'brandname_e': u'C&A', 'brandname_c': u'C&A'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch_countries(data): url = data["url"] try: body = cm.get_data(url) except Exception, e: cm.dump("Error in fetching countries: %s" % url, log_name) return []
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 商店列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_store_list(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_store_details(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = {'url': 'http://www.chaumet.com/all-points-of-sale', 'brand_id': 10076, 'brandname_e': u'Chaumet', 'brandname_c': u'尚美巴黎'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def run(opts): try: api = finding(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=True) api_request = { #'keywords': u'niño', 'keywords': u'GRAMMY Foundation®', 'itemFilter': [ {'name': 'Condition', 'value': 'Used'}, {'name': 'LocatedIn', 'value': 'GB'}, ], 'affiliate': {'trackingId': 1}, 'sortOrder': 'CountryDescending', } response = api.execute('findItemsAdvanced', api_request) dump(api) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching stores: %s' % url, log_name) return [] store_list = [] for m in re.finditer(ur'<item id="\d+">', body): sub = cm.extract_closure(body[m.start():], ur'<item\b', ur'</item>')[0] entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) m1 = re.search(ur'<country>([^<>]+)</country>', sub) if m1 is not None: tmp = m1.group(1).split('/') for v in tmp: ret = gs.look_up(v.strip().upper(), 1) if ret is not None: entry[cm.country_e] = ret['name_e'] break m1 = re.search(ur'<city>([^<>]+)</city>', sub) if m1 is not None: val = cm.reformat_addr(m1.group(1)) if entry[cm.country_e] == 'UNITED STATES': tmp_list = tuple(tmp.strip() for tmp in cm.reformat_addr(val).strip(',')) if len(tmp_list) == 2: if re.search('[A-Z]{2}', tmp_list[1]): entry[cm.province_e] = tmp_list[1] entry[cm.city_e] = cm.extract_city(m1.group(1))[0] m1 = re.search(ur'<brands>([^<>]+)</brands>', sub) if m1 is not None: tmp = m1.group(1).split('/') brand_list = [] for v in tmp: if v.strip() != '': brand_list.append(v) entry[cm.store_type] = ', '.join(brand_map[key] for key in brand_list) m1 = re.search(ur'<name>([^<>]+)</name>', sub) if m1 is not None: entry[cm.name_e] = m1.group(1).strip() m1 = re.search(ur'<address>([^<>]+)</address>', sub) if m1 is not None: entry[cm.addr_e] = cm.reformat_addr(m1.group(1)) m1 = re.search(ur'<tel>([^<>]+)</tel>', sub) if m1 is not None: entry[cm.tel] = m1.group(1).strip() m1 = re.search(ur'sll=(-?\d+\.\d+),(-?\d+\.\d+)', sub) if m1 is not None: entry[cm.lat] = string.atof(m1.group(1)) entry[cm.lng] = string.atof(m1.group(2)) gs.field_sense(entry) ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e]) if ret[1] is not None: entry[cm.province_e] = ret[1] gs.field_sense(entry) cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]), log_name) db.insert_record(entry, 'stores') store_list.append(entry)
def fetch_stores(data): # <h2 property="dc:title" url = data[cm.url] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] store_list = [] for m in re.finditer(ur'<h2 property="dc:title"', html): end = html.find('</header>', m.start()) if end == -1: continue sub = html[m.start():end] m1 = re.search(ur'<a href="(.+?)">(.+?)</a></h2>', sub) if m1 is None: print 'Error: no more details for %s' % url continue d = data.copy() d[cm.url] = data['host'] + m1.group(1) d[cm.name_e] = cm.html2plain(m1.group(2)).strip() store_list.append(d)
def fetch_countries(data): url = data['country_url'] try: body = cm.get_data(url, {'display_country': 'CN'}) except Exception, e: cm.dump('Error in fetching countries: %s' % url, log_name) return ()
def fetch_countries(data): url = data['url'] try: body, data['cookie'] = cm.get_data_cookie(url) except Exception, e: cm.dump('Error in fetching countries: %s' % url, log_name) return ()
def fetch_cities_beauty(data): url = '%s/%s' % (data['city_url'], data['country_code']) try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching countries: %s' % url, log_name) return ()
def get_cities(data): try: d = {"country": data["country_code"], "city": "", "service": -1} html = common.post_data(url, d) except Exception: print "Error occured in getting the list of countries: %s" % url dump_data = {"level": 1, "time": common.format_time(), "data": {"data": url}, "brand_id": brand_id} common.dump(dump_data) return [] start = html.find(u'<select id="city" name="city">') if start == -1: return [] end = html.find(u"</select>", start) html = html[start:end] city_list = [] for m in re.findall(ur'<option value="(.+?)">', html): if data["country_code"] == "GB" and "2 davies street" in m.lower(): continue elif data["country_code"] == "RO" and "13 september street" in m.lower(): continue elif "b1603daq" in m.lower(): continue else: city_list.append({"city_e": m, "country_e": data["country_e"], "country_code": data["country_code"]})
def fetch_stores(data): url = data['url'] try: body = cm.get_data(url, {'m': data['m']}) except Exception, e: cm.dump('Error in fetching stores: %s' % url, log_name) return ()
def fetch_store_details(data): url = data['url'] try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching store details: %s' % url, log_name) return ()
def fetch_store_details(data): url = data['url'] try: body, data['cookie'] = cm.get_data_cookie(url, cookie=data['cookie']) except Exception, e: cm.dump('Error in fetching stores: %s' % url, log_name) return ()
def fetch_stores(data): url = data['url'] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] store_list = [] for m in re.findall(ur'var markerContent\s*?=\s*?"(.+?)".+?' ur'createMarker\(.+?new google.maps.LatLng\((-?\d+\.\d+),(-?\d+\.\d+)\)', html, re.S): entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c']) lat, lng = map(string.atof, [m[1], m[2]]) cm.update_entry(entry, {cm.lat: lat, cm.lng: lng}) sub = m[0].strip() m1 = re.search(ur'<b>(.+?)</b>', sub) if m1 is None: continue entry[cm.name_c] = m1.group(1) sub = sub.replace(m1.group(0), '') m1=re.search(ur'聯系電話(?::|:)(.+?)<', sub) if m1 is not None: entry[cm.tel]=m1.group(1) sub=sub.replace(m1.group(0), '<') sub = re.sub(ur'<img\b.*?/>', '', sub) entry[cm.addr_c] = cm.reformat_addr(sub) print '(%s/%d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'], entry[cm.name_c], entry[cm.addr_e], entry[cm.country_e], entry[cm.continent_e]) store_list.append(entry) db.insert_record(entry, 'stores')
def fetch_countries_eu(data): url = data['url_eu'] try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching EU countries: %s' % url, log_name) return ()
def fetch_cities(data): url = data['host'] + data['url'] try: body = cm.get_data(url) except Exception, e: cm.dump('Error in fetching cities: %s' % url, log_name) return []
def fetch_countries(data): url = data["url"] try: body, data["cookie"] = cm.get_data_cookie(url) except Exception, e: cm.dump("Error in fetching AppKey: %s" % url, log_name) return ()
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{ 'func': lambda data: func(data, level + 1), 'data': s } for s in fetch_countries(data)] if level == 1: # 城市列表 return [{ 'func': lambda data: func(data, level + 1), 'data': s } for s in fetch_cities(data)] if level == 2: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = { 'url': 'http://www.levi.com/GB/en_GB/findAStore', 'brand_id': 10215, 'brandname_e': u"Levi's", 'brandname_c': u"Levi's", 'city_map': gen_city_map() } global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) # db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def run2(opts): try: api = finding(debug=opts.debug, appid=opts.appid, config_file=opts.yaml) response = api.execute('findItemsByProduct', '<productId type="ReferenceID">53039031</productId><paginationInput><entriesPerPage>1</entriesPerPage></paginationInput>') if hasattr(response.reply.searchResult, 'item'): for r in response.reply.searchResult.item: print("ID(%s) TITLE(%s)" % (r.itemId, r.title)) else: print("No Items Found.") dump(api) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_continents(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching continents: %s' % url, log_name) return [] m = re.search(ur'<nav\s+id\s*=\s*"retailers_nav">', body) if m is None: cm.dump('Error in fetching continents: %s' % url, log_name) return [] body = cm.extract_closure(body[m.start():], ur'<nav\b', ur'</nav>')[0] results = [] for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body): d = data.copy() d['url'] = data['host'] + m[0] if 'europe' in d['url']: results.append(d)
def fetch_stores(data): if 'content' in data: sub = data['content'] else: url = data['data_url'] param = { 'action': 'fmd', 'id': 50, 'g': 1, 'page': 7, 'country': data['country_code'], '_escaped_fragment_': data['fragment'] } try: body = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching store: %s, %s' % (url, param), log_name) return [] sub = json.loads(body)['content']
def fetch_cities(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching cities: %s' % url, log_name) return [] m = re.search(ur'Choose a city', body) if m is None: cm.dump('Error in fetching cities: %s' % url, log_name) return [] body = cm.extract_closure(body[m.start():], ur'<ul>', ur'</ul>')[0] results = [] for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body): d = data.copy() d['url'] = data['host'] + cm.html2plain(m[0]) d['city'] = cm.html2plain(m[1]).strip().upper() results.append(d)
def getUser(opts): try: api = Trading(debug=opts.debug, config_file=opts.yaml, appid=opts.appid, domain=opts.domain, certid=opts.certid, devid=opts.devid, warnings=True, timeout=20, siteid='101') api.execute('GetUser', {'UserID': 'sallyma789'}) dump(api, full=False) except ConnectionError as e: print(e) print(e.response.dict())
def get_store_hours(data): url = data['store_url'] param = {'id': data['store_id']} try: s = json.loads(cm.get_data(url, param))['stores'][0] workdays = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun') open_days = dict( (key, s['hoursOpenFlag%s' % key] == 'Y') for key in workdays) workhours = [] for d in workdays: if not open_days[d]: continue workhours.append( '%s: %s - %s' % (d, s['hoursOpenTime%s' % d], s['hoursCloseTime%s' % d])) return ', '.join(workhours) except Exception, e: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return ''
def fetch_stores(data): url = data['data_url'] param = { 'module': 'pointsOfSaleAdvanced', 'action': 'ajaxGetURI', 'code_uri': '%s/%s/%s/' % tuple(data[key] for key in ('continent_code', 'country_code', 'place_code')), 'languageId': 1 } try: new_url = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return []
def fetch_countries(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching countries: %s' % url, log_name) return [] m = re.search(ur'<select class="country-select"', body) if m is None: cm.dump('Error in fetching countries: %s' % url, log_name) return [] body = cm.extract_closure(body[m.start():], ur'<select\b', ur'</select>')[0] results = [] for m in re.findall(ur'<option value="([A-Z]{2})">', body): d = data.copy() d['country_code'] = m results.append(d)
def feedback(opts): try: api = Trading(debug=opts.debug, config_file=opts.yaml, appid=opts.appid, certid=opts.certid, devid=opts.devid, warnings=False) api.execute('GetFeedback', {'UserID': 'tim0th3us'}) dump(api) if int(api.response_dict().FeedbackScore) > 50: print("Doing good!") else: print("Sell more, buy more..") except ConnectionError as e: print e
def run(opts): try: api = finding(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=True) api_request = { #'keywords': u'niño', 'keywords': u'GRAMMY Foundation®', 'itemFilter': [ { 'name': 'Condition', 'value': 'Used' }, { 'name': 'LocatedIn', 'value': 'GB' }, ], 'affiliate': { 'trackingId': 1 }, 'sortOrder': 'CountryDescending', } response = api.execute('findItemsAdvanced', api_request) dump(api) assert (response.reply.ack == 'Success') assert (type(response.reply.timestamp) == datetime.datetime) #assert(type(response.reply.searchResult.item) == list) items = response.reply.searchResult.item print(len(items)) except ConnectionError as e: print(e) print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 商店列表 return [{ 'func': lambda data: func(data, level + 1), 'data': s } for s in fetch_store_list(data)] if level == 1: # 商店 return [{ 'func': None, 'data': s } for s in fetch_store_details(data)] else: return [] # Walk from the root node, where level == 1. if data is None: data = { 'url': 'http://www.alexandermcqueen.com/experience/en/alexandermcqueen/store/', 'brand_id': 10008, 'brandname_e': u'Alexander McQueen', 'brandname_c': u'亚历山大·麦昆' } global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{ 'func': lambda data: func(data, level + 1), 'data': s } for s in fetch_countries(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = { 'data_url1': 'xxxxxxxxxx', 'data_url': 'http://dms.wolverineworldwide.com/SearchResults.aspx', 'brand_id': 10170, 'brandname_e': u'Hush Puppies', 'brandname_c': u'暇步士' } global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) results.extend(fetch_stores_cn(data)) results.extend(fetch_stores_au(data)) db.disconnect_db() cm.dump('Done!', log_name) return results
def memberMessages(args): try: api = Trading(debug=args.debug, appid=app_id, token=token_id, certid=cert_id, devid=dev_id, config_file=None) now = datetime.datetime.now() memberData = { "WarningLevel": "High", "MailMessageType": "All", # "MessageStatus": "Unanswered", "StartCreationTime": now - datetime.timedelta(days=90), "EndCreationTime": now, "Pagination": { "EntriesPerPage": "10", "PageNumber": "1" } } api.execute('GetMemberMessages', memberData) dump(api) if api.response.reply.has_key('MemberMessage'): messages = api.response.reply.MemberMessage.MemberMessageExchange if type(messages) != list: messages = [messages] for m in messages: print( "%s %s: %s" % (m.CreationDate, m.Question.Subject[:50], m.Question.Body)) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_countries(data): url = data['url'] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = { 'level': 0, 'time': cm.format_time(), 'data': { 'url': url }, 'brand_id': data['brand_id'] } cm.dump(dump_data) return [] # 处理重定向 m = re.search('<h2>Object moved to <a href="(.+?)">', html) if m is not None: data['url'] = data['host'] + m.group(1) return fetch_countries(data) m = re.search('<span class="country">Choose a country</span>', html) if m is None: return [] sub, start, end = cm.extract_closure(html[m.end():], r'<ul\b', r'</ul>') if end == 0: return [] country_list = [] for m in re.findall('<li><a .*?href="(.+?)">(.+?)</a></li>', sub): d = data.copy() country_e = cm.html2plain(m[1]).strip().upper() ret = gs.look_up(country_e, 1) if ret is not None: country_e = ret['name_e'] d['country_e'] = country_e d['province_e'] = '' d['url'] = data['host'] + m[0] country_list.append(d) return country_list
def fetch_countries(data): url = data['home_url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching countries: %s' % url, log_name) return [] m = re.search(ur'<select\s+id\s*=\s*"store-locator-country"', body) if m is None: cm.dump('Error in fetching countries: %s' % url, log_name) return [] body = cm.extract_closure(body[m.start():], ur'<select\b', ur'</select>')[0] results = [] for m in re.findall(ur'<option value="([A-Z]{2})">', body): d = data.copy() d['country_code'] = m # if m.upper() == 'GR': results.append(d)
def get_store(args): try: api = Trading(debug=args.debug, appid=app_id, token=token_id, certid=cert_id, devid=dev_id, warnings=True, timeout=20, siteid=site_id, config_file=None) storeData = { 'MessageID': 'IT CIRCLE CONSULT PYTHON - MAFFAS', 'TaskID': '1', } api.execute('GetStoreCustomPage', storeData) dump(api) f = open('E:\\ITCIRCLECONSULT\\GITHUB\\maf-ebay\\maf-ebay-dev\\get-store\\'+ file_name + '.xml','a') f.write(str(api.response.content)[2:-1]) # write result, removes the b' from the start, from position [2] until the ' at the end [-1] f.close() #close html except ConnectionError as e: print(e) print(e.response.dict())
def run(opts): api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=True) print("Shopping samples for SDK version %s" % ebaysdk.get_version()) try: response = api.execute('FindPopularItems', {'QueryKeywords': 'Python'}) dump(api) print("Matching Titles:") for item in response.reply.ItemArray.Item: print(item.Title) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_store_list(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching store list: %s' % url, 'debeers_log.txt') return [] start = body.find(u'<ul class="store-link-set">') if start == -1: cm.dump('Error in fetching store list: %s' % url, 'debeers_log.txt') body = cm.extract_closure(body[start:], ur'<ul\b', ur'</ul>')[0] results = [] for m in re.findall(ur'<li .*?>\s*<a href="(.+?)" title="(.+?)"', body, re.S): d = data.copy() d['url'] = data['host'] + m[0] d['name'] = m[1].strip() results.append(d)
def with_affiliate_info(opts): try: api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=True, trackingid=1234, trackingpartnercode=9) mySearch = { "MaxKeywords": 10, "QueryKeywords": 'shirt', } response = api.execute('FindPopularSearches', mySearch) dump(api, full=False) except ConnectionError as e: print(e) print(e.response.dict())
def fetch_stores(data): url = data['data_url'] c_code = data['country_code'] param = { 'xml_request': '<request><appkey>%s</appkey><formdata id="locatorsearch"><order>rank,_distance</order><geolocs><geoloc><longitude>%f</longitude><latitude>%f</latitude><country>%s</country></geoloc></geolocs><searchradius>100</searchradius><where><country><eq>%s</eq></country><or><columbia><eq>1</eq></columbia><outlet><eq>1</eq></outlet><store><eq>1</eq></store></or></where></formdata></request>' % (data['cookie']['AppKey'], data['city_lat'], data['city_lng'], c_code, c_code) } param = { 'xml_request': '<request><appkey>%s</appkey><formdata id="locatorsearch"><order>rank,_distance</order><geolocs><geoloc><addressline></addressline><city></city><state></state><province></province><postalcode></postalcode><longitude>%f</longitude><latitude>%f</latitude><country>%s</country></geoloc></geolocs><searchradius>100</searchradius><where><country><eq>%s</eq></country><or><columbia><eq>1</eq></columbia><outlet><eq>1</eq></outlet><store><eq>1</eq></store></or></where></formdata></request>' % (data['cookie']['AppKey'], data['city_lng'], data['city_lat'], c_code, c_code) } try: body = cm.get_data(url, param) except Exception, e: cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name) return ()
def fetch_countries(data): url = data['home_url'] try: body = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']} cm.dump(dump_data) return [] start = body.find(u'<label>COUNTRY</label>') if start==-1: print 'Error occured in fetching country list: %s' % url body = cm.extract_closure(body[start:], ur'<select\b', ur'</select>')[0] results=[] for m in re.findall(ur'<option value="([A-Z]{2})">.+?</option>', body): d=data.copy() d['country_code']=m results.append(d)
def fetch_countries(data): url = data['url'] try: body = cm.get_data(url) except Exception: cm.dump('Error in fetching countries: %s' % url, log_name) return [] m = re.search(ur'<a href="[^"]*" class="[^"]*">BOUTIQUE</a>', body) if m is None: cm.dump('Error in fetching countries: %s' % url, log_name) return [] body = cm.extract_closure(body[m.end():], ur'<ul\b', ur'</ul>')[0] results = [] for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body): d = data.copy() d['region'] = m[1].strip() d['url'] = data['host'] + m[0].strip() results.append(d)
def parse_store(data, body=None): if body is None: url = data['url'] try: body = cm.post_data(url) except Exception: cm.dump('Error in fetching stores: %s' % url, log_name) return [] start = body.find(ur'jQuery.extend(Drupal.settings,') latlng_map = {} if start != -1: for item in json.loads( cm.extract_closure( body[start:], ur'\{', ur'\}')[0])['getlocations']['key_1']['latlons']: latlng_map[cm.reformat_addr(item[3])] = { 'lat': string.atof(item[0]), 'lng': string.atof(item[1]) }
def getOrders(args): try: api = Trading( debug=args.debug, appid=app_id, token=token_id, certid=cert_id, devid=dev_id, warnings=True, timeout=20, config_file=None, ) api.execute('GetOrders', {'NumberOfDays': 30}) dump(api, full=False) except ConnectionError as e: print(e) print(e.response.dict())
def feedback(args): try: api = Trading(debug=args.debug, appid=app_id, token=token_id, config_file=None, certid=cert_id, devid=dev_id) api.execute('GetFeedback', {'UserID': 'parties_wrapped_up'}) dump(api) if int(api.response.reply.FeedbackScore) > 50: print("Onwards and Upwards!") else: print("Onwards and Forwards!") except ConnectionError as e: print(e) print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{ 'func': lambda data: func(data, level + 1), 'data': s } for s in fetch_countries(data)] if level == 1: # 商店 return [{'func': None, 'data': s} for s in fetch_stores(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = { 'url': 'http://global.tommy.com/int/en/Stores/Find-A-Store', 'brand_id': 10355, 'brandname_e': u'Tommy Hilfiger', 'brandname_c': u'汤米·希尔费格' } global db, id_set db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) # db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) rs = db.query_all( 'SELECT native_id FROM stores WHERE brand_id=%d and not native_id=""' % data['brand_id']) id_set = [tmp[0] for tmp in rs] results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def fetch(level=1, data=None, user='******', passwd=''): def func(data, level): """ :param data: :param level: 0:国家;1:城市;2:商店列表 """ if level == 0: # 国家列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)] if level == 1: # 州列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_states(data)] if level == 2: # 城市列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)] if level == 3: # 城市列表 return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_store_list(data)] if level == 4: # 商店 return [{'func': None, 'data': s} for s in fetch_store_details(data)] else: return () # Walk from the root node, where level == 1. if data is None: data = { 'url': 'http://www.swarovski.com.cn/is-bin/INTERSHOP.enfinity/WFS/SCO-Web_CN-Site/en_US/-/CNY/SMOD_Storefinder-ViewStorefinderSelects', 'store_url': 'http://www.swarovski.com.cn/Web_CN/en/boutique_search', 'brand_id': 10339, 'brandname_e': u'Swarovski', 'brandname_c': u'施华洛世奇'} global db db = cm.StoresDb() db.connect_db(user=user, passwd=passwd) db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id'])) results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data}) db.disconnect_db() cm.dump('Done!', log_name) return results
def get_countries(data): """ 返回国家列表 :rtype : [{'country_code':**, 'country':**}, ...] :param data: :return: """ url = data['url'] try: html = cm.get_data(url) except Exception: print 'Error occured: %s' % url dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': brand_id} cm.dump(dump_data) return [] pat = '<option value="0">Choose a country</option>' splits = [m.start() for m in re.finditer(pat, html)] splits.append(-1) sub_html = [] for i in xrange(len(splits) - 1): sub_html.append(html[splits[i]:splits[i + 1]]) # 1:州信息 # s_map = [{'state_code':m[0], 'state':m[1].strip} state_list = [] for m in re.findall(ur'<option value="(.+?)"\s*?>(.+?)</option>', sub_html[0][len(pat):]): code = m[0].strip().upper() state = m[1].strip().upper() ret = gs.look_up(state, 2) if ret is not None: # state_list.append({'state': ret[0]['province_e'], 'state_code': ret[0]['state_code']}) state_list.append({'state': ret['name_e'], 'state_code': ret['code']}) else: # state其实是写成是代码 for key in gs.province_map['data']: state = gs.province_map['data'][key] if state['code'] == code: state = state['name_e'] state_list.append({'state': state, 'state_code': code}) break