def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries_eu(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores_eu(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'data_url_eu': 'http://eu.lee.com/ajax/getMapIcons.php',
'url_eu': 'http://eu.lee.com/stores',
'brand_id': 10213, 'brandname_e': u'LEE', 'brandname_c': u'LEE'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
# db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
# results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
results = fetch_cn(data)
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)]
if level == 1:
# 城市列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)]
if level == 2:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'url': 'http://www.levi.com/GB/en_GB/findAStore',
'brand_id': 10215, 'brandname_e': u"Levi's", 'brandname_c': u"Levi's",
'city_map': gen_city_map()}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
# db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'data_url': 'xxxxxxxxxx',
'url': 'http://us.levi.com/storeLocServ',
'brand_id': 10215, 'brandname_e': u"Levi's", 'brandname_c': u"Levi's"}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
# db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
# results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
# results.extend(fetch_hk(data))
results = fetch_jp(data)
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch_continents(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching continents: %s' % url, log_name)
return []
def memberMessages(opts):
try:
api = Trading(debug=opts.debug, config_file=opts.yaml, appid=opts.appid,
certid=opts.certid, devid=opts.devid, warnings=True)
now = datetime.datetime.now()
memberData = {
"WarningLevel": "High",
"MailMessageType": "All",
# "MessageStatus": "Unanswered",
"StartCreationTime": now - datetime.timedelta(days=60),
"EndCreationTime": now,
"Pagination": {
"EntriesPerPage": "5",
"PageNumber": "1"
}
}
api.execute('GetMemberMessages', memberData)
dump(api)
if api.response_dict().MemberMessage:
messages = api.response_dict().MemberMessage.MemberMessageExchange
if type(messages) != list:
messages = [ messages ]
for m in messages:
print("%s: %s" % (m.CreationDate, m.Question.Subject[:50]))
except ConnectionError as e:
print e
def get_frag_countries(url):
# 获得国家代码
"""
获得国家的名字和代码
:rtype : [{'id':**, 'country':**}, ...]
:param url:
:return:
"""
try:
html = common.get_data(url)
except Exception:
print 'Error occured: %s' % url_fragrance
dump_data = {'level': 1, 'time': common.format_time(), 'data': {'url': url_fragrance},
'brand_id': brand_id}
common.dump(dump_data)
return [], False
start = html.find('<select name="country" id="id_country">')
if start == -1:
return [], False
sub, s, e = common.extract_closure(html[start:], ur'<select\b', ur'</select>')
if e == 0:
return [], False
return [{'id': string.atoi(m[0]), 'country': m[1].strip().upper()}
for m in re.findall(ur'<option value="(\d+)".*?>(.+?)</option>', sub)]
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return []
# Walk from the root node, where level == 1.
if data is None:
data = {'url': 'http://www.tommy.com.cn/storelocator/index/getmapajax',
'brand_id': 10355, 'brandname_e': u'Tommy Hilfiger', 'brandname_c': u'汤米·希尔费格'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def run(opts):
try:
shopping = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
response = shopping.execute('FindPopularItems',
{'QueryKeywords': 'Python'})
nodes = response.dom().xpath('//ItemID')
itemIds = [n.text for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id, config_file=opts.yaml)
records = api.find_items_by_ids([itemIds[0]])
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'url': 'http://viviennetam.com/contact/store-locator',
'brand_id': 10400, 'brandname_e': u'Vivienne Tam', 'brandname_c': u'Vivienne Tam'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)]
if level == 1:
# 城市列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)]
if level == 2:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'data_url': 'http://www.breguet.com/en/content/action',
'url': 'http://www.breguet.com/en/Worldwide-retailers',
'brand_id': 10053, 'brandname_e': u'Breguet', 'brandname_c': u'宝玑'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def run(opts):
try:
p = Parallel()
apis = []
api1 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml)
api1.execute('findItemsAdvanced', {'keywords': 'python'})
apis.append(api1)
api4 = html(parallel=p)
api4.execute('http://www.ebay.com/sch/i.html?_nkw=Shirt&_rss=1')
apis.append(api4)
api2 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml)
api2.execute('findItemsAdvanced', {'keywords': 'perl'})
apis.append(api2)
api3 = finding(parallel=p, debug=opts.debug, appid=opts.appid, config_file=opts.yaml)
api3.execute('findItemsAdvanced', {'keywords': 'php'})
apis.append(api3)
p.wait()
if p.error():
print(p.error())
for api in apis:
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_states(data):
global national_added
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching states: %s' % url, log_name)
return []
national_added = False
m = re.search(ur'Choose a (state|region|province)', body)
if m is None:
d = data.copy()
d['state'] = ''
return [d]
body = cm.extract_closure(body[m.start():], ur'<ul>', ur'</ul>')[0]
results = []
for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body):
d = data.copy()
d['url'] = data['host'] + m[0]
d['state'] = cm.html2plain(m[1]).strip().upper()
results.append(d)
def fetch_countries(data):
url = data['host'] + 'boutique'
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {
'data_url': 'http://www.c-and-a.com/uk/en/corporate/company/stores/storefinder/?no_cache=1&tx_iostorefinder_pi1%5Baction%5D=search&tx_iostorefinder_pi1%5Bcontroller%5D=store',
'url': 'http://www.c-and-a.com/uk/en/corporate/fashion/stores/',
'brand_id': 10059, 'brandname_e': u'C&A', 'brandname_c': u'C&A'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch_countries(data):
url = data["url"]
try:
body = cm.get_data(url)
except Exception, e:
cm.dump("Error in fetching countries: %s" % url, log_name)
return []
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 商店列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_store_list(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_store_details(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {'url': 'http://www.chaumet.com/all-points-of-sale',
'brand_id': 10076, 'brandname_e': u'Chaumet', 'brandname_c': u'尚美巴黎'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def run(opts):
try:
api = finding(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True)
api_request = {
#'keywords': u'niño',
'keywords': u'GRAMMY Foundation®',
'itemFilter': [
{'name': 'Condition',
'value': 'Used'},
{'name': 'LocatedIn',
'value': 'GB'},
],
'affiliate': {'trackingId': 1},
'sortOrder': 'CountryDescending',
}
response = api.execute('findItemsAdvanced', api_request)
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_stores(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching stores: %s' % url, log_name)
return []
store_list = []
for m in re.finditer(ur'<item id="\d+">', body):
sub = cm.extract_closure(body[m.start():], ur'<item\b', ur'</item>')[0]
entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c'])
m1 = re.search(ur'<country>([^<>]+)</country>', sub)
if m1 is not None:
tmp = m1.group(1).split('/')
for v in tmp:
ret = gs.look_up(v.strip().upper(), 1)
if ret is not None:
entry[cm.country_e] = ret['name_e']
break
m1 = re.search(ur'<city>([^<>]+)</city>', sub)
if m1 is not None:
val = cm.reformat_addr(m1.group(1))
if entry[cm.country_e] == 'UNITED STATES':
tmp_list = tuple(tmp.strip() for tmp in cm.reformat_addr(val).strip(','))
if len(tmp_list) == 2:
if re.search('[A-Z]{2}', tmp_list[1]):
entry[cm.province_e] = tmp_list[1]
entry[cm.city_e] = cm.extract_city(m1.group(1))[0]
m1 = re.search(ur'<brands>([^<>]+)</brands>', sub)
if m1 is not None:
tmp = m1.group(1).split('/')
brand_list = []
for v in tmp:
if v.strip() != '':
brand_list.append(v)
entry[cm.store_type] = ', '.join(brand_map[key] for key in brand_list)
m1 = re.search(ur'<name>([^<>]+)</name>', sub)
if m1 is not None:
entry[cm.name_e] = m1.group(1).strip()
m1 = re.search(ur'<address>([^<>]+)</address>', sub)
if m1 is not None:
entry[cm.addr_e] = cm.reformat_addr(m1.group(1))
m1 = re.search(ur'<tel>([^<>]+)</tel>', sub)
if m1 is not None:
entry[cm.tel] = m1.group(1).strip()
m1 = re.search(ur'sll=(-?\d+\.\d+),(-?\d+\.\d+)', sub)
if m1 is not None:
entry[cm.lat] = string.atof(m1.group(1))
entry[cm.lng] = string.atof(m1.group(2))
gs.field_sense(entry)
ret = gs.addr_sense(entry[cm.addr_e], entry[cm.country_e])
if ret[1] is not None:
entry[cm.province_e] = ret[1]
gs.field_sense(entry)
cm.dump('(%s / %d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'],
entry[cm.name_e], entry[cm.addr_e], entry[cm.country_e],
entry[cm.continent_e]), log_name)
db.insert_record(entry, 'stores')
store_list.append(entry)
def fetch_stores(data):
# <h2 property="dc:title"
url = data[cm.url]
try:
html = cm.get_data(url)
except Exception:
print 'Error occured: %s' % url
dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']}
cm.dump(dump_data)
return []
store_list = []
for m in re.finditer(ur'<h2 property="dc:title"', html):
end = html.find('</header>', m.start())
if end == -1:
continue
sub = html[m.start():end]
m1 = re.search(ur'<a href="(.+?)">(.+?)</a></h2>', sub)
if m1 is None:
print 'Error: no more details for %s' % url
continue
d = data.copy()
d[cm.url] = data['host'] + m1.group(1)
d[cm.name_e] = cm.html2plain(m1.group(2)).strip()
store_list.append(d)
def fetch_countries(data):
url = data['country_url']
try:
body = cm.get_data(url, {'display_country': 'CN'})
except Exception, e:
cm.dump('Error in fetching countries: %s' % url, log_name)
return ()
def fetch_countries(data):
url = data['url']
try:
body, data['cookie'] = cm.get_data_cookie(url)
except Exception, e:
cm.dump('Error in fetching countries: %s' % url, log_name)
return ()
def fetch_cities_beauty(data):
url = '%s/%s' % (data['city_url'], data['country_code'])
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching countries: %s' % url, log_name)
return ()
def get_cities(data):
try:
d = {"country": data["country_code"], "city": "", "service": -1}
html = common.post_data(url, d)
except Exception:
print "Error occured in getting the list of countries: %s" % url
dump_data = {"level": 1, "time": common.format_time(), "data": {"data": url}, "brand_id": brand_id}
common.dump(dump_data)
return []
start = html.find(u'<select id="city" name="city">')
if start == -1:
return []
end = html.find(u"</select>", start)
html = html[start:end]
city_list = []
for m in re.findall(ur'<option value="(.+?)">', html):
if data["country_code"] == "GB" and "2 davies street" in m.lower():
continue
elif data["country_code"] == "RO" and "13 september street" in m.lower():
continue
elif "b1603daq" in m.lower():
continue
else:
city_list.append({"city_e": m, "country_e": data["country_e"], "country_code": data["country_code"]})
def fetch_stores(data):
url = data['url']
try:
body = cm.get_data(url, {'m': data['m']})
except Exception, e:
cm.dump('Error in fetching stores: %s' % url, log_name)
return ()
def fetch_store_details(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching store details: %s' % url, log_name)
return ()
def fetch_store_details(data):
url = data['url']
try:
body, data['cookie'] = cm.get_data_cookie(url, cookie=data['cookie'])
except Exception, e:
cm.dump('Error in fetching stores: %s' % url, log_name)
return ()
def fetch_stores(data):
url = data['url']
try:
html = cm.get_data(url)
except Exception:
print 'Error occured: %s' % url
dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']}
cm.dump(dump_data)
return []
store_list = []
for m in re.findall(ur'var markerContent\s*?=\s*?"(.+?)".+?'
ur'createMarker\(.+?new google.maps.LatLng\((-?\d+\.\d+),(-?\d+\.\d+)\)', html, re.S):
entry = cm.init_store_entry(data['brand_id'], data['brandname_e'], data['brandname_c'])
lat, lng = map(string.atof, [m[1], m[2]])
cm.update_entry(entry, {cm.lat: lat, cm.lng: lng})
sub = m[0].strip()
m1 = re.search(ur'<b>(.+?)</b>', sub)
if m1 is None:
continue
entry[cm.name_c] = m1.group(1)
sub = sub.replace(m1.group(0), '')
m1=re.search(ur'聯系電話(?::|：)(.+?)<', sub)
if m1 is not None:
entry[cm.tel]=m1.group(1)
sub=sub.replace(m1.group(0), '<')
sub = re.sub(ur'<img\b.*?/>', '', sub)
entry[cm.addr_c] = cm.reformat_addr(sub)
print '(%s/%d) Found store: %s, %s (%s, %s)' % (data['brandname_e'], data['brand_id'],
entry[cm.name_c], entry[cm.addr_e], entry[cm.country_e],
entry[cm.continent_e])
store_list.append(entry)
db.insert_record(entry, 'stores')
def fetch_countries_eu(data):
url = data['url_eu']
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching EU countries: %s' % url, log_name)
return ()
def fetch_cities(data):
url = data['host'] + data['url']
try:
body = cm.get_data(url)
except Exception, e:
cm.dump('Error in fetching cities: %s' % url, log_name)
return []
def fetch_countries(data):
url = data["url"]
try:
body, data["cookie"] = cm.get_data_cookie(url)
except Exception, e:
cm.dump("Error in fetching AppKey: %s" % url, log_name)
return ()
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{
'func': lambda data: func(data, level + 1),
'data': s
} for s in fetch_countries(data)]
if level == 1:
# 城市列表
return [{
'func': lambda data: func(data, level + 1),
'data': s
} for s in fetch_cities(data)]
if level == 2:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {
'url': 'http://www.levi.com/GB/en_GB/findAStore',
'brand_id': 10215,
'brandname_e': u"Levi's",
'brandname_c': u"Levi's",
'city_map': gen_city_map()
}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
# db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def run2(opts):
try:
api = finding(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml)
response = api.execute('findItemsByProduct',
'<productId type="ReferenceID">53039031</productId><paginationInput><entriesPerPage>1</entriesPerPage></paginationInput>')
if hasattr(response.reply.searchResult, 'item'):
for r in response.reply.searchResult.item:
print("ID(%s) TITLE(%s)" % (r.itemId, r.title))
else:
print("No Items Found.")
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_continents(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching continents: %s' % url, log_name)
return []
m = re.search(ur'<nav\s+id\s*=\s*"retailers_nav">', body)
if m is None:
cm.dump('Error in fetching continents: %s' % url, log_name)
return []
body = cm.extract_closure(body[m.start():], ur'<nav\b', ur'</nav>')[0]
results = []
for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body):
d = data.copy()
d['url'] = data['host'] + m[0]
if 'europe' in d['url']:
results.append(d)
def fetch_stores(data):
if 'content' in data:
sub = data['content']
else:
url = data['data_url']
param = {
'action': 'fmd',
'id': 50,
'g': 1,
'page': 7,
'country': data['country_code'],
'_escaped_fragment_': data['fragment']
}
try:
body = cm.get_data(url, param)
except Exception, e:
cm.dump('Error in fetching store: %s, %s' % (url, param), log_name)
return []
sub = json.loads(body)['content']
def fetch_cities(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching cities: %s' % url, log_name)
return []
m = re.search(ur'Choose a city', body)
if m is None:
cm.dump('Error in fetching cities: %s' % url, log_name)
return []
body = cm.extract_closure(body[m.start():], ur'<ul>', ur'</ul>')[0]
results = []
for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body):
d = data.copy()
d['url'] = data['host'] + cm.html2plain(m[0])
d['city'] = cm.html2plain(m[1]).strip().upper()
results.append(d)
def getUser(opts):
try:
api = Trading(debug=opts.debug,
config_file=opts.yaml,
appid=opts.appid,
domain=opts.domain,
certid=opts.certid,
devid=opts.devid,
warnings=True,
timeout=20,
siteid='101')
api.execute('GetUser', {'UserID': 'sallyma789'})
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_store_hours(data):
url = data['store_url']
param = {'id': data['store_id']}
try:
s = json.loads(cm.get_data(url, param))['stores'][0]
workdays = ('Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun')
open_days = dict(
(key, s['hoursOpenFlag%s' % key] == 'Y') for key in workdays)
workhours = []
for d in workdays:
if not open_days[d]:
continue
workhours.append(
'%s: %s - %s' %
(d, s['hoursOpenTime%s' % d], s['hoursCloseTime%s' % d]))
return ', '.join(workhours)
except Exception, e:
cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name)
return ''
def fetch_stores(data):
url = data['data_url']
param = {
'module':
'pointsOfSaleAdvanced',
'action':
'ajaxGetURI',
'code_uri':
'%s/%s/%s/' %
tuple(data[key]
for key in ('continent_code', 'country_code', 'place_code')),
'languageId':
1
}
try:
new_url = cm.get_data(url, param)
except Exception, e:
cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name)
return []
def fetch_countries(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
m = re.search(ur'<select class="country-select"', body)
if m is None:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
body = cm.extract_closure(body[m.start():], ur'<select\b',
ur'</select>')[0]
results = []
for m in re.findall(ur'<option value="([A-Z]{2})">', body):
d = data.copy()
d['country_code'] = m
results.append(d)
def feedback(opts):
try:
api = Trading(debug=opts.debug,
config_file=opts.yaml,
appid=opts.appid,
certid=opts.certid,
devid=opts.devid,
warnings=False)
api.execute('GetFeedback', {'UserID': 'tim0th3us'})
dump(api)
if int(api.response_dict().FeedbackScore) > 50:
print("Doing good!")
else:
print("Sell more, buy more..")
except ConnectionError as e:
print e
def run(opts):
try:
api = finding(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=True)
api_request = {
#'keywords': u'niño',
'keywords':
u'GRAMMY Foundation®',
'itemFilter': [
{
'name': 'Condition',
'value': 'Used'
},
{
'name': 'LocatedIn',
'value': 'GB'
},
],
'affiliate': {
'trackingId': 1
},
'sortOrder':
'CountryDescending',
}
response = api.execute('findItemsAdvanced', api_request)
dump(api)
assert (response.reply.ack == 'Success')
assert (type(response.reply.timestamp) == datetime.datetime)
#assert(type(response.reply.searchResult.item) == list)
items = response.reply.searchResult.item
print(len(items))
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
try:
p = Parallel()
apis = []
api1 = finding(parallel=p,
debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml)
api1.execute('findItemsAdvanced', {'keywords': 'python'})
apis.append(api1)
api4 = html(parallel=p)
api4.execute('http://www.ebay.com/sch/i.html?_nkw=Shirt&_rss=1')
apis.append(api4)
api2 = finding(parallel=p,
debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml)
api2.execute('findItemsAdvanced', {'keywords': 'perl'})
apis.append(api2)
api3 = finding(parallel=p,
debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml)
api3.execute('findItemsAdvanced', {'keywords': 'php'})
apis.append(api3)
p.wait()
if p.error():
print(p.error())
for api in apis:
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 商店列表
return [{
'func': lambda data: func(data, level + 1),
'data': s
} for s in fetch_store_list(data)]
if level == 1:
# 商店
return [{
'func': None,
'data': s
} for s in fetch_store_details(data)]
else:
return []
# Walk from the root node, where level == 1.
if data is None:
data = {
'url':
'http://www.alexandermcqueen.com/experience/en/alexandermcqueen/store/',
'brand_id': 10008,
'brandname_e': u'Alexander McQueen',
'brandname_c': u'亚历山大·麦昆'
}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' %
('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{
'func': lambda data: func(data, level + 1),
'data': s
} for s in fetch_countries(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {
'data_url1': 'xxxxxxxxxx',
'data_url': 'http://dms.wolverineworldwide.com/SearchResults.aspx',
'brand_id': 10170,
'brandname_e': u'Hush Puppies',
'brandname_c': u'暇步士'
}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' %
('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
results.extend(fetch_stores_cn(data))
results.extend(fetch_stores_au(data))
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def memberMessages(args):
try:
api = Trading(debug=args.debug,
appid=app_id,
token=token_id,
certid=cert_id,
devid=dev_id,
config_file=None)
now = datetime.datetime.now()
memberData = {
"WarningLevel": "High",
"MailMessageType": "All",
# "MessageStatus": "Unanswered",
"StartCreationTime": now - datetime.timedelta(days=90),
"EndCreationTime": now,
"Pagination": {
"EntriesPerPage": "10",
"PageNumber": "1"
}
}
api.execute('GetMemberMessages', memberData)
dump(api)
if api.response.reply.has_key('MemberMessage'):
messages = api.response.reply.MemberMessage.MemberMessageExchange
if type(messages) != list:
messages = [messages]
for m in messages:
print(
"%s %s: %s" %
(m.CreationDate, m.Question.Subject[:50], m.Question.Body))
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_countries(data):
url = data['url']
try:
html = cm.get_data(url)
except Exception:
print 'Error occured: %s' % url
dump_data = {
'level': 0,
'time': cm.format_time(),
'data': {
'url': url
},
'brand_id': data['brand_id']
}
cm.dump(dump_data)
return []
# 处理重定向
m = re.search('<h2>Object moved to <a href="(.+?)">', html)
if m is not None:
data['url'] = data['host'] + m.group(1)
return fetch_countries(data)
m = re.search('<span class="country">Choose a country</span>', html)
if m is None:
return []
sub, start, end = cm.extract_closure(html[m.end():], r'<ul\b', r'</ul>')
if end == 0:
return []
country_list = []
for m in re.findall('<li><a .*?href="(.+?)">(.+?)</a></li>', sub):
d = data.copy()
country_e = cm.html2plain(m[1]).strip().upper()
ret = gs.look_up(country_e, 1)
if ret is not None:
country_e = ret['name_e']
d['country_e'] = country_e
d['province_e'] = ''
d['url'] = data['host'] + m[0]
country_list.append(d)
return country_list
def fetch_countries(data):
url = data['home_url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
m = re.search(ur'<select\s+id\s*=\s*"store-locator-country"', body)
if m is None:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
body = cm.extract_closure(body[m.start():], ur'<select\b',
ur'</select>')[0]
results = []
for m in re.findall(ur'<option value="([A-Z]{2})">', body):
d = data.copy()
d['country_code'] = m
# if m.upper() == 'GR':
results.append(d)
def get_store(args):
try:
api = Trading(debug=args.debug, appid=app_id, token=token_id,
certid=cert_id, devid=dev_id, warnings=True, timeout=20, siteid=site_id, config_file=None)
storeData = {
'MessageID': 'IT CIRCLE CONSULT PYTHON - MAFFAS',
'TaskID': '1',
}
api.execute('GetStoreCustomPage', storeData)
dump(api)
f = open('E:\\ITCIRCLECONSULT\\GITHUB\\maf-ebay\\maf-ebay-dev\\get-store\\'+ file_name + '.xml','a')
f.write(str(api.response.content)[2:-1]) # write result, removes the b' from the start, from position [2] until the ' at the end [-1]
f.close() #close html
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
api = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
dump(api)
print("Matching Titles:")
for item in response.reply.ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_store_list(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching store list: %s' % url, 'debeers_log.txt')
return []
start = body.find(u'<ul class="store-link-set">')
if start == -1:
cm.dump('Error in fetching store list: %s' % url, 'debeers_log.txt')
body = cm.extract_closure(body[start:], ur'<ul\b', ur'</ul>')[0]
results = []
for m in re.findall(ur'<li .*?>\s*<a href="(.+?)" title="(.+?)"', body,
re.S):
d = data.copy()
d['url'] = data['host'] + m[0]
d['name'] = m[1].strip()
results.append(d)
def with_affiliate_info(opts):
try:
api = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=True,
trackingid=1234,
trackingpartnercode=9)
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": 'shirt',
}
response = api.execute('FindPopularSearches', mySearch)
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch_stores(data):
url = data['data_url']
c_code = data['country_code']
param = {
'xml_request':
'<request><appkey>%s</appkey><formdata id="locatorsearch"><order>rank,_distance</order><geolocs><geoloc><longitude>%f</longitude><latitude>%f</latitude><country>%s</country></geoloc></geolocs><searchradius>100</searchradius><where><country><eq>%s</eq></country><or><columbia><eq>1</eq></columbia><outlet><eq>1</eq></outlet><store><eq>1</eq></store></or></where></formdata></request>'
% (data['cookie']['AppKey'], data['city_lat'], data['city_lng'],
c_code, c_code)
}
param = {
'xml_request':
'<request><appkey>%s</appkey><formdata id="locatorsearch"><order>rank,_distance</order><geolocs><geoloc><addressline></addressline><city></city><state></state><province></province><postalcode></postalcode><longitude>%f</longitude><latitude>%f</latitude><country>%s</country></geoloc></geolocs><searchradius>100</searchradius><where><country><eq>%s</eq></country><or><columbia><eq>1</eq></columbia><outlet><eq>1</eq></outlet><store><eq>1</eq></store></or></where></formdata></request>'
% (data['cookie']['AppKey'], data['city_lng'], data['city_lat'],
c_code, c_code)
}
try:
body = cm.get_data(url, param)
except Exception, e:
cm.dump('Error in fetching stores: %s, %s' % (url, param), log_name)
return ()
def fetch_countries(data):
url = data['home_url']
try:
body = cm.get_data(url)
except Exception:
print 'Error occured: %s' % url
dump_data = {'level': 0, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': data['brand_id']}
cm.dump(dump_data)
return []
start = body.find(u'<label>COUNTRY</label>')
if start==-1:
print 'Error occured in fetching country list: %s' % url
body = cm.extract_closure(body[start:], ur'<select\b', ur'</select>')[0]
results=[]
for m in re.findall(ur'<option value="([A-Z]{2})">.+?</option>', body):
d=data.copy()
d['country_code']=m
results.append(d)
def fetch_countries(data):
url = data['url']
try:
body = cm.get_data(url)
except Exception:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
m = re.search(ur'<a href="[^"]*" class="[^"]*">BOUTIQUE</a>', body)
if m is None:
cm.dump('Error in fetching countries: %s' % url, log_name)
return []
body = cm.extract_closure(body[m.end():], ur'<ul\b', ur'</ul>')[0]
results = []
for m in re.findall(ur'<a href="([^"]+)">([^<>]+)</a>', body):
d = data.copy()
d['region'] = m[1].strip()
d['url'] = data['host'] + m[0].strip()
results.append(d)
def parse_store(data, body=None):
if body is None:
url = data['url']
try:
body = cm.post_data(url)
except Exception:
cm.dump('Error in fetching stores: %s' % url, log_name)
return []
start = body.find(ur'jQuery.extend(Drupal.settings,')
latlng_map = {}
if start != -1:
for item in json.loads(
cm.extract_closure(
body[start:], ur'\{',
ur'\}')[0])['getlocations']['key_1']['latlons']:
latlng_map[cm.reformat_addr(item[3])] = {
'lat': string.atof(item[0]),
'lng': string.atof(item[1])
}
def getOrders(args):
try:
api = Trading(
debug=args.debug,
appid=app_id,
token=token_id,
certid=cert_id,
devid=dev_id,
warnings=True,
timeout=20,
config_file=None,
)
api.execute('GetOrders', {'NumberOfDays': 30})
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def feedback(args):
try:
api = Trading(debug=args.debug,
appid=app_id,
token=token_id,
config_file=None,
certid=cert_id,
devid=dev_id)
api.execute('GetFeedback', {'UserID': 'parties_wrapped_up'})
dump(api)
if int(api.response.reply.FeedbackScore) > 50:
print("Onwards and Upwards!")
else:
print("Onwards and Forwards!")
except ConnectionError as e:
print(e)
print(e.response.dict())
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{
'func': lambda data: func(data, level + 1),
'data': s
} for s in fetch_countries(data)]
if level == 1:
# 商店
return [{'func': None, 'data': s} for s in fetch_stores(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {
'url': 'http://global.tommy.com/int/en/Stores/Find-A-Store',
'brand_id': 10355,
'brandname_e': u'Tommy Hilfiger',
'brandname_c': u'汤米·希尔费格'
}
global db, id_set
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
# db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
rs = db.query_all(
'SELECT native_id FROM stores WHERE brand_id=%d and not native_id=""' %
data['brand_id'])
id_set = [tmp[0] for tmp in rs]
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def fetch(level=1, data=None, user='******', passwd=''):
def func(data, level):
"""
:param data:
:param level: 0：国家；1：城市；2：商店列表
"""
if level == 0:
# 国家列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_countries(data)]
if level == 1:
# 州列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_states(data)]
if level == 2:
# 城市列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_cities(data)]
if level == 3:
# 城市列表
return [{'func': lambda data: func(data, level + 1), 'data': s} for s in fetch_store_list(data)]
if level == 4:
# 商店
return [{'func': None, 'data': s} for s in fetch_store_details(data)]
else:
return ()
# Walk from the root node, where level == 1.
if data is None:
data = {
'url': 'http://www.swarovski.com.cn/is-bin/INTERSHOP.enfinity/WFS/SCO-Web_CN-Site/en_US/-/CNY/SMOD_Storefinder-ViewStorefinderSelects',
'store_url': 'http://www.swarovski.com.cn/Web_CN/en/boutique_search',
'brand_id': 10339, 'brandname_e': u'Swarovski', 'brandname_c': u'施华洛世奇'}
global db
db = cm.StoresDb()
db.connect_db(user=user, passwd=passwd)
db.execute(u'DELETE FROM %s WHERE brand_id=%d' % ('stores', data['brand_id']))
results = cm.walk_tree({'func': lambda data: func(data, 0), 'data': data})
db.disconnect_db()
cm.dump('Done!', log_name)
return results
def get_countries(data):
"""
返回国家列表
:rtype : [{'country_code':**, 'country':**}, ...]
:param data:
:return:
"""
url = data['url']
try:
html = cm.get_data(url)
except Exception:
print 'Error occured: %s' % url
dump_data = {'level': 1, 'time': cm.format_time(), 'data': {'url': url}, 'brand_id': brand_id}
cm.dump(dump_data)
return []
pat = '<option value="0">Choose a country</option>'
splits = [m.start() for m in re.finditer(pat, html)]
splits.append(-1)
sub_html = []
for i in xrange(len(splits) - 1):
sub_html.append(html[splits[i]:splits[i + 1]])
# 1：州信息
# s_map = [{'state_code':m[0], 'state':m[1].strip}
state_list = []
for m in re.findall(ur'<option value="(.+?)"\s*?>(.+?)</option>', sub_html[0][len(pat):]):
code = m[0].strip().upper()
state = m[1].strip().upper()
ret = gs.look_up(state, 2)
if ret is not None:
# state_list.append({'state': ret[0]['province_e'], 'state_code': ret[0]['state_code']})
state_list.append({'state': ret['name_e'], 'state_code': ret['code']})
else:
# state其实是写成是代码
for key in gs.province_map['data']:
state = gs.province_map['data'][key]
if state['code'] == code:
state = state['name_e']
state_list.append({'state': state, 'state_code': code})
break
