def run(opts):
try:
shopping = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
response = shopping.execute('FindPopularItems',
{'QueryKeywords': 'Python'})
nodes = response.dom().xpath('//ItemID')
itemIds = [n.text for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id, config_file=opts.yaml)
records = api.find_items_by_ids([itemIds[0]])
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def getShippingCost(itemId):
api = Shopping(appid=myAppId, config_file=None)
response = api.execute('GetShippingCosts', {
'itemID': itemId,
'DestinationPostalCode': '19711'
})
return response.dict()
def run(opts):
try:
shopping = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=False)
shopping.execute('FindPopularItems', {'QueryKeywords': 'Python'})
nodes = shopping.response_dom().getElementsByTagName('ItemID')
itemIds = [getNodeText(n) for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id,
config_file=opts.yaml)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print e
def get_single_item(self, item_id):
shopping_api = Shopping(appid=self.app_id, config_file=None)
shopping_response = shopping_api.execute(
"GetSingleItem", {
"ItemID": item_id,
'IncludeSelector': ['Details,ItemSpecifics']
})
return shopping_response.dict()
def get_session():
if not hasattr(thread_local, "api"):
thread_local.api = Shopping(config_file=None, domain='open.api.ebay.com',
appid="SatyaPra-MyEBPrac-PRD-abce464fb-dd2ae5fe",
devid="6c042b69-e90f-4897-9045-060858248665",
certid="PRD-bce464fbd03b-273a-4416-a299-7d41"
)
return thread_local.api
def connect(self, connection_type="Finding"):
if connection_type == "Finding":
self.api = Finding(appid=os.getenv("EBAY_API_ID"),
siteid='EBAY-GB',
config_file=None)
elif connection_type == "Shopping":
self.api = Shopping(appid=os.getenv("EBAY_API_ID"),
siteid='EBAY-GB',
config_file=None)
def download_images():
ebay_ids = []
etsy_ids = []
listings = Listing.objects.filter(image='default.jpg')
for listing in listings:
service, oid = listing.original_id.split('_')
if service == 'ebay':
ebay_ids.append(oid)
elif service == 'etsy':
etsy_ids.append(oid)
for etsy_id in etsy_ids:
request_url = 'https://openapi.etsy.com/v2/listings/{}/images/?api_key={}'.format(
etsy_id, creds['etsy']['key'])
response = requests.get(request_url).json().get('results', [])
oid = 'etsy_' + etsy_id
listing = Listing.objects.get(original_id=oid)
# print(listing.listing_image_set)
listing_images = []
for i, r in enumerate(response):
basename = '{}_{}.jpg'.format(oid, str(i).zfill(3))
outname = 'listing_images/' + basename
print(outname)
url = r['url_fullxfull']
download_file(url, outname)
listing.listingimage_set.create(image=basename)
listing_images.append(basename)
# listing.listingimage_set = listing_images
listing.image = listing_images[0]
listing.save()
ebay_api = Shopping(appid=creds['ebay']['appID'], config_file=None)
for _ebay_ids in chunks(ebay_ids, 20):
response = ebay_api.execute('GetMultipleItems', {
'ItemID': _ebay_ids
}).dict()
for item in response['Item']:
try:
oid = 'ebay_' + item['ItemID']
listing = Listing.objects.get(original_id=oid)
listing_images = []
urls = item['PictureURL']
for i, url in enumerate(urls[0:5]):
basename = '{}_{}.jpg'.format(oid, str(i).zfill(3))
outname = 'listing_images/' + basename
print(url, outname)
download_file(url, outname)
listing_images.append(basename)
listing.listingimage_set.create(image=basename)
listing.image = listing_images[0]
listing.save()
except:
continue
def getCategories(self):
catList = Shopping().execute("GetCategoryInfo", {
"CategoryID": "-1",
"IncludeSelector": "ChildCategories"
}).dict()["CategoryArray"]["Category"]
for cat in catList: #meow
if cat["CategoryID"] != "-1": #perhaps not, would give randomly selecting from all categories
catList.remove(cat)
return catList
def find_related_searches(searchword):
api = Shopping(appid=Ebay.KARL_ID, config_file=None)
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": searchword,
}
api.execute('FindPopularSearches', mySearch)
return (api.response_json())
def get_categorie_info(self):
""" Get category info for specific item """
shopping_api = Shopping()
# UK = 3, Germany = 77
response = shopping_api.execute('GetCategoryInfo', {
'CategoryID': -1,
'siteid': 77,
'IncludeSelector': 'ChildCategories'
})
pprint(response.reply)
def categoryInfo(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
api.execute('GetCategoryInfo', {"CategoryID": 3410})
dump(api, full=False)
except ConnectionError as e:
print e
def __init__(self):
try:
self.finding_api = Finding(appid='GUANGZHE-ece1779a-PRD-7c62ba413-e3800b20', siteid='EBAY-ENCA', config_file=None)
except ConnectionError as e:
print(e)
print(e.response.dict())
try:
self.shopping_api = Shopping(appid='GUANGZHE-ece1779a-PRD-7c62ba413-e3800b20', siteid='EBAY-ENCA', config_file=None)
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_singeItem(self, itemID):
from ebaysdk.shopping import Connection as Shopping
api = Shopping(debug=False,
appid=None,
config_file='ebay.yaml',
warnings=False)
response = api.execute('GetSingleItem', {
'ItemID': itemID,
'IncludeSelector': 'ItemSpecifics,Details'
})
return response
def categoryInfo(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
response = api.execute('GetCategoryInfo', {"CategoryID": 183473})
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def popularSearches(opts):
api = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=True)
choice = True
while choice:
choice = input('Search: ')
if choice == 'quit':
break
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": choice,
}
try:
response = api.execute('FindPopularSearches', mySearch)
dump(api, full=False)
print("Related: %s" %
response.reply.PopularSearchResult.RelatedSearches)
for term in response.reply.PopularSearchResult.AlternativeSearches.split(
';')[:3]:
api.execute('FindPopularItems', {
'QueryKeywords': term,
'MaxEntries': 3
})
print("Term: %s" % term)
try:
for item in response.reply.ItemArray.Item:
print(item.Title)
except AttributeError:
pass
dump(api)
print("\n")
except ConnectionError as e:
print(e)
print(e.response.dict())
def using_attributes(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True)
api.execute('FindProducts', {
"ProductID": {'@attrs': {'type': 'ISBN'},
'#text': '0596154488'}})
dump(api, full=False)
except ConnectionError as e:
print e
def get_shipping_cost(self, item_id, zip_code):
try:
api = Shopping(appid=self.app_id, config_file=None)
response = api.execute(
'GetShippingCosts', {
'DestinationCountryCode': 'US',
'DestinationPostalCode': zip_code,
'IncludeDetails': 'true',
'ItemID': item_id,
'QuantitySold': '1'
})
return response.dict()
except ConnectionError as e:
print(e)
print(e.response.dict())
def with_affiliate_info(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True,
trackingid=1234, trackingpartnercode=9)
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": 'shirt',
}
api.execute('FindPopularSearches', mySearch)
dump(api, full=True)
except ConnectionError as e:
print e
def with_affiliate_info(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True,
trackingid='1234', trackingpartnercode='9')
mySearch = {
"MaxEntries": 2,
"QueryKeywords": 'shirt',
}
response = api.execute('FindProducts', mySearch)
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def getResponseFromItemId(itemId):
""" Execute a request to the eBay API using an itemId.
The response returned contains informations about the ad.
Attributes:
itemId (str): The id of the eBay ad.
Returns:
ebaysdk.response.Response corresponding to the itemId."""
try:
api = Shopping(config_file="ebaysdk/ebay.yaml", siteid="EBAY-US")
response = api.execute("GetSingleItem", {"ItemID": itemId})
return response
except ConnectionError as e:
print(e)
print(e.response.dict())
def popularSearches(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
choice = True
while choice:
choice = input('Search: ')
if choice == 'quit':
break
mySearch = {
# "CategoryID": " string ",
# "IncludeChildCategories": " boolean ",
"MaxKeywords": 10,
"QueryKeywords": choice,
}
try:
api.execute('FindPopularSearches', mySearch)
#dump(api, full=True)
print("Related: %s" % api.response_dict().PopularSearchResult.RelatedSearches)
for term in api.response_dict().PopularSearchResult.AlternativeSearches.split(';')[:3]:
api.execute('FindPopularItems', {'QueryKeywords': term, 'MaxEntries': 3})
print("Term: %s" % term)
try:
for item in api.response_dict().ItemArray.Item:
print(item.Title)
except AttributeError:
pass
# dump(api)
print("\n")
except ConnectionError as e:
print e
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindProducts', {'QueryKeywords': 'Harry Potter', 'MaxEntries': 2, 'AvailableItemsOnly': 'true'})
dump(api)
print("Matching Titles:")
for item in response.reply.Product:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, domain=opts.domain,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
dump(api)
print("Matching Titles:")
for item in response.reply.ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def updateQuantitySoldEtc1(items):
#this multiple items api takes at a time 20 itemids only. So calling it repeatedly.
print("shopping")
api = Shopping(config_file=None,
domain='open.api.ebay.com',
appid="SatyaPra-MyEBPrac-PRD-abce464fb-dd2ae5fe",
devid="6c042b69-e90f-4897-9045-060858248665",
certid="PRD-bce464fbd03b-273a-4416-a299-7d41")
inputObj = {"ItemID": [], "IncludeSelector": "Details"}
inputObjects = []
j = 0
_ = 0
while _ < (len(items)):
print("_ values is: ", _, " , ", j)
if _ + 20 > len(items):
j = len(items)
else:
j = _ + 20
inputObj["ItemID"] = list(map(lambda x: x['itemId'], items[_:j]))
inputObjects.append(inputObj)
_ = j
print("started processing shopping api")
tic = time.perf_counter()
with concurrent.futures.ThreadPoolExecutor(int(len(items) /
(20))) as executor:
results = [
executor.submit(shoppingAPIUse, inputObject)
for inputObject in inputObjects
]
for future in results:
try:
yield future.result()
except:
traceback.print_exc()
print("leaving ")
print(shoppingApiResults)
toc = time.perf_counter()
print(f"time taken with multithread: {toc-tic}")
def updateQuantitySoldEtc(items):
#this multiple items api takes at a time 20 itemids only. So calling it repeatedly.
api = Shopping(config_file=None,
domain='open.api.ebay.com',
appid="SatyaPra-MyEBPrac-PRD-abce464fb-dd2ae5fe",
devid="6c042b69-e90f-4897-9045-060858248665",
certid="PRD-bce464fbd03b-273a-4416-a299-7d41")
inputObj = {"ItemID": [], "IncludeSelector": "Details"}
j = 0
_ = 0
tic = time.perf_counter()
while _ < (len(items)):
#print("_ values is: ",_," , ",j)
if _ + 20 > len(items):
j = len(items)
else:
j = _ + 20
inputObj["ItemID"] = list(map(lambda x: x['itemId'], items[_:j]))
#print("before adding sold, hitcount ",items[_:j])
response = api.execute('GetMultipleItems', inputObj).dict()
#print("response after executing multiple api call: ",response)
for i in range(len(response['Item'])):
items[_ +
i]['QuantitySold'] = response['Item'][i].get('QuantitySold')
items[_ + i]['HitCount'] = response['Item'][i].get('HitCount')
_ = j
#print("remaining items to process ",len(items)-i)
#correcting duration to start and end dates diff
for item in items:
#print("start time and ",item['listingInfo']['startTime']," end time; ", item['listingInfo']['endTime'])
startTime = datetime.datetime.strptime(
item['listingInfo']['startTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
endTime = datetime.datetime.strptime(item['listingInfo']['endTime'],
"%Y-%m-%dT%H:%M:%S.%fZ")
item['DurationCalc'] = (endTime.__sub__(startTime)).days
#print("duration is , ",item['DurationCalc'])
toc = time.perf_counter()
print(f"stopwatch: {toc - tic}")
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
if api.response_content():
print("Call Success: %s in length" % len(api.response_content()))
print("Response code: %s" % api.response_code())
print("Response DOM: %s" % api.response_dom())
dictstr = "%s" % api.response_dict()
print("Response dictionary: %s..." % dictstr[:50])
print("Matching Titles:")
for item in api.response_dict().ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print e
def run(opts):
# # SubCategories of Women Accessories
# category_ids = {'163573': 'belt-buckles',
# '3003' : 'belts',
# '177651' : 'collar-tips',
# '168998' : 'fascinators-headpieces',
# '105559' : 'gloves-mittens',
# '45220' : 'hair-accessories',
# '167906' : 'handkerchiefs',
# '45230' : 'hats',
# '169285' : 'id-document-holders',
# '45237' : 'key-chains-rings-finders',
# '15735' : 'organizers-day-planners',
# '45238' : 'scarves-wraps',
# '150955' : 'shoe-charms-jibbitz',
# '179247' : 'sunglasses-fashion-eyewear',
# '151486' : 'ties',
# # '105569' : 'umbrellas',
# '45258' : 'wallets',
# '175634' : 'wigs-extensions-supplies',
# # '106129' : 'wristbands',
# '15738' : 'mixed-items-lots',
# '1063' : 'other',
# }
# collectibles category
# category_ids = {'1': 'general',
# '34' : 'Advertising',
# '1335' : 'Animals',
# '13658' : 'Animation Art & Characters',
# '66502' : 'Arcade, Jukeboxes & Pinball',
# '14429' : 'Autographs',
# '66503' : 'Banks, Registers & Vending',
# '3265' : 'Barware',
# '156277' : 'Beads',
# '29797' : 'Bottles & Insulators',
# '562' : 'Breweriana, Beer',
# '898' : 'Casino',
# '397' : 'Clocks',
# '63' : 'Comics',
# '3913' : 'Cultures & Ethnicities',
# '13777' : 'Decorative Collectibles',
# '137' : 'Disneyana',
# '10860' : 'Fantasy, Mythical & Magic',
# '13877' : 'Historical Memorabilia',
# '907' : 'Holiday & Seasonal',
# '13905' : 'Kitchen & Home',
# '1401' : 'Knives, Swords & Blades',
# '1404' : 'Lamps, Lighting',
# '940' : 'Linens & Textiles (1930-Now)',
# '1430' : 'Metalware',
# '13956' : 'Militaria',
# '124' : 'Paper',
# '966' : 'Pens & Writing Instruments',
# '14005' : 'Pez, Keychains, Promo Glasses',
# '14277' : 'Photographic Images',
# '39507' : 'Pinbacks, Bobbles, Lunchboxes',
# '914' : 'Postcards',
# '29832' : 'Radio, Phonograph, TV, Phone',
# '1446' : 'Religion & Spirituality',
# '3213' : 'Rocks, Fossils & Minerals',
# '152' : 'Science Fiction & Horror',
# '412' : 'Science & Medicine (1930-Now)',
# '113' : 'Sewing (1930-Now)',
# '165800' : 'Souvenirs & Travel Memorabilia',
# '593' : 'Tobacciana',
# '13849' : 'Tools, Hardware & Locks',
# '868' : 'Trading Cards',
# '417' : 'Transportation',
# '597' : 'Vanity, Perfume & Shaving',
# '69851' : 'Vintage, Retro, Mid-Century',
# '45058' : 'Wholesale Lots',
# }
# category_ids = {
# '137085' : 'Athletic Apparel',
# '63862' : 'Coats & Jackets',
# '63861' : 'Dresses',
# '11524' : 'Hosiery & Socks',
# '11514' : 'Intimates & Sleep',
# '11554' : 'Jeans',
# '3009' : 'Jumpsuits & Rompers',
# '169001' : 'Leggings',
# '172378' : 'Maternity',
# '63863' : 'Pants',
# '11555' : 'Shorts',
# '63864' : 'Skirts',
# '63865' : 'Suits & Blazers',
# '63866' : 'Sweaters',
# '155226' : 'Sweats & Hoodies',
# '63867' : 'Swimwear',
# '63869' : 'T-Shirts',
# '53159' : 'Tops & Blouses',
# '15775' : 'Vests',
# '84275' : 'Mixed Items & Lots',
# '314' : 'Other'
# }
# category_ids = {
# '3034' : 'general',
# '95672' : 'Athletic',
# '53557' : 'Boots',
# '45333' : 'Flats & Oxfords',
# '55793' : 'Heels',
# '53548' : 'Occupational',
# '62107' : 'Sandals & Flip Flops',
# '11632' : 'Slippers',
# '63889' : 'Mixed Items & Lots',
# }
# category_ids = {
# '220' : 'general',
# '246' : 'Action Figures',
# '49019' : 'Beanbag Plush',
# '18991' : 'Building Toys',
# '19016' : 'Classic Toys',
# '222' : 'Diecast & Toy Vehicles',
# '11731' : 'Educational',
# '19071' : 'Electronic, Battery & Wind-Up',
# '19077' : 'Fast Food & Cereal Premiums',
# '233' : 'Games',
# '771' : 'Marbles',
# '479' : 'Model Railroads & Trains',
# '1188' : 'Models & Kits',
# '11743' : 'Outdoor Toys & Structures',
# '19169' : 'Preschool Toys & Pretend Play',
# '2613' : 'Puzzles',
# '2562' : 'Radio Control & Control Line',
# '19192' : 'Robots, Monsters & Space Toys',
# '2616' : 'Slot Cars',
# '436' : 'Stuffed Animals',
# '2631' : 'Toy Soldiers',
# '2536' : 'Trading Card Games',
# '2624' : 'TV, Movie & Character Toys',
# '717' : 'Vintage & Antique Toys',
# '40149' : 'Wholesale Lots',
# }
# general categories for CSA - toys
category_ids = {
'1063' : 'womens accessories',
'220' : 'toys',
'3034' : 'womens shoes',
'314' : 'womens clothing'
'1' : 'collectibles',
}
for category_id, category_name in category_ids.items():
directory = '/var/www/html/ebay-data/toys-and-hobbies/' + category_name
for page in range(1, 101):
if not os.path.exists(directory):
os.makedirs(directory)
if not os.path.exists(os.path.join(directory, str(page))):
try:
api = finding(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True)
api_request = {
'categoryId': category_id,
'paginationInput': {'pageNumber': page,
'entriesPerPage': 100}
}
response = api.execute('findItemsByCategory', api_request)
nodes = response.dom().xpath('//itemId')
item_ids = [n.text for n in nodes]
print category_id, category_name
if len(item_ids) > 0:
shop = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
prim_resp = shop.execute('GetMultipleItems',
{'IncludeSelector': 'ItemSpecifics',
'ItemID': item_ids[0:20]})
for j in range(20, len(item_ids), 20):
sub_resp = shop.execute('GetMultipleItems',
{'IncludeSelector': 'ItemSpecifics',
'ItemID': item_ids[j:j+20]})
prim_resp.dom().extend(sub_resp.dom().xpath('//Item'))
xml_file = open(os.path.join(directory, str(page)), 'w+')
stylesheet_tag = '<?xml-stylesheet type="text/xsl" href="/ebay-data/xslItemSpecifics.xsl"?>\n'
xml_file.write(stylesheet_tag)
xml_file.write(lxml.etree.tostring(prim_resp.dom(),
pretty_print=True))
xml_file.close()
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_product_from_ebay(item_id):
shopping = Shopping(siteid='EBAY-US', appid=keys.EBAY_APP_ID)
return shopping.execute('GetSingleItem', {'ItemID': item_id}).dict()['Item']
def get_new_posting_attrs(posting_ids):
n_items = len(posting_ids)
bike_attrs = defaultdict(list)
for var in ['itemId','title','price','URL','imageURL','imagefile','description',\
'endtime','location','biketype']:
bike_attrs[var] = []
api = Shopping(config_file='ebay.yaml')
if not (os.path.exists('data/' + 'ebay_postings.csv')):
fid = open('data/' + 'ebay_postings.csv', 'w')
for key in bike_attrs.keys():
fid.write(key + ',')
fid.write('\n')
fid.close()
# Get unique counter for saving images in dir 'data'
n = 0
files = os.listdir('data')
for name in files:
if name.endswith('.jpg') and name.startswith('ebay'):
if int(name[-9:-4]) >= n:
n = int(name[-9:-4]) + 1
print("Starting at image", n)
fid = open('data/' + 'ebay_postings.csv', 'a')
j = 0
for posting_id in posting_ids:
print("Posting", j, "of", n_items)
try:
response = api.execute(
'GetSingleItem', {
'ItemID': posting_id,
'version': '981',
'IncludeSelector': ['PictureDetails', 'ItemSpecifics'],
'outputSelector': 'PictureURLLarge'
})
item = response.reply.Item
try:
bike_attrs['imageURL'].append(item.PictureURL[0].replace(
',', ' '))
imagefile = 'ebay_image' + '{0:05d}'.format(n) + '.jpg'
urllib.request.urlretrieve(item.PictureURL[0],
'data/' + imagefile)
bike_attrs['imagefile'].append(imagefile)
try:
bike_attrs['location'].append(
item.Location.replace(',', ':'))
except:
locations.append([])
bike_attrs['URL'].append(item.ViewItemURLForNaturalSearch)
bike_attrs['itemId'].append(item.ItemID)
bike_attrs['title'].append(item.Title.replace(',', ' '))
bike_attrs['price'].append(item.ConvertedCurrentPrice.value)
bike_attrs['endtime'].append(str(item.EndTime.date()))
bike_attrs['biketype'].append(
get_biketype(item).replace(',', ' '))
response = api.execute(
'GetSingleItem', {
'ItemID': posting_id,
'version': '981',
'IncludeSelector': ['TextDescription']
})
item = response.reply.Item
bike_attrs['description'].append(
item.Description.replace('\n', ' ').replace(',', ' '))
j += 1
n += 1
for key in bike_attrs.keys():
fid.write(bike_attrs[key][-1] + ', ')
fid.write('\n')
except:
pass
except:
print("Item no longer exists")
fid.close()
return bike_attrs
def main():
"""
Main function
:return:
"""
# Read user input and initialize writer
store_name, country_code, pages_to_crawl = get_user_input()
raw_writer = get_csv_writer("%s_raw.csv" % store_name, raw_field_names)
raw_writer.writeheader()
shopee_dataframe = pd.DataFrame([])
# Initialize API connection
api_finding = Finding(appid=ebay_app_id, config_file=None, siteid=country_code)
api_shopping = Shopping(appid=ebay_app_id, config_file=None, siteid=country_code)
print "API Connection established"
# Find Item IDs in the store
sku_id_list = get_sku_id_list(store_name, pages_to_crawl, api_finding)
print "SKU ID information is obtained from API"
if sku_id_list:
times_to_query = int(len(sku_id_list) / 20) # One time can only query 20 skus
print "Start to crawl info for each sku..."
for i in range(times_to_query):
# Get sku details
response_sku = api_shopping.execute('GetMultipleItems', {'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'Details'}).dict()["Item"]
# Get sku variations
response_variations = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'Variations'}).dict()["Item"]
# Get sku descriptions
response_descriptions = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'TextDescription'}).dict()["Item"]
# Parse the api response and store them into 2 files
parse_sku_details_raw(details=response_sku, variations=response_variations,
descriptions=response_descriptions,
raw_writer=raw_writer)
shopee_dataframe = pd.concat([shopee_dataframe,
parse_sku_details_shopee(details=response_sku, variations=response_variations,
descriptions=response_descriptions)])
print "%s SKUs are crawled" % str((i + 1) * 20)
# In case there are skus that have not been crawled
# If there are 21 skus, the last sku will be crawled by the following code
if times_to_query * 20 < len(sku_id_list):
response_sku = api_shopping.execute('GetMultipleItems', {'ItemID': sku_id_list[times_to_query * 20:],
'IncludeSelector': 'Details'}).dict()["Item"]
response_variations = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[times_to_query * 20:],
'IncludeSelector': 'Variations'}).dict()["Item"]
parse_sku_details_raw(details=response_sku, variations=response_variations,
descriptions=response_descriptions,
raw_writer=raw_writer)
shopee_dataframe = pd.concat([shopee_dataframe,
parse_sku_details_shopee(details=response_sku, variations=response_variations,
descriptions=response_descriptions)])
print "All SKU info is crawled"
# Write to file
excel_writer = pd.ExcelWriter("%s_shopee_format.xlsx" % store_name)
shopee_dataframe = shopee_dataframe[shopee_field_names]
shopee_dataframe.to_excel(excel_writer, 'Sheet1', index=False)
excel_writer.save()
print "Data is written to files"
else:
# In case this store has no skus
print "Store %s does not have any available items" % store_name
