def update_items():
"""Update the prices, quantities, and availability of all items in the database"""
items = list(ITEM_DB.fetch())
items = [Item(**i) for i in items[0]]
api = Connection(appid=app.config['EBAY_APP_ID'], config_file=None)
for item in items:
try:
r = api.execute('GetSingleItem', {
'ItemID': str(item.ebay_id),
'IncludeSelector': 'Details'
})
except:
print('call failed for {}'.format(item.ebay_id))
continue
d = r.dict()
if d['Item']['ListingStatus'] == 'Completed':
ITEM_DB.delete(item.key)
else:
item.price = float(d['Item']['ConvertedCurrentPrice']['value'])
quantity = int(d['Item']['Quantity']) - int(
r.dict()['Item']['QuantitySold'])
item.quantity = quantity
item.available = True if quantity > 0 else False
item.picture_url = d['Item']['PictureURL'][0]
item.seller = d['Item']['Seller']['UserID']
ITEM_DB.put(item.dict())
def on_treeWidget_itemExpanded(self, item):
if item.checkState(1) != Qt.Checked:
categoryParentID = item.text(1)
session = Session()
count = session.query(Category).filter(Category.category_parent_id == categoryParentID).count()
if count == 0:
shopping = Shopping(warnings = False)
response = shopping.execute(
'GetCategoryInfo',
{
'CategoryID': categoryParentID,
'IncludeSelector': 'ChildCategories'
}
)
reply = response.reply
categoryArray = reply.CategoryArray.Category
for category in categoryArray:
if category.CategoryID == categoryParentID:
continue
session.add(Category(
name=category.CategoryName,
path=category.CategoryNamePath,
category_id=category.CategoryID,
category_parent_id=category.CategoryParentID,
leaf_category=category.LeafCategory)
)
session.commit()
for category in session.query(Category).filter(Category.category_parent_id == categoryParentID).order_by(Category.id).all():
child = QTreeWidgetItem(item, [category.name, category.category_id, category.path, category.leaf_category])
if category.leaf_category == 'false':
child.setChildIndicatorPolicy(QTreeWidgetItem.ShowIndicator)
item.setCheckState(1, Qt.Checked)
def run(opts):
try:
shopping = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
response = shopping.execute('FindPopularItems',
{'QueryKeywords': 'Python'})
nodes = response.dom().xpath('//ItemID')
itemIds = [n.text for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id, config_file=opts.yaml)
records = api.find_items_by_ids([itemIds[0]])
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def getShippingCost(itemId):
api = Shopping(appid=myAppId, config_file=None)
response = api.execute('GetShippingCosts', {
'itemID': itemId,
'DestinationPostalCode': '19711'
})
return response.dict()
def run(opts):
try:
shopping = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=False)
shopping.execute('FindPopularItems', {'QueryKeywords': 'Python'})
nodes = shopping.response_dom().getElementsByTagName('ItemID')
itemIds = [getNodeText(n) for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id,
config_file=opts.yaml)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print e
def run(opts):
try:
shopping = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
response = shopping.execute('FindPopularItems',
{'QueryKeywords': 'Python'})
nodes = response.dom().xpath('//ItemID')
itemIds = [n.text for n in nodes]
api = FindItem(debug=opts.debug,
consumer_id=opts.consumer_id, config_file=opts.yaml)
records = api.find_items_by_ids([itemIds[0]])
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_single_item(self, item_id):
shopping_api = Shopping(appid=self.app_id, config_file=None)
shopping_response = shopping_api.execute(
"GetSingleItem", {
"ItemID": item_id,
'IncludeSelector': ['Details,ItemSpecifics']
})
return shopping_response.dict()
def connect(self, connection_type="Finding"):
if connection_type == "Finding":
self.api = Finding(appid=os.getenv("EBAY_API_ID"),
siteid='EBAY-GB',
config_file=None)
elif connection_type == "Shopping":
self.api = Shopping(appid=os.getenv("EBAY_API_ID"),
siteid='EBAY-GB',
config_file=None)
def getCategories(self):
catList = Shopping().execute("GetCategoryInfo", {
"CategoryID": "-1",
"IncludeSelector": "ChildCategories"
}).dict()["CategoryArray"]["Category"]
for cat in catList: #meow
if cat["CategoryID"] != "-1": #perhaps not, would give randomly selecting from all categories
catList.remove(cat)
return catList
def download_images():
ebay_ids = []
etsy_ids = []
listings = Listing.objects.filter(image='default.jpg')
for listing in listings:
service, oid = listing.original_id.split('_')
if service == 'ebay':
ebay_ids.append(oid)
elif service == 'etsy':
etsy_ids.append(oid)
for etsy_id in etsy_ids:
request_url = 'https://openapi.etsy.com/v2/listings/{}/images/?api_key={}'.format(
etsy_id, creds['etsy']['key'])
response = requests.get(request_url).json().get('results', [])
oid = 'etsy_' + etsy_id
listing = Listing.objects.get(original_id=oid)
# print(listing.listing_image_set)
listing_images = []
for i, r in enumerate(response):
basename = '{}_{}.jpg'.format(oid, str(i).zfill(3))
outname = 'listing_images/' + basename
print(outname)
url = r['url_fullxfull']
download_file(url, outname)
listing.listingimage_set.create(image=basename)
listing_images.append(basename)
# listing.listingimage_set = listing_images
listing.image = listing_images[0]
listing.save()
ebay_api = Shopping(appid=creds['ebay']['appID'], config_file=None)
for _ebay_ids in chunks(ebay_ids, 20):
response = ebay_api.execute('GetMultipleItems', {
'ItemID': _ebay_ids
}).dict()
for item in response['Item']:
try:
oid = 'ebay_' + item['ItemID']
listing = Listing.objects.get(original_id=oid)
listing_images = []
urls = item['PictureURL']
for i, url in enumerate(urls[0:5]):
basename = '{}_{}.jpg'.format(oid, str(i).zfill(3))
outname = 'listing_images/' + basename
print(url, outname)
download_file(url, outname)
listing_images.append(basename)
listing.listingimage_set.create(image=basename)
listing.image = listing_images[0]
listing.save()
except:
continue
def get_categorie_info(self):
""" Get category info for specific item """
shopping_api = Shopping()
# UK = 3, Germany = 77
response = shopping_api.execute('GetCategoryInfo',
{'CategoryID': -1,
'siteid': 77,
'IncludeSelector': 'ChildCategories'})
pprint(response.reply)
def find_related_searches(searchword):
api = Shopping(appid=Ebay.KARL_ID, config_file=None)
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": searchword,
}
api.execute('FindPopularSearches', mySearch)
return (api.response_json())
def get_categorie_info(self):
""" Get category info for specific item """
shopping_api = Shopping()
# UK = 3, Germany = 77
response = shopping_api.execute('GetCategoryInfo', {
'CategoryID': -1,
'siteid': 77,
'IncludeSelector': 'ChildCategories'
})
pprint(response.reply)
def categoryInfo(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
api.execute('GetCategoryInfo', {"CategoryID": 3410})
dump(api, full=False)
except ConnectionError as e:
print e
def categoryInfo(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
response = api.execute('GetCategoryInfo', {'CategoryID': '63863'})
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def categoryInfo(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
response = api.execute('GetCategoryInfo', {"CategoryID": 183473})
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_singeItem(self, itemID):
from ebaysdk.shopping import Connection as Shopping
api = Shopping(debug=False,
appid=None,
config_file='ebay.yaml',
warnings=False)
response = api.execute('GetSingleItem', {
'ItemID': itemID,
'IncludeSelector': 'ItemSpecifics,Details'
})
return response
def popularSearches(opts):
api = Shopping(debug=opts.debug,
appid=opts.appid,
config_file=opts.yaml,
warnings=True)
choice = True
while choice:
choice = input('Search: ')
if choice == 'quit':
break
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": choice,
}
try:
response = api.execute('FindPopularSearches', mySearch)
dump(api, full=False)
print("Related: %s" %
response.reply.PopularSearchResult.RelatedSearches)
for term in response.reply.PopularSearchResult.AlternativeSearches.split(
';')[:3]:
api.execute('FindPopularItems', {
'QueryKeywords': term,
'MaxEntries': 3
})
print("Term: %s" % term)
try:
for item in response.reply.ItemArray.Item:
print(item.Title)
except AttributeError:
pass
dump(api)
print("\n")
except ConnectionError as e:
print(e)
print(e.response.dict())
def using_attributes(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True)
api.execute('FindProducts', {
"ProductID": {'@attrs': {'type': 'ISBN'},
'#text': '0596154488'}})
dump(api, full=False)
except ConnectionError as e:
print e
def get_category_info(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, warnings=True)
api_request = {
'CategoryID': 63863,
}
#response = api.execute('GetCategoryInfo', api_request)
response = api.execute('GetCategoryInfo', {"CategoryID": 63863})
print (response.dict())
except ConnectionError as e:
print(e)
print(e.response.dict())
def get_shipping_cost(self, item_id, zip_code):
try:
api = Shopping(appid=self.app_id, config_file=None)
response = api.execute(
'GetShippingCosts', {
'DestinationCountryCode': 'US',
'DestinationPostalCode': zip_code,
'IncludeDetails': 'true',
'ItemID': item_id,
'QuantitySold': '1'
})
return response.dict()
except ConnectionError as e:
print(e)
print(e.response.dict())
def with_affiliate_info(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True,
trackingid=1234, trackingpartnercode=9)
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": 'shirt',
}
api.execute('FindPopularSearches', mySearch)
dump(api, full=True)
except ConnectionError as e:
print e
def popularSearches(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
choice = True
while choice:
choice = input('Search: ')
if choice == 'quit':
break
mySearch = {
"MaxKeywords": 10,
"QueryKeywords": choice,
}
try:
response = api.execute('FindPopularSearches', mySearch)
dump(api, full=False)
print("Related: %s" %
response.reply.PopularSearchResult.RelatedSearches)
for term in response.reply.PopularSearchResult.AlternativeSearches.split(';')[:3]:
api.execute('FindPopularItems', {
'QueryKeywords': term, 'MaxEntries': 3})
print("Term: %s" % term)
try:
for item in response.reply.ItemArray.Item:
print(item.Title)
except AttributeError:
pass
dump(api)
print("\n")
except ConnectionError as e:
print(e)
print(e.response.dict())
def getResponseFromItemId(itemId):
""" Execute a request to the eBay API using an itemId.
The response returned contains informations about the ad.
Attributes:
itemId (str): The id of the eBay ad.
Returns:
ebaysdk.response.Response corresponding to the itemId."""
try:
api = Shopping(config_file="ebaysdk/ebay.yaml", siteid="EBAY-US")
response = api.execute("GetSingleItem", {"ItemID": itemId})
return response
except ConnectionError as e:
print(e)
print(e.response.dict())
def with_affiliate_info(opts):
try:
api = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True,
trackingid='1234', trackingpartnercode='9')
mySearch = {
"MaxEntries": 2,
"QueryKeywords": 'shirt',
}
response = api.execute('FindProducts', mySearch)
dump(api, full=False)
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
dump(api)
print("Matching Titles:")
for item in response.reply.ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml, domain=opts.domain,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
dump(api)
print("Matching Titles:")
for item in response.reply.ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
response = api.execute('FindProducts', {'QueryKeywords': 'Harry Potter', 'MaxEntries': 2, 'AvailableItemsOnly': 'true'})
dump(api)
print("Matching Titles:")
for item in response.reply.Product:
print(item.Title)
except ConnectionError as e:
print(e)
print(e.response.dict())
def on_searchPushButton_clicked(self, checked):
categoryCount = self.ui.categoryTreeWidget.topLevelItemCount()
queryKeywords = self.ui.queryKeywordsLineEdit.text()
if categoryCount == 0 and queryKeywords == '':
return
categoryID = None
includeChild = self.ui.includeChildCheckBox.checkState() == Qt.Checked
if categoryCount == 1:
categoryID = self.ui.categoryTreeWidget.topLevelItem(0).text(0)
elif categoryCount > 1:
if includeChild:
categoryID = self.ui.categoryTreeWidget.topLevelItem(0).text(0)
else:
categoryID = list()
for i in range(0, categoryCount):
categoryID.append(self.ui.categoryTreeWidget.topLevelItem(i).text(0))
shopping = Shopping(warnings = False, trackingpartnercode=2)
data = {'MaxKeywords': self.ui.spinBox.value(),
'MaxResultsPerPage': 200,
'PageNumber': 1}
if includeChild:
data['IncludeChildCategories'] = True
if categoryID != None:
data['CategoryID'] = categoryID
if queryKeywords != '':
data['QueryKeywords'] = self.ui.queryKeywordsLineEdit.text()
response = shopping.execute('FindPopularSearches', data)
reply = response.reply
popularSearchResult = iterable(reply.PopularSearchResult)
row = self.ui.tableWidget.rowCount()
for result in popularSearchResult:
self.ui.tableWidget.insertRow(row)
if result.has_key('CategoryParentName'):
self.ui.tableWidget.setItem(row, 0, QTableWidgetItem(result.CategoryParentName))
if result.has_key('CategoryName'):
self.ui.tableWidget.setItem(row, 1, QTableWidgetItem(result.CategoryName))
if result.has_key('RelatedSearches'):
self.ui.tableWidget.setItem(row, 2, QTableWidgetItem(result.RelatedSearches))
if result.has_key('AlternativeSearches'):
self.ui.tableWidget.setItem(row, 3, QTableWidgetItem(result.AlternativeSearches))
if result.has_key('QueryKeywords'):
self.ui.tableWidget.setItem(row, 4, QTableWidgetItem(result.QueryKeywords))
row += 1
def getItemDetails(itemId):
# initialise eBay Shopping API
api2 = Shopping(appid='INSERT_APP_ID')
# get auction item including textual description and item specific information
api2.execute('GetSingleItem', {
'ItemID': str(itemId),
'IncludeSelector': 'TextDescription,ItemSpecifics'
})
response = json.loads(api2.response_json())
# check if all desired text fields are available
if set(('Description','PictureURL','ConditionDisplayName','ConvertedCurrentPrice','PrimaryCategoryName','Title')).issubset(response['Item'].keys()):
# collect the data
if 'value' in response['Item']['Description'].keys():
description = unicodedata.normalize('NFKD', response['Item']['Description']['value']).encode('ascii','ignore')
else:
description = ''
pictures = response['Item']['PictureURL']
condition = str(response['Item']['ConditionDisplayName']['value'])
price = float(response['Item']['ConvertedCurrentPrice']['value'])
category = str(response['Item']['PrimaryCategoryName']['value'])
title = str(response['Item']['Title']['value'])
# return the item only if the auction has a description longer than 100 characters and more than 2 images
if (len(description) >= 100) & (len(pictures) > 2) & (len(condition) > 0) & (price != 0):
pictures_list = list()
for y in range(0,len(pictures)):
pictures_list.append(str(pictures[y]['value']))
item_list = list()
item_list.append(title)
item_list.append(description)
item_list.append(category)
item_list.append(condition)
item_list.append(price)
item_list.append(pictures_list)
return item_list
return 'null'
def updateQuantitySoldEtc(items):
#this multiple items api takes at a time 20 itemids only. So calling it repeatedly.
api = Shopping(config_file=None,
domain='open.api.ebay.com',
appid="SatyaPra-MyEBPrac-PRD-abce464fb-dd2ae5fe",
devid="6c042b69-e90f-4897-9045-060858248665",
certid="PRD-bce464fbd03b-273a-4416-a299-7d41")
inputObj = {"ItemID": [], "IncludeSelector": "Details"}
j = 0
_ = 0
tic = time.perf_counter()
while _ < (len(items)):
#print("_ values is: ",_," , ",j)
if _ + 20 > len(items):
j = len(items)
else:
j = _ + 20
inputObj["ItemID"] = list(map(lambda x: x['itemId'], items[_:j]))
#print("before adding sold, hitcount ",items[_:j])
response = api.execute('GetMultipleItems', inputObj).dict()
#print("response after executing multiple api call: ",response)
for i in range(len(response['Item'])):
items[_ +
i]['QuantitySold'] = response['Item'][i].get('QuantitySold')
items[_ + i]['HitCount'] = response['Item'][i].get('HitCount')
_ = j
#print("remaining items to process ",len(items)-i)
#correcting duration to start and end dates diff
for item in items:
#print("start time and ",item['listingInfo']['startTime']," end time; ", item['listingInfo']['endTime'])
startTime = datetime.datetime.strptime(
item['listingInfo']['startTime'], "%Y-%m-%dT%H:%M:%S.%fZ")
endTime = datetime.datetime.strptime(item['listingInfo']['endTime'],
"%Y-%m-%dT%H:%M:%S.%fZ")
item['DurationCalc'] = (endTime.__sub__(startTime)).days
#print("duration is , ",item['DurationCalc'])
toc = time.perf_counter()
print(f"stopwatch: {toc - tic}")
def _call_shopping_api(self, ids, ebay_site):
"""
Call Ebay's shopping API to get complete information about a listing.
"""
try:
api = SConnection(config_file=self.keyfile, siteid=ebay_site)
response = api.execute('GetMultipleItems',
{'IncludeSelector': 'Description,Details,ItemSpecifics,ShippingCosts',
'ItemID': ids})
except (ebaysdk.exception.ConnectionError,
requests.exceptions.ConnectionError) as err:
err_text = 'Downloading full item information from Ebay failed! ' \
'Error: ' + str(err)
logging.error(err_text)
logging.debug(err.response.dict())
raise EbayError(err_text)
# #TODO: react on the following status information
# # Returns the HTTP response code.
# response_code()
# # Returns the HTTP response status
# response_status()
# # Returns an array of eBay response codes
# response_codes()
resp_dict = response.dict()
# Act on resonse status
if resp_dict['Ack'] == 'Success':
logging.debug('Successfully called Ebay shopping API.')
elif resp_dict['Ack'] in ['Warning', 'PartialFailure']:
logging.warning('Ebay shopping API returned warning.')
logging.debug(pformat(resp_dict['Errors']))
else:
logging.error('Ebay shopping API returned error.')
logging.debug(pformat(resp_dict))
raise EbayError('Ebay shopping API returned error.')
return resp_dict
def run(opts):
try:
shopping = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
shopping.execute('FindPopularItems', {'QueryKeywords': 'Python'})
nodes = shopping.response_dom().getElementsByTagName('ItemID')
itemIds = [getNodeText(n) for n in nodes]
api = FindItem(debug=opts.debug, consumer_id=opts.consumer_id, config_file=opts.yaml)
records = api.find_items_by_ids(itemIds)
for r in records:
print("ID(%s) TITLE(%s)" % (r['ITEM_ID'], r['TITLE'][:35]))
dump(api)
except ConnectionError as e:
print e
def update_item(ebay_id):
"""Update the info for the specified item"""
api = Connection(appid=app.config['EBAY_APP_ID'], config_file=None)
item = next(ITEM_DB.fetch(query={"ebay_id": ebay_id}))
if item:
item = Item(**item[0])
try:
r = api.execute('GetSingleItem', {
'ItemID': str(item.ebay_id),
'IncludeSelector': 'Details'
})
except:
print('call failed for {}'.format(item.ebay_id))
return
item.price = float(r.dict()['Item']['ConvertedCurrentPrice']['value'])
quantity = int(r.dict()['Item']['Quantity']) - int(
r.dict()['Item']['QuantitySold'])
item.quantity = quantity
item.available = True if quantity > 0 else False
item.picture_url = r.dict()['Item']['PictureURL'][0]
item.seller = r.dict()['Item']['Seller']['UserID']
ITEM_DB.put(item.dict())
else:
print('No item with that ID')
def popularSearches(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
choice = True
while choice:
choice = input('Search: ')
if choice == 'quit':
break
mySearch = {
# "CategoryID": " string ",
# "IncludeChildCategories": " boolean ",
"MaxKeywords": 10,
"QueryKeywords": choice,
}
try:
api.execute('FindPopularSearches', mySearch)
#dump(api, full=True)
print("Related: %s" % api.response_dict().PopularSearchResult.RelatedSearches)
for term in api.response_dict().PopularSearchResult.AlternativeSearches.split(';')[:3]:
api.execute('FindPopularItems', {'QueryKeywords': term, 'MaxEntries': 3})
print("Term: %s" % term)
try:
for item in api.response_dict().ItemArray.Item:
print(item.Title)
except AttributeError:
pass
# dump(api)
print("\n")
except ConnectionError as e:
print e
def run(opts):
api = Shopping(debug=opts.debug, appid=opts.appid, config_file=opts.yaml,
warnings=True)
print("Shopping samples for SDK version %s" % ebaysdk.get_version())
try:
api.execute('FindPopularItems', {'QueryKeywords': 'Python'})
if api.response_content():
print("Call Success: %s in length" % len(api.response_content()))
print("Response code: %s" % api.response_code())
print("Response DOM: %s" % api.response_dom())
dictstr = "%s" % api.response_dict()
print("Response dictionary: %s..." % dictstr[:50])
print("Matching Titles:")
for item in api.response_dict().ItemArray.Item:
print(item.Title)
except ConnectionError as e:
print e
def run(opts):
# # SubCategories of Women Accessories
# category_ids = {'163573': 'belt-buckles',
# '3003' : 'belts',
# '177651' : 'collar-tips',
# '168998' : 'fascinators-headpieces',
# '105559' : 'gloves-mittens',
# '45220' : 'hair-accessories',
# '167906' : 'handkerchiefs',
# '45230' : 'hats',
# '169285' : 'id-document-holders',
# '45237' : 'key-chains-rings-finders',
# '15735' : 'organizers-day-planners',
# '45238' : 'scarves-wraps',
# '150955' : 'shoe-charms-jibbitz',
# '179247' : 'sunglasses-fashion-eyewear',
# '151486' : 'ties',
# # '105569' : 'umbrellas',
# '45258' : 'wallets',
# '175634' : 'wigs-extensions-supplies',
# # '106129' : 'wristbands',
# '15738' : 'mixed-items-lots',
# '1063' : 'other',
# }
# collectibles category
# category_ids = {'1': 'general',
# '34' : 'Advertising',
# '1335' : 'Animals',
# '13658' : 'Animation Art & Characters',
# '66502' : 'Arcade, Jukeboxes & Pinball',
# '14429' : 'Autographs',
# '66503' : 'Banks, Registers & Vending',
# '3265' : 'Barware',
# '156277' : 'Beads',
# '29797' : 'Bottles & Insulators',
# '562' : 'Breweriana, Beer',
# '898' : 'Casino',
# '397' : 'Clocks',
# '63' : 'Comics',
# '3913' : 'Cultures & Ethnicities',
# '13777' : 'Decorative Collectibles',
# '137' : 'Disneyana',
# '10860' : 'Fantasy, Mythical & Magic',
# '13877' : 'Historical Memorabilia',
# '907' : 'Holiday & Seasonal',
# '13905' : 'Kitchen & Home',
# '1401' : 'Knives, Swords & Blades',
# '1404' : 'Lamps, Lighting',
# '940' : 'Linens & Textiles (1930-Now)',
# '1430' : 'Metalware',
# '13956' : 'Militaria',
# '124' : 'Paper',
# '966' : 'Pens & Writing Instruments',
# '14005' : 'Pez, Keychains, Promo Glasses',
# '14277' : 'Photographic Images',
# '39507' : 'Pinbacks, Bobbles, Lunchboxes',
# '914' : 'Postcards',
# '29832' : 'Radio, Phonograph, TV, Phone',
# '1446' : 'Religion & Spirituality',
# '3213' : 'Rocks, Fossils & Minerals',
# '152' : 'Science Fiction & Horror',
# '412' : 'Science & Medicine (1930-Now)',
# '113' : 'Sewing (1930-Now)',
# '165800' : 'Souvenirs & Travel Memorabilia',
# '593' : 'Tobacciana',
# '13849' : 'Tools, Hardware & Locks',
# '868' : 'Trading Cards',
# '417' : 'Transportation',
# '597' : 'Vanity, Perfume & Shaving',
# '69851' : 'Vintage, Retro, Mid-Century',
# '45058' : 'Wholesale Lots',
# }
# category_ids = {
# '137085' : 'Athletic Apparel',
# '63862' : 'Coats & Jackets',
# '63861' : 'Dresses',
# '11524' : 'Hosiery & Socks',
# '11514' : 'Intimates & Sleep',
# '11554' : 'Jeans',
# '3009' : 'Jumpsuits & Rompers',
# '169001' : 'Leggings',
# '172378' : 'Maternity',
# '63863' : 'Pants',
# '11555' : 'Shorts',
# '63864' : 'Skirts',
# '63865' : 'Suits & Blazers',
# '63866' : 'Sweaters',
# '155226' : 'Sweats & Hoodies',
# '63867' : 'Swimwear',
# '63869' : 'T-Shirts',
# '53159' : 'Tops & Blouses',
# '15775' : 'Vests',
# '84275' : 'Mixed Items & Lots',
# '314' : 'Other'
# }
# category_ids = {
# '3034' : 'general',
# '95672' : 'Athletic',
# '53557' : 'Boots',
# '45333' : 'Flats & Oxfords',
# '55793' : 'Heels',
# '53548' : 'Occupational',
# '62107' : 'Sandals & Flip Flops',
# '11632' : 'Slippers',
# '63889' : 'Mixed Items & Lots',
# }
# category_ids = {
# '220' : 'general',
# '246' : 'Action Figures',
# '49019' : 'Beanbag Plush',
# '18991' : 'Building Toys',
# '19016' : 'Classic Toys',
# '222' : 'Diecast & Toy Vehicles',
# '11731' : 'Educational',
# '19071' : 'Electronic, Battery & Wind-Up',
# '19077' : 'Fast Food & Cereal Premiums',
# '233' : 'Games',
# '771' : 'Marbles',
# '479' : 'Model Railroads & Trains',
# '1188' : 'Models & Kits',
# '11743' : 'Outdoor Toys & Structures',
# '19169' : 'Preschool Toys & Pretend Play',
# '2613' : 'Puzzles',
# '2562' : 'Radio Control & Control Line',
# '19192' : 'Robots, Monsters & Space Toys',
# '2616' : 'Slot Cars',
# '436' : 'Stuffed Animals',
# '2631' : 'Toy Soldiers',
# '2536' : 'Trading Card Games',
# '2624' : 'TV, Movie & Character Toys',
# '717' : 'Vintage & Antique Toys',
# '40149' : 'Wholesale Lots',
# }
# general categories for CSA - toys
category_ids = {
'1063' : 'womens accessories',
'220' : 'toys',
'3034' : 'womens shoes',
'314' : 'womens clothing'
'1' : 'collectibles',
}
for category_id, category_name in category_ids.items():
directory = '/var/www/html/ebay-data/toys-and-hobbies/' + category_name
for page in range(1, 101):
if not os.path.exists(directory):
os.makedirs(directory)
if not os.path.exists(os.path.join(directory, str(page))):
try:
api = finding(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=True)
api_request = {
'categoryId': category_id,
'paginationInput': {'pageNumber': page,
'entriesPerPage': 100}
}
response = api.execute('findItemsByCategory', api_request)
nodes = response.dom().xpath('//itemId')
item_ids = [n.text for n in nodes]
print category_id, category_name
if len(item_ids) > 0:
shop = Shopping(debug=opts.debug, appid=opts.appid,
config_file=opts.yaml, warnings=False)
prim_resp = shop.execute('GetMultipleItems',
{'IncludeSelector': 'ItemSpecifics',
'ItemID': item_ids[0:20]})
for j in range(20, len(item_ids), 20):
sub_resp = shop.execute('GetMultipleItems',
{'IncludeSelector': 'ItemSpecifics',
'ItemID': item_ids[j:j+20]})
prim_resp.dom().extend(sub_resp.dom().xpath('//Item'))
xml_file = open(os.path.join(directory, str(page)), 'w+')
stylesheet_tag = '<?xml-stylesheet type="text/xsl" href="/ebay-data/xslItemSpecifics.xsl"?>\n'
xml_file.write(stylesheet_tag)
xml_file.write(lxml.etree.tostring(prim_resp.dom(),
pretty_print=True))
xml_file.close()
except ConnectionError as e:
print(e)
print(e.response.dict())
def getItemDetails(item, nbr):
itemId = item['itemId']['value']
# initialise eBay Shopping API
api2 = Shopping(appid='INSERT_APP_ID')
try:
# get item details
api2.execute('GetSingleItem', {
'ItemID': str(itemId),
'IncludeSelector': 'TextDescription,ItemSpecifics,Details,Variations'
})
response = json.loads(api2.response_json())
except ConnectionError:
return 'null'
fields = set(('Description','PictureURL','ConditionDisplayName','ConvertedCurrentPrice','PrimaryCategoryName','Title','ItemSpecifics','Seller','StartTime','EndTime','HandlingTime','GlobalShipping','ReturnPolicy','Quantity'))
pure_auction = (item['listingInfo']['bestOfferEnabled']['value'] == 'false') & (item['listingInfo']['buyItNowAvailable']['value'] == 'false')
# check if all needed fields are available
if fields.issubset(response['Item'].keys()) & pure_auction:
# collect the data (features)
if 'value' in response['Item']['Description'].keys():
description = unicodedata.normalize('NFKD', response['Item']['Description']['value']).encode('ascii','ignore')
else:
description = ''
pictures = response['Item']['PictureURL']
condition = convertConditionToNumber(response['Item']['ConditionDisplayName']['value'])
price = float(response['Item']['ConvertedCurrentPrice']['value'])
category = str(response['Item']['PrimaryCategoryName']['value'])
title = str(unicodedata.normalize('NFKD', response['Item']['Title']['value']).encode('ascii','ignore'))
if ((title.lower().find('lot') >= 0) | (title.lower().find('set') >= 0) | (title.lower().find('pack') >= 0)):
return 'null'
model = -1
specifics = response['Item']['ItemSpecifics']['NameValueList']
seller_rating = float(response['Item']['Seller']['PositiveFeedbackPercent']['value'])
seller_rating_count = convertRatingStarToNumber(response['Item']['Seller']['FeedbackRatingStar']['value'])
start_time = dateutil.parser.parse(response['Item']['StartTime']['value'])
end_time = dateutil.parser.parse(response['Item']['EndTime']['value'])
end_weekday = end_time.isoweekday()
start_weekday = start_time.isoweekday()
end_hour = end_time.hour
if (start_time.hour == end_time.hour) & (start_time.minute == end_time.minute) & (start_time.second == end_time.second):
duration = (end_time - start_time).days
else:
duration = 0
shipping_locations = convertShipToLocationsToNumber(item['shippingInfo']['shipToLocations'])
shipping_type = convertShippingTypeToNumber(str(item['shippingInfo']['shippingType']['value']))
handling_time = int(response['Item']['HandlingTime']['value'])
global_shipping = 0
if str(response['Item']['GlobalShipping']['value']) == 'true':
global_shipping = 1
nbr_of_pictures = len(pictures)
len_of_description = len(description)
returns_accepted = 1
if str(response['Item']['ReturnPolicy']['ReturnsAccepted']['value']).find('ReturnsNotAccepted') >= 0:
returns_accepted = 0
if len(specifics) > 3:
for entry in specifics:
if (entry['Name']['value'] == 'Scale'):
if not (entry['Value']['value'] == '1:64'):
return 'null'
if (entry['Name']['value'] == 'Brand'):
if not (entry['Value']['value'] == 'Hot Wheels'):
return 'null'
if (entry['Name']['value'] == 'Vehicle Make'):
if not (entry['Value']['value'] == 'Ford'):
return 'null'
model = convertModelToNumber(title)
quantity = int(response['Item']['Quantity']['value'])
# take only items which have an assigned model number, the quantity field is 1, ...
if (len(description) > 0) & (len(pictures) > 0) & (condition > 0) & (price != 0) & (model >= 0) & (duration > 0) & (seller_rating_count >= 0) & (quantity == 1):
pictures_list = list()
if len(pictures) == 1:
for y in range(0,3):
pictures_list.append(str(pictures['value']))
else:
for y in range(0,len(pictures)):
pictures_list.append(str(pictures[y]['value']))
if len(pictures) == 2:
pictures_list.append(str(pictures[1]['value']))
# save the features to a list
item_list = list()
item_list.append(title) #0
item_list.append(description) #1
item_list.append(category) #2
item_list.append(int(nbr+1)) #3
item_list.append(int(itemId)) #4
item_list.append(condition) #5
item_list.append(price) #6
item_list.append(model) #7
item_list.append(model) #8
item_list.append(seller_rating) #9
item_list.append(seller_rating_count) #10
item_list.append(duration) #11
item_list.append(nbr_of_pictures) #12
item_list.append(len_of_description) #13
item_list.append(end_weekday) #14
item_list.append(start_weekday) #15
item_list.append(end_hour) #16
item_list.append(handling_time) #17
item_list.append(global_shipping) #18
item_list.append(shipping_locations) #19
item_list.append(shipping_type) #20
item_list.append(returns_accepted) #21
item_list.append(pictures_list) #22
return item_list
return 'null'
def get_product_from_ebay(item_id):
shopping = Shopping(siteid='EBAY-US', appid=keys.EBAY_APP_ID)
return shopping.execute('GetSingleItem', {'ItemID': item_id}).dict()['Item']
def get_new_posting_attrs(posting_ids):
n_items = len(posting_ids)
bike_attrs = defaultdict(list)
for var in ['itemId','title','price','URL','imageURL','imagefile','description',\
'endtime','location','biketype']:
bike_attrs[var] = []
api = Shopping(config_file='ebay.yaml')
if not (os.path.exists('data/' + 'ebay_postings.csv')):
fid = open('data/' + 'ebay_postings.csv', 'w')
for key in bike_attrs.keys():
fid.write(key + ',')
fid.write('\n')
fid.close()
# Get unique counter for saving images in dir 'data'
n = 0
files = os.listdir('data')
for name in files:
if name.endswith('.jpg') and name.startswith('ebay'):
if int(name[-9:-4]) >= n:
n = int(name[-9:-4]) + 1
print("Starting at image", n)
fid = open('data/' + 'ebay_postings.csv', 'a')
j = 0
for posting_id in posting_ids:
print("Posting", j, "of", n_items)
try:
response = api.execute(
'GetSingleItem', {
'ItemID': posting_id,
'version': '981',
'IncludeSelector': ['PictureDetails', 'ItemSpecifics'],
'outputSelector': 'PictureURLLarge'
})
item = response.reply.Item
try:
bike_attrs['imageURL'].append(item.PictureURL[0].replace(
',', ' '))
imagefile = 'ebay_image' + '{0:05d}'.format(n) + '.jpg'
urllib.request.urlretrieve(item.PictureURL[0],
'data/' + imagefile)
bike_attrs['imagefile'].append(imagefile)
try:
bike_attrs['location'].append(
item.Location.replace(',', ':'))
except:
locations.append([])
bike_attrs['URL'].append(item.ViewItemURLForNaturalSearch)
bike_attrs['itemId'].append(item.ItemID)
bike_attrs['title'].append(item.Title.replace(',', ' '))
bike_attrs['price'].append(item.ConvertedCurrentPrice.value)
bike_attrs['endtime'].append(str(item.EndTime.date()))
bike_attrs['biketype'].append(
get_biketype(item).replace(',', ' '))
response = api.execute(
'GetSingleItem', {
'ItemID': posting_id,
'version': '981',
'IncludeSelector': ['TextDescription']
})
item = response.reply.Item
bike_attrs['description'].append(
item.Description.replace('\n', ' ').replace(',', ' '))
j += 1
n += 1
for key in bike_attrs.keys():
fid.write(bike_attrs[key][-1] + ', ')
fid.write('\n')
except:
pass
except:
print("Item no longer exists")
fid.close()
return bike_attrs
def getCategories(self):
catList = Shopping().execute("GetCategoryInfo", {"CategoryID": "-1", "IncludeSelector": "ChildCategories"}).dict()["CategoryArray"]["Category"]
for cat in catList: #meow
if cat["CategoryID"] != "-1": #perhaps not, would give randomly selecting from all categories
catList.remove(cat)
return catList
def query_product(request, itemID):
api = Shopping(appid='*****')
response = api.execute('GetSingleItem', {'ItemID': itemID, 'IncludeSelector':'ShippingCosts, ItemSpecifics, Variations'})
item = response.dict()
#price calculations
p = item['Item']['ConvertedCurrentPrice']['value']
price = float(p)
price = price + price*0.15
price = round(price, 2)
if price < 10:
price +=10
elif 10 < price < 20:
price +=5
#item specifics
title = item['Item']['Title']
title = title.rsplit(' ', 2)[0] #cutting Blemish blabla
price = str(price)
pics = item['Item']['PictureURL']
condition = item['Item']['ConditionDisplayName']
if condition == 'New with defects':
condition_desc = item['Item']['ConditionDescription']
else:
condition_desc = '---'
shipping = item['Item']['ShippingCostSummary']['ShippingServiceCost']['value']
#retrieving item aspects by value
dicts = item['Item']['ItemSpecifics']['NameValueList'] #if item has not variations
var = item['Item'] # ready to retrieve variations
size = []
if var.has_key('Variations'):
var = var['Variations']['Variation']
for v in var:
size.append(v['VariationSpecifics']['NameValueList']['Value'])
size = ", ".join((str(s) for s in size))
else:
try:
size = (it for it in dicts if it["Name"] == "US Shoe Size (Women's)").next()['Value']
except:
size = "no size"
try:
width = (it for it in dicts if it["Name"] == "Width").next()['Value']
except:
width = u"Не указано"
material = (it for it in dicts if it["Name"] == "Material").next()['Value']
color = (it for it in dicts if it["Name"] == "Color").next()['Value']
kabluk = (it for it in dicts if it["Name"] == "Heel Height").next()['Value']
return {'title':title,
'price':price,
'pics':pics,
'condition': condition,
'condition_desc':condition_desc,
'shipping':shipping,
'size':size,
'width':width,
'material':material,
'color':color,
'kabluk':kabluk}
from ebaysdk.shopping import Connection as Shopping
try:
api = Shopping(appid="JennyHun-7ae2-4c50-abd7-3c1af9cedea5")
response = api.execute('FindPopularItems', {'QueryKeywords': 'willow tree creche'})
print(response.dict())
print(response.reply)
except:
print "error"
#ConnectionError as e:
# print(e)
# print(e.response.dict())
from ebaysdk.shopping import Connection
if __name__ == '__main__':
api = Connection(config_file='ebay.yaml', siteid="EBAY-US")
request = {
'ItemID': '324258671850',
'outputSelector': 'SellerInfo',
}
response = api.execute('GetSingleItem', request)
with open('data_byitemID.xml', 'w') as f:
f.write(response.text)
def main():
"""
Main function
:return:
"""
# Read user input and initialize writer
store_name, country_code, pages_to_crawl = get_user_input()
raw_writer = get_csv_writer("%s_raw.csv" % store_name, raw_field_names)
raw_writer.writeheader()
shopee_dataframe = pd.DataFrame([])
# Initialize API connection
api_finding = Finding(appid=ebay_app_id, config_file=None, siteid=country_code)
api_shopping = Shopping(appid=ebay_app_id, config_file=None, siteid=country_code)
print "API Connection established"
# Find Item IDs in the store
sku_id_list = get_sku_id_list(store_name, pages_to_crawl, api_finding)
print "SKU ID information is obtained from API"
if sku_id_list:
times_to_query = int(len(sku_id_list) / 20) # One time can only query 20 skus
print "Start to crawl info for each sku..."
for i in range(times_to_query):
# Get sku details
response_sku = api_shopping.execute('GetMultipleItems', {'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'Details'}).dict()["Item"]
# Get sku variations
response_variations = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'Variations'}).dict()["Item"]
# Get sku descriptions
response_descriptions = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[i * 20:(i + 1) * 20],
'IncludeSelector': 'TextDescription'}).dict()["Item"]
# Parse the api response and store them into 2 files
parse_sku_details_raw(details=response_sku, variations=response_variations,
descriptions=response_descriptions,
raw_writer=raw_writer)
shopee_dataframe = pd.concat([shopee_dataframe,
parse_sku_details_shopee(details=response_sku, variations=response_variations,
descriptions=response_descriptions)])
print "%s SKUs are crawled" % str((i + 1) * 20)
# In case there are skus that have not been crawled
# If there are 21 skus, the last sku will be crawled by the following code
if times_to_query * 20 < len(sku_id_list):
response_sku = api_shopping.execute('GetMultipleItems', {'ItemID': sku_id_list[times_to_query * 20:],
'IncludeSelector': 'Details'}).dict()["Item"]
response_variations = api_shopping.execute('GetMultipleItems',
{'ItemID': sku_id_list[times_to_query * 20:],
'IncludeSelector': 'Variations'}).dict()["Item"]
parse_sku_details_raw(details=response_sku, variations=response_variations,
descriptions=response_descriptions,
raw_writer=raw_writer)
shopee_dataframe = pd.concat([shopee_dataframe,
parse_sku_details_shopee(details=response_sku, variations=response_variations,
descriptions=response_descriptions)])
print "All SKU info is crawled"
# Write to file
excel_writer = pd.ExcelWriter("%s_shopee_format.xlsx" % store_name)
shopee_dataframe = shopee_dataframe[shopee_field_names]
shopee_dataframe.to_excel(excel_writer, 'Sheet1', index=False)
excel_writer.save()
print "Data is written to files"
else:
# In case this store has no skus
print "Store %s does not have any available items" % store_name
import cgi, cgitb
cgitb.enable()
from ebaysdk.shopping import Connection as Shopping
print """
<!DOCTYPE HTML>
<html>
<head>
<title>
Results
</title>
</head>
<body>
<center>
"""
formdata = cgi.FieldStorage()
form = cgi.FieldStorage()
keyword = form.getvalue("keyword")
try:
api = Shopping(appid="Defhacks-933d-417e-b64f-ce70a53266e2")
response = api.execute('FindPopularItems', {'QueryKeywords': keyword})
print(response.dict())
print(response.reply)
except ConnectionError as e:
print(e)
print(e.response.dict())
print """</center></body></html>"""
