def main():
import sys
ecs.setLicenseKey(api_key())
ecs.setSecretKey(secret_key())
if "--test" in sys.argv:
test_parse_url()
url = sys.stdin.readline().strip()
(domain, asin) = parse_url(url)
locale = get_locale(domain)
page = get_page(domain, asin, locale)
isbn = extract(page, ['ISBN'])
title = extract(page, ["Title"])
if not title and isbn:
print "status\tredirect\thttp://www.worldcat.org/isbn/%s" % isbn
sys.exit(0)
print "begin_tsv"
for (k, v) in fetch(page, asin, locale):
print unicode("%s\t%s" % (k, v)).encode("utf8")
print "end_tsv"
def main():
import sys
ecs.setLicenseKey( api_key() )
ecs.setSecretKey( secret_key() )
if "--test" in sys.argv:
test_parse_url()
url = sys.stdin.readline().strip()
(domain,asin) = parse_url(url)
locale = get_locale(domain)
page = get_page(domain, asin, locale)
isbn = extract(page, ['ISBN'])
title = extract(page,["Title"])
if not title and isbn:
print "status\tredirect\thttp://www.worldcat.org/isbn/%s" % isbn
sys.exit(0)
print "begin_tsv"
for (k,v) in fetch(page, asin, locale):
print unicode("%s\t%s" % (k,v)).encode("utf8")
print "end_tsv"
def testParameterOutOfRange(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2")
self.assertRaises(ecs.ParameterOutOfRange,
ecs.ItemSearch,
"python",
SearchIndex="Books",
ItemPage="9999")
def process(self, file):
if not file.has_key(u'asin'):
return file
if file[u'asin'] in self.covercache:
return self.fill_file(file, self.covercache[file[u'asin']])
ecs.setLicenseKey(config['S3.accesskey'])
try:
aitem = ecs.ItemLookup(
file[u'asin'],
IdType='ASIN',
ResponseGroup='Images'
)
except ecs.AWSException, e:
sleep(1)
return file #just keep going, we don't need albumart
def getCoverArt(self,var):
# np = os.popen("curl http://localhost:8080/[email protected]/Audio/Boxee/Now_Playing").read()
#np = pipe = os.popen("curl http://192.168.1.131","r")
# np = re.sub(r'<[^>]*?>', '', np)
#np = np.replace("Back","").replace("Result","")
global np
print "\n*********", np
(artist, song, album) = np.split(' - ')
#print artist, song, album
#np = artist
if not self.lastCover == artist+song+album:
self.lastCover = artist+song+album
ecs.setLicenseKey('AKIAIICBON46BQIRCOUQ')
ecs.setSecretKey('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')
print "** Searching Amazon for "+artist+" - "+album
try:
search = ecs.ItemSearch(Keywords='Music', SearchIndex='Music',Artist=artist, Title=album, ResponseGroup='Images')
print "\n*** serach result: " , search
except:
print 'No Cover found, trying Artist only'
try:
search = ecs.ItemSearch(Keywords='Music', SearchIndex='Music',Artist=artist, ResponseGroup='Images')
except:
print "\n *** No cover found \n"
self.lastImg = '/Filesystem/interfaces/images/nocoverart.png'
return self.lastImg
img=search.next().LargeImage.URL
print img
self.lastImg = img
#print "LastFM getCoverArt!!!!!!!!!!!"
return img
else:
#print 'LOADING CACHED IMAGE FOR COVERART!!!'+self.lastImg
return self.lastImg
def getCoverArt(self, var):
np = os.popen("echo info | nc " + host + " " + port)
np = np.read().replace('"', "")
if np == "":
return "/Filesystem/interfaces/images/nocoverart.png"
elif "- -" in np:
return "/Filesystem/interfaces/images/nocoverart.png"
try:
(artist, song, album) = np.split(" - ")
except:
return "/Filesystem/interfaces/images/nocoverart.png"
if not self.lastCover == artist + song + album:
self.lastCover = artist + song + album
import ecs
ecs.setLicenseKey("AKIAIICBON46BQIRCOUQ")
ecs.setSecretKey("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")
try:
# search = ecs.ItemSearch(Keywords='Music', SearchIndex='Music',Artist=artist, Title=album, ResponseGroup='Images')
search = ecs.ItemSearch(Keywords="Music", SearchIndex="Music", Artist=artist, ResponseGroup="Images")
except:
print "\n *** No Cover found, trying Artist only"
try:
search = ecs.ItemSearch(
Keywords="Music", SearchIndex="Music", Artist=artist, ResponseGroup="Images"
)
except:
print "\n *** No cover found \n"
self.lastImg = "/Filesystem/interfaces/images/nocoverart.png"
return self.lastImg
img = search.next().LargeImage.URL
self.lastImg = img
return img
else:
return self.lastImg
localeArgs = sys.argv[1]
fieldArgs = sys.argv[2]
query = sys.argv[3]
fieldMap = {
"ASIN": "ISBN",
"Author": "authors",
"Manufacturer": "publisher",
"Title": "title",
"DetailPageURL": "url",
}
locales = localeArgs.split(",")
fields = fieldArgs.split(",")
ecs.setLicenseKey('1M21AJ49MF6Y0DJ4D1G2')
doc = Document()
root = doc.createElement("importedData")
doc.appendChild(root)
for searchLocale in locales:
searchMode = "Books"
if (searchLocale != "us"):
searchMode = "Books-" + searchLocale
for searchField in fields:
if (searchLocale != "" and searchField != ""):
collection = doc.createElement("List")
def testParameterOutOfRange(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2");
self.assertRaises( ecs.ParameterOutOfRange, ecs.ItemSearch, "python", SearchIndex="Books", ItemPage="9999")
def testBadLicenseKey(self): ecs.setLicenseKey( "1MGVS72Y8JF7EC7JDZG0" ) self.assertRaises( ecs.InvalidParameterValue, ecs.ItemLookup, "0596002818" )
sys.stderr.write(e % __file__)
sys.exit(1)
from product.models import *
from ecs import setLicenseKey, setSecretKey
#from site import addsitedir
#from django.core.management import execute_manager
#from string import *
from awsProductCrawler import LastSavedId, get_products_and_save
if __name__ == "__main__" :
setLicenseKey("AKIAJPVEF4ZXJTBXRJOA");
setSecretKey('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')
# global last_browse_id
last_browse_id = LastSavedId('last_browse_id.txt')
if last_browse_id.last_id:
last_saved_id = last_browse_id.last_id
else:
last_saved_id = '0'
# global last_asin_id
# last_asin_id = LastSavedId('last_id.txt')
# global cart
# cart = None
#
# global admin
try:
def setUp(self):
# prepare the python books to add
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2");
self.books = ecs.ItemSearch("python", SearchIndex="Books")
self.cart = None
from pprint import pprint
import ecs
def get_infos(books):
book_info = dict()
for book in books:
book_info[book.ASIN] = dict()
for mtd in dir(book):
if mtd[0] != '_':
book_info[book.ASIN][mtd] = eval('book.%s' % mtd)
return book_info
if __name__ == "__main__":
ecs.setLicenseKey('0J356Z09CN88KB743582')
#books = ecs.ItemSearch('python', SearchIndex='Books')
books = ecs.ItemSearch('0465027814', SearchIndex='Books', ResponseGroup='TagsSummary,Tags,ItemAttributes,EditorialReview,Images,Similarities,PromotionalTag')
pprint(get_infos(books))
def lookup_by_upc(self,upc):
if len(upc)>0:
#first we check our database
titles = Title.select(Title.q.isbn==upc)
#print titles #debug
self.known_title= False
the_titles=list(titles)
if len(the_titles) > 0:
self.known_title= the_titles[0]
ProductName = the_titles[0].booktitle
authors = [x.authorName for x in the_titles[0].authors]
authors_as_string = string.join(authors,',')
categories = [x.categoryName for x in the_titles[0].categorys]
categories_as_string = string.join(categories,',')
if len(the_titles[0].books) > 0:
ListPrice = the_titles[0].books[0].listprice
else:
ListPrice = 0
Manufacturer = the_titles[0].publisher
else: #we don't have it yet
sleep(1) # so amazon doesn't get huffy
ecs.setLicenseKey(amazon_license_key)
ecs.setSecretAccessKey(amazon_secret_key)
ecs.setAssociatTag(amazon_associate_tag)
pythonItems = ecs.searchByUPC(upc)
if pythonItems:
result={}
authors=[]
author_object="none"
categories=[]
b=pythonItems[0]
try:
author_object=b.Artists.Artist
except AttributeError:
author_object="none"
if type(author_object) in types.StringTypes:
authors.append(author_object)
else:
authors=author_object
authors_as_string = string.join(authors,',')
for category in b.BrowseNode:
categories.append(category.BrowseName)
categories_as_string = string.join(categories,',')
ProductName=""
try:
if b.ProductName:
ProductName=b.ProductName
except AttributeError:
x=1
Manufacturer=""
try:
if b.Manufacturer:
Manufacturer=b.Manufacturer
except AttributeError:
x=1
ListPrice=""
try:
if b.ListPrice:
ListPrice=b.ListPrice
except AttributeError:
x=1
ReleaseDate=""
try:
if b.ReleaseDate:
ReleaseDate=b.ReleaseDate
except AttributeError:
x=1
return {"title":ProductName,
"authors":authors,
"authors_as_string":authors_as_string,
"categories_as_string":categories_as_string,
"list_price":ListPrice,
"publisher":Manufacturer,
"isbn":upc,
"known_title": self.known_title}
localeArgs = sys.argv[1]
fieldArgs = sys.argv[2]
query = sys.argv[3]
fieldMap = {
"ASIN" : "ISBN",
"Author" : "authors",
"Manufacturer" : "publisher",
"Title" : "title",
"DetailPageURL" : "url",
}
locales = localeArgs.split(",")
fields = fieldArgs.split (",");
ecs.setLicenseKey('1M21AJ49MF6Y0DJ4D1G2')
doc = Document ()
root = doc.createElement ("importedData")
doc.appendChild (root)
for searchLocale in locales:
searchMode = "Books"
if (searchLocale != "us"):
searchMode = "Books-" + searchLocale
for searchField in fields:
if (searchLocale != "" and searchField != ""):
collection = doc.createElement ("List")
def testBadLicenseKey(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG0")
self.assertRaises(ecs.InvalidParameterValue, ecs.ItemLookup,
"0596002818")
def setUp(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2");
self.ItemId = "0596009259"
def setUp(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2");
from pprint import pprint
import ecs
def get_infos(books):
book_info = dict()
for book in books:
book_info[book.ASIN] = dict()
for mtd in dir(book):
if mtd[0] != '_':
book_info[book.ASIN][mtd] = eval('book.%s' % mtd)
return book_info
if __name__ == "__main__":
ecs.setLicenseKey('0J356Z09CN88KB743582')
#books = ecs.ItemSearch('python', SearchIndex='Books')
books = ecs.ItemSearch(
'0465027814',
SearchIndex='Books',
ResponseGroup=
'TagsSummary,Tags,ItemAttributes,EditorialReview,Images,Similarities,PromotionalTag'
)
pprint(get_infos(books))
def setUp(self):
ecs.setLicenseKey("1MGVS72Y8JF7EC7JDZG2")
self.ItemId = "0596009259"
def lookup_by_isbn(self,number):
isbn=""
number=re.sub("^([\'\"])(.*)\\1$", '\\2', number)
#print "number is now: ", number
if len(number)>=9:
number=re.sub("[-\s]", '', number)
#print "number is now: ", number
if len(number)==10 and isbnlib.isValid(number):
isbn=isbnlib.convert(number)
else:
isbn=number
#print "NUMBER was " +number+ ",ISBN was "+isbn
if (len(isbn)>0 and not re.match('^n(\s|/){0,1}a|none', isbn, re.I)):
#first we check our database
titles = Title.select(Title.q.isbn==isbn)
#print titles #debug
self.known_title= False
the_titles=list(titles)
if len(the_titles) > 0:
#print "in titles"
self.known_title= the_titles[0]
ProductName = the_titles[0].booktitle.format()
authors=[]
if len(the_titles[0].author) > 0:
authors = [x.authorName.format() for x in the_titles[0].author]
authors_as_string = string.join(authors,',')
categories=[]
if len(the_titles[0].categorys) > 0:
#print len(the_titles[0].categorys)
#print the_titles[0].categorys
categories = [x.categoryName.format() for x in the_titles[0].categorys]
categories_as_string = string.join(categories,',')
if len(the_titles[0].books) > 0:
ListPrice = the_titles[0].books[0].listprice
else:
ListPrice = 0
Manufacturer = the_titles[0].publisher.format()
Format=the_titles[0].type.format()
Kind=the_titles[0].kind.kindName
return {"title":ProductName,
"authors":authors,
"authors_as_string":authors_as_string,
"categories_as_string":categories_as_string,
"list_price":ListPrice,
"publisher":Manufacturer,
"isbn":isbn,
"format":Format,
"kind":Kind,
"known_title": self.known_title}
else: #we don't have it yet
#print "in isbn"
sleep(1) # so amazon doesn't get huffy
ecs.setLicenseKey(amazon_license_key)
ecs.setSecretAccessKey(amazon_secret_key)
ecs.setAssociateTag(amazon_associate_tag)
#print "about to search", isbn, isbn[0]
pythonBooks=[]
try:
pythonBooks = ecs.ItemLookup(isbn,IdType="ISBN",SearchIndex="Books",ResponseGroup="ItemAttributes,BrowseNodes")
except ecs.InvalidParameterValue:
pass
#print pythonBooks
if pythonBooks:
result={}
authors=[]
categories=[]
b=pythonBooks[0]
for x in ['Author','Creator']:
if hasattr(b,x):
if type(getattr(b,x))==type([]):
authors.extend(getattr(b,x))
else:
authors.append(getattr(b,x))
authors_as_string = string.join(authors,',')
categories_as_string =""
# a bit more complicated of a tree walk than it needs be.
# set up to still have the option of category strings like "history -- us"
# switched to sets to quickly remove redundancies.
def parseBrowseNodes(bNodes):
def parseBrowseNodesInner(item):
bn=set()
if hasattr(item, 'Name'):
bn.add(item.Name)
if hasattr(item, 'Ancestors'):
#print "hasansc"
for i in item.Ancestors:
bn.update(parseBrowseNodesInner(i))
if hasattr(item, 'Children'):
for i in item.Children:
bn.update(parseBrowseNodesInner(i))
#print "bn ", bn
if not (hasattr(item, 'Ancestors') or hasattr(item, 'Children')):
if hasattr(item, 'Name'):
return set([item.Name])
else:
return set()
return bn
nodeslist=[parseBrowseNodesInner(i) for i in bNodes ]
nodes=set()
for n in nodeslist:
nodes = nodes.union(n)
return nodes
categories=parseBrowseNodes(b.BrowseNodes)
categories_as_string = string.join(categories,',')
categories=parseBrowseNodes(b.BrowseNodes)
categories_as_string = string.join(categories,',')
#print categories, categories_as_string
ProductName=""
if hasattr(b,'Title'):
ProductName=b.Title
Manufacturer=""
if hasattr(b,'Manufacturer'):
Manufacturer=b.Manufacturer
ListPrice=""
if hasattr(b,'ListPrice'):
ListPrice=b.ListPrice.FormattedPrice.replace("$",'')
Format=''
if hasattr(b, "Binding"):
Format=b.Binding
Kind='books'
return {"title":ProductName,
"authors":authors,
"authors_as_string":authors_as_string,
"categories_as_string":categories_as_string,
"list_price":ListPrice,
"publisher":Manufacturer,
"isbn":isbn,
"format":Format,
"kind":Kind,
"known_title": self.known_title}
else:
return []
else:
return []
def lookup_by_isbn(number):
isbn = ""
number = re.sub("^(['\"])(.*)\\1$", "\\2", number)
# print "number is now: ", number
if len(number) >= 9:
number = re.sub("[-\s]", "", number)
# print "number is now: ", number
if len(number) == 13 or len(number) == 18:
isbn = upc2isbn(number)
else:
isbn = number
# print "NUMBER was " +number+ ",ISBN was "+isbn
if len(isbn) > 0:
# first we check our database
titles = Title.select(Title.q.isbn == isbn)
# print titles #debug
known_title = False
the_titles = list(titles)
if len(the_titles) > 0:
# print "in titles"
known_title = the_titles[0]
ProductName = the_titles[0].booktitle.format()
authors = [x.authorName.format() for x in the_titles[0].author] or []
authors_as_string = string.join(authors, ",")
if len(the_titles[0].categorys) > 0:
# print len(the_titles[0].categorys)
# print the_titles[0].categorys
categories = [x.categoryName.format() for x in the_titles[0].categorys]
categories_as_string = string.join(categories, ",")
if len(the_titles[0].books) > 0:
ListPrice = the_titles[0].books[0].listprice
else:
ListPrice = 0
Manufacturer = the_titles[0].publisher.format()
Format = the_titles[0].type.format()
Kind = the_titles[0].kind.kindName
else: # we don't have it yet
# print "in isbn"
sleep(1) # so amazon doesn't get huffy
ecs.setLicenseKey(amazon_license_key)
ecs.setSecretAccessKey(amazon_secret_key)
ecs.setAssociateTag(amazon_associate_tag)
# print "about to search", isbn, isbn[0]
pythonBooks = ecs.ItemLookup(
isbn, IdType="ISBN", SearchIndex="Books", ResponseGroup="ItemAttributes,BrowseNodes"
)
# print pythonBooks
if pythonBooks:
result = {}
authors = []
categories = []
b = pythonBooks[0]
for x in ["Author", "Creator"]:
if hasattr(b, x):
if type(getattr(b, x)) == type([]):
authors.extend(getattr(b, x))
else:
authors.append(getattr(b, x))
authors_as_string = string.join(authors, ",")
categories_as_string = ""
# a bit more complicated of a tree walk than it needs be.
# set up to still have the option of category strings like "history -- us"
# switched to sets to quickly remove redundancies.
def parseBrowseNodes(bNodes):
def parseBrowseNodesInner(item):
bn = set()
if hasattr(item, "Name"):
bn.add(item.Name)
if hasattr(item, "Ancestors"):
# print "hasansc"
for i in item.Ancestors:
bn.update(parseBrowseNodesInner(i))
if hasattr(item, "Children"):
for i in item.Children:
bn.update(parseBrowseNodesInner(i))
# print "bn ", bn
if not (hasattr(item, "Ancestors") or hasattr(item, "Children")):
if hasattr(item, "Name"):
return set([item.Name])
else:
return set()
return bn
nodeslist = [parseBrowseNodesInner(i) for i in bNodes]
nodes = set()
for n in nodeslist:
nodes = nodes.union(n)
return nodes
categories = parseBrowseNodes(b.BrowseNodes)
categories_as_string = string.join(categories, ",")
ProductName = ""
if hasattr(b, "Title"):
ProductName = b.Title
Manufacturer = ""
if hasattr(b, "Manufacturer"):
Manufacturer = b.Manufacturer
ListPrice = ""
if hasattr(b, "ListPrice"):
ListPrice = b.ListPrice.FormattedPrice.replace("$", "")
Format = ""
if hasattr(b, "Binding"):
Format = b.Binding
Kind = "books"
return {
"title": ProductName,
"authors": authors,
"authors_as_string": authors_as_string,
"categories_as_string": categories_as_string,
"list_price": ListPrice,
"publisher": Manufacturer,
"isbn": isbn,
"format": Format,
"kind": Kind,
"known_title": known_title,
}
