Exemplo n.º 1
0
def addquery(keyword, dollarlimit):
	mainbox_table = pymongo.Connection('localhost', 27017)[APP_CONFIG["DBNAME"]]['mainbox']
	username = request.cookies.get('username')
	queries = [query for query in mainbox_table.find({'username': username})]		
	dollar = int(dollarlimit)
	keyword = string.strip(keyword)
	pricehigh = int(dollar + (APP_CONFIG['PRICE_HIGH_PER'] * dollar))
	pricelow = int(dollar - (APP_CONFIG['PRICE_LOW_PER'] * dollar))
	pricemax = pricehigh * APP_CONFIG['PRICE_MAX_PER']
	pricemin = pricelow / APP_CONFIG['PRICE_MIN_PER']
	queryhash = feedslib.gethash(username + keyword + dollarlimit + str(pricehigh) + str(pricelow))
	uniqqueryhash = feedslib.gethash(keyword + dollarlimit + str(pricehigh) + str(pricelow))
	querymatches = [query for query in mainbox_table.find({'_id': queryhash})]
	currdatetime = datetime.utcnow()
	if(len(querymatches) == 0):
		users_table = pymongo.Connection('localhost', 27017)[APP_CONFIG["DBNAME"]]['users']
		userinfo = users_table.find_one({'username': username})
		
		inserthash = {'_id': queryhash, 'qid': uniqqueryhash, 'username': username, 'keyword': keyword, 'dollar_limit': dollar, 'price_high': pricehigh, 'price_low': pricelow, 'lastmodified': currdatetime, 'created': currdatetime, 'dayfilter': APP_CONFIG['DEFAULT_DAYS_FILTER'], 'pricemax': pricemax, 'pricemin': pricemin, 'city': userinfo['city'], 'state': userinfo['state'], 'zip': userinfo['zip'], 'lat': userinfo['lat'], 'long': userinfo['long']}
		
		results1 = feedslib.getFeedDeals("craigslist", feedsconfig.CONFIG, keyword, keyword, pricehigh, pricelow, "",userinfo['zip'],userinfo['city'],userinfo['state'])
		deals1 = results1['deals']
		print "feed1 deals found: " + str(len(deals1))
		results2 = feedslib.getFeedDeals("google", feedsconfig.CONFIG, keyword, keyword, pricehigh, pricelow, 25)
		deals2 = results2['deals']
		print "feed2 deals found: " + str(len(deals2))
		results3 = feedslib.getFeedDeals("milo", feedsconfig.CONFIG, keyword, keyword, int(pricehigh * 100), int(pricelow * 100), 30, userinfo['zip'])
		deals3 = results3['deals']
		print "feed3 deals found: " + str(len(deals3))
		sys.stdout.flush()
		if(len(deals3) > 0):
			print "Getting merchant info from milo..."
			sys.stdout.flush()
			deals3 = milo.updateMiloMerchantInfo(deals3, results3['specialreturn'], userinfo['zip'], 10)
		
		
		deals = []
		deals.extend(deals1)
		deals.extend(deals2)
		deals.extend(deals3)
		print "Total deals found: " + str(len(deals))
		
		tagcloudlist = [results1['tagcloud'], results2['tagcloud'], results3['tagcloud']]
		print "************** All tags ********************"
		print tagcloudlist
		print "************** All tags ********************"
		sys.stdout.flush()
		mtagcloud = feedslib.consolidateTagClouds(tagcloudlist)
		print "************** consolidated tags ********************"
		print mtagcloud
		print "************** consolidated tags ********************"
		sys.stdout.flush()
		tagcloud = feedslib.getSortedTagCloudList(mtagcloud)
		print "************** final tags ********************"
		print tagcloud
		print "************** final tags ********************"
		sys.stdout.flush()
		tagslist = []
		facetcloudlist = [results1['facetcloud'], results2['facetcloud'], results3['facetcloud']]
		
		
		mfacetcloud = feedslib.consolidateTagClouds(facetcloudlist,4, "channel,condition", "facets")
		print "mfacetcloud: " + str(mfacetcloud)
		sys.stdout.flush()
		facetcloud = feedslib.getSortedTagCloudList(mfacetcloud,100)
		print "facetcloud: " + str(facetcloud)
		facetslist = []
		for tag in tagcloud:
			inserthash[tag[0]] = tag[1]
			tagslist.append(tag[0])
		inserthash["tags"] = ','.join(tagslist)
		for facet in facetcloud:
			inserthash[facet[0]] = facet[1]
			facetslist.append(facet[0])
		inserthash["facets"] = ','.join(facetslist)
		sys.stdout.flush()
		if(len(deals) > 0):
			deals_table = pymongo.Connection('localhost', 27017)[APP_CONFIG["DBNAME"]]['deals']
			print "Inserting into deals table..."
			sys.stdout.flush()
			deals_table.insert(deals, continue_on_error=True)
			
			print "Done inserting."
			sys.stdout.flush()
		
		
		print "Inserting query: " + str(inserthash)
		sys.stdout.flush()
		mainbox_table.insert(inserthash)
	else:
		print "Query already exists"
		sys.stdout.flush()
   		
    
	return {'status': 'ok'}
Exemplo n.º 2
0
def updateQuery(query, mainbox_table, db):
	
	keywords = []
	keywords.append(query['keyword'])
	filters = []
	if(query.has_key('filters')):
		filters = query['filters'].lstrip(",").split(",")
	
	for filter in filters:
		keyval = filter.split("|")
		if(len(keyval) == 2):
			if(keyval[0] != "channel"):
				keywords.append(keyval[1])
			
	keyword = " ".join(keywords)
	pricehigh = int(query["price_high"] + (query["price_high"] * 0.10))
	pricelow = int(query["price_low"] - (query["price_low"] * 0.10))
	zip = query['zip']
	city = query['city']
	state = query['state']
	print "Updating deals for (%s): keyword(%s), pricehigh(%s), pricelow(%s) @%s, %s %s" % (query['_id'], keyword, pricehigh, pricelow, city, state, zip)
	
	results1 = feedslib.getFeedDeals("craigslist", feedsconfig.CONFIG, keyword, query['keyword'], pricehigh, pricelow, "",zip,city,state)
	deals1 = results1['deals']
	print "feed1 deals found: " + str(len(deals1))
	results2 = feedslib.getFeedDeals("google", feedsconfig.CONFIG, keyword, query['keyword'], pricehigh, pricelow, 25)
	deals2 = results2['deals']
	print "feed2 deals found: " + str(len(deals2))
	results3 = feedslib.getFeedDeals("milo", feedsconfig.CONFIG, keyword, query['keyword'], int(pricehigh * 100), int(pricelow * 100), 30, zip)
	deals3 = results3['deals']
	if(len(deals3) > 0):
		deals3 = milo.updateMiloMerchantInfo(deals3, results3['specialreturn'], zip, 10)
	print "feed3 deals found: " + str(len(deals3))
	sys.stdout.flush()
	deals = []
	deals.extend(deals1)
	deals.extend(deals2)
	deals.extend(deals3)
	for deal in deals:
		deal['keyword'] = query['keyword']
	print "Total deals found: " + str(len(deals))
	
	tagcloudlist = [results1['tagcloud'], results2['tagcloud'], results3['tagcloud']]
	mtagcloud = feedslib.consolidateTagClouds(tagcloudlist)
	tagcloud = feedslib.getSortedTagCloudList(mtagcloud)
	tagslist = []
	facetcloudlist = [results1['facetcloud'], results2['facetcloud'], results3['facetcloud']]
	
	
	mfacetcloud = feedslib.consolidateTagClouds(facetcloudlist,4, "channel,condition","facets")
	print "mfacetcloud: " + str(mfacetcloud)
	sys.stdout.flush()
	facetcloud = feedslib.getSortedTagCloudList(mfacetcloud,100)
	print "facetcloud: " + str(facetcloud)
	facetslist = []
	for tag in tagcloud:
		if(query.has_key(tag[0])):
			query[tag[0]] = query[tag[0]] + tag[1]
		else:
			query[tag[0]] = tag[1]
		tagslist.append(tag[0])

	querytaglist = query["tags"].split(",")
	for tag in tagslist:
		if(not tag in querytaglist):
			querytaglist.append(tag)
			
	query["tags"] = ','.join(querytaglist)


	for facet in facetcloud:
		
		if(query.has_key(facet[0])):
			query[facet[0]] = query[facet[0]] + facet[1]
		else:
			query[facet[0]] = facet[1]
		facetslist.append(facet[0])
	
	queryfacetlist = query["facets"].split(",")
	for facet in facetslist:
		if(not facet in queryfacetlist):
			queryfacetlist.append(facet)
			
	query["facets"] = ','.join(queryfacetlist)
	query["lastmodified"] = datetime.datetime.utcnow()
	mainbox_table.update({'_id': query['_id']}, query)
	sys.stdout.flush()
	if(len(deals) > 0):
		deals_table = pymongo.Connection('localhost', 27017)[db]['deals']
		print "Inserting into deals table..."
		sys.stdout.flush()
		deals_table.insert(deals, continue_on_error=True)
				
		print "Done inserting."
		sys.stdout.flush()
Exemplo n.º 3
0
print "----------------------------------------------"


jsonconfig = feedsconfig.CONFIG
print jsonconfig


results = feedslib.getFeedDeals(feedsource, jsonconfig, keyword, keyword, pricehigh, pricelow, maxresults, zip, city, state)
deals = results['deals']

ct = 1
for deal in deals:
	print "Result " + str(ct) + ":"
	for key in deal.keys():
		pass
		print "\t" + key + " : " + str(deal[key])
	print "----------------------------------------------"
	ct = ct + 1
print "Total result: " + str(len(deals))
print "Tagcloud: " + str(results['tagcloud'])
print "Facetcloud: " + str(results['facetcloud'])
tagcloud1 = results['tagcloud']
tagcloud2 = results['tagcloud']
facetcloud1 = results['facetcloud']
facetcloud2 = results['facetcloud']
mtagcloud = feedslib.consolidateTagClouds([tagcloud1, tagcloud2])
print feedslib.getSortedTagCloudList(mtagcloud)
mfacetcloud = feedslib.consolidateTagClouds([facetcloud1, facetcloud2])
print feedslib.getSortedTagCloudList(mfacetcloud)
print results['specialreturn']