def get_copy(self, a): copy = Article() copy.title = a.title copy.date = a.date copy.price = a.price copy.user_id = a.user_id copy.nickname = a.nickname copy.keywords = a.keywords copy.source = a.source copy.content = a.content copy.posted = False return copy
def regex_test(path):
global parser
parser = DealParser(path)
import sha
for filename in os.listdir(decoded_path):
doc = open(decoded_path + '\\' + filename, 'r')
doc = doc.read().split('||')
a = Article()
try:
a.filename = filename
a.category = doc[0]
a.title = doc[1]
a.date = doc[2]
except:
sys.stdout.write('content error: ' + filename + '\n')
continue
try:
a.hashKey = sha.new(a.title).hexdigest()
except:
pass
if a.hashKey in db_hash:
continue
else:
db_hash[a.hashKey] = a
a.price = parser.get_price(a.title)
if not a.price:
#sys.stdout.write("can't find price: " + filename + ', title: ' + a.title + '\n')
continue
a.keywords = parser.extract_keywords(a.title)
insert_article_db(a)
print "now printing"
print_db()
def regex_test(path):
decoded_path = os.path.join(path, "decoded1000")
db_hash = {}
parser = DealParser(path)
engine = Engine(parser.get_adjusted_price)
for filename in os.listdir(decoded_path):
doc = open(decoded_path + "\\" + filename, "r")
doc = doc.read().split("||")
a = Article()
try:
a.filename = filename
a.category = doc[0]
a.title = doc[1]
a.date = doc[2]
except:
sys.stdout.write("content error: " + filename + "\n")
continue
try:
a.hashKey = sha.new(a.title).hexdigest()
except:
pass
if a.hashKey in db_hash:
continue
else:
db_hash[a.hashKey] = a
a.price = parser.get_price(a.title)
if not a.price:
# sys.stdout.write("can't find price: " + filename + ', title: ' + a.title + '\n')
continue
a.keywords = parser.extract_keywords(a.title)
engine.insert_article_db(a)
print "now printing"
engine.print_db()
def test_insert_article_db(self):
self.assertEquals(20, len(self.test.db_price))
a = Article()
a.title = 'test title'
a.date = datetime.utcnow()
a.price = '$1'
a.keywords = ['test', 'title']
a.source = 'test category 1'
self.test.insert_article_db(a)
self.test.print_db(self)
expected = '*** $1.0: 0|test category 1|test title 1|$1|test,keywords1 1|test category 1|test title|$1|test,title *** $2.0: 0|test category 2|test title 2|$2|test,keywords2 *** $3.0: 0|test category 3|test title 3|$3|test,keywords3 *** $4.0: 0|test category 4|test title 4|$4|test,keywords4 *** $5.0: 0|test category 5|test title 5|$5|test,keywords5 *** $6.0: 0|test category 6|test title 6|$6|test,keywords6 *** $7.0: 0|test category 7|test title 7|$7|test,keywords7 *** $8.0: 0|test category 8|test title 8|$8|test,keywords8 *** $9.0: 0|test category 9|test title 9|$9|test,keywords9 *** $10.0: 0|test category 10|test title 10|$10|test,keywords10 *** $11.0: 0|test category 11|test title 11|$11|test,keywords11 *** $12.0: 0|test category 12|test title 12|$12|test,keywords12 *** $13.0: 0|test category 13|test title 13|$13|test,keywords13 *** $14.0: 0|test category 14|test title 14|$14|test,keywords14 *** $15.0: 0|test category 15|test title 15|$15|test,keywords15 *** $16.0: 0|test category 16|test title 16|$16|test,keywords16 *** $17.0: 0|test category 17|test title 17|$17|test,keywords17 *** $18.0: 0|test category 18|test title 18|$18|test,keywords18 *** $19.0: 0|test category 19|test title 19|$19|test,keywords19 *** $20.0: 0|test category 20|test title 20|$20|test,keywords20 **************************'
self.assertEquals(expected, self.output.replace('\n', ' ').replace('\r', ''), '')#'price: ' + case[2] + ' not found in ' + case[0])
def post_article_to_pt(post, category=None, board_id=None): from pt_poster import Article, Article_Poster article = Article() poster = Article_Poster(board_id=board_id if board_id else post.board.id) article.title = post.subject article.category = category if category else "Amazon" article.user_id = 'priceandtalk' article.nickname = 'PriceAndTalk' # article.user_id = post.user.username # article.nickname = post.user.get_profile().nickname if post.user.get_profile() and post.user.get_profile().nickname else post.user.username article.content = post.text poster.add_article(article) poster.write()
def test_get_not_posted_deals_w_children(self): self.assertEquals(20, len(self.test.db_price)) a = Article() a.title = 'test title' a.date = datetime.utcnow() a.price = '$1' a.keywords = ['test', 'keywords1'] a.source = 'test category 2' inserted = self.test.insert_article_db(a) self.assertFalse(inserted.posted) list = self.test.get_not_posted_deals_w_children() self.assertEquals(1, len(list)) for deal in list: deal.posted = True list = self.test.get_not_posted_deals_w_children() self.assertEquals(0, len(list))
def setUp(self): self.output = '' pathname = os.path.dirname(sys.argv[1]) orig_work_path = os.path.abspath(pathname) self.parser = DealParser(orig_work_path) self.test_work_path = os.path.join(orig_work_path, 'test') self.test = Engine(self.test_work_path, self.parser.get_adjusted_price) for i in range(1,21): u = datetime.utcnow() d = u if i <= 10 else u-timedelta(days=4) a = Article() a.title = 'test title ' + str(i) a.date = d a.price = '$' + str(i) a.keywords = ['test','keywords' + str(i)] a.source = 'test category ' + str(i) self.test.insert_article_db(a)
