def product_train_test_set(self): session = Session() product_num = session.query(Product).count() product_train_num = int(0.95 * int(product_num)) product_test_num = product_num - product_train_num product_train_list = session.query(Product)\ .order_by(Product.Enrolltime, Product.ProductNo)\ .slice(0, product_train_num)\ .all() for product_train in product_train_list: update_tt = session.query(Product)\ .filter(Product.ProductNo == product_train.ProductNo)\ .one() update_tt.TrainTest = 'train' session.commit() #The latest 5% of products is test set. product_test_list = session.query(Product)\ .order_by(Product.Enrolltime, Product.ProductNo)\ .slice(product_train_num, product_num+1)\ .all() for product_test in product_test_list: update_tt = session.query(Product)\ .filter(Product.ProductNo == product_test.ProductNo)\ .one() update_tt.TrainTest = 'test' session.commit() session.close() print 'updated product'
def get_user_vectors(self): user_vectors = {} session = Session() try: result = session.query(Rating) \ .order_by(Rating.user_id, Rating.movie_id) \ .all() prev_user = None for row in result: cur_user = row.user_id movie_id = row.movie_id rating = row.rating if prev_user == None or prev_user != cur_user: uv = {} uv[movie_id] = rating user_vectors[cur_user] = uv else: user_vectors[cur_user][movie_id] = rating prev_user = cur_user return user_vectors except Exception as e: print e return {} finally: session.close()
def get_userid(self, userid): session = Session() if session.query(User).filter(User.UserId == userid).all(): return True else: return False session.close()
def get_unrated_movies(self, user_id): ''' select movie_id from movies where movie_id not in (select movie_id from ratings where user_id = 1) ''' session = Session() try: subquery = session.query(Rating.movie_id) \ .filter(Rating.user_id == user_id) \ .subquery() result = session.query(Movie.movie_id) \ .filter(Movie.movie_id.notin_(subquery)) \ .all() return [int(row[0]) for row in result] except Exception as e: print e finally: session.close()
def get_user_list(self, tt): session = Session() user_list = [] result = session.query(User).filter(User.TrainTest == tt).all() for row in result: user_list.append(row.UserId) session.close() return user_list
def get_item_list(self, tt): session = Session() item_list = [] result = session.query(Product).filter(Product.TrainTest == tt).all() for row in result: item_list.append(row.ProductNo) session.close() return item_list
def update_user_list(self, userid, count): session = Session() update_user = session.query(User).filter(User.UserId == userid).one() update_user.Written = count update_user.TrainTest = 'train' #ALl of user is train. session.commit() session.close() print 'updated'
def get_recent_news(self): session = Session() recent_news = session.query(NewsArticle).order_by(desc(NewsArticle.ReportDate)).limit(10) for row in recent_news: CacheNews().cache_recent_news(str(row.Title.decode('utf-8'))) session.close()
def get_ratings(self, place_id): try: session = Session() row = session.query(Ratings).filter(Ratings.place_id == place_id).first() if row: print '<RATING>', row, 'exists.' return row except Exception as e: print e finally: session.close()
def save_users(self, user_id, place_id): session = Session() if not self.get_users(user_id, place_id): user = Users(user_id = user_id, place_id = place_id) session.add(user) session.commit() print '<USER>', user_id, place_id, 'saved successfully.' session.close()
def save_types(self, place_id, types): session = Session() if not self.get_types(place_id, types): type = Types(place_id = place_id, types = types) session.add(type) session.commit() print '<TYPE>', place_id, types, 'saved successfully!' session.close()
def delete_news(self, link): try: session = Session() session.query(NewsArticle).filter(NewsArticle.Link == link).delete() #session.commit() return 'Success' except Exception as e: return "Failure" finally: session.close()
def get_news_contents(self, link): try: session = Session() result = session.query(NewsArticle).filter(NewsArticle.Link == link).first() return result.Content except Exception as e: print e return '' finally: session.close()
def save_place(self, place_id, city, name, address, lng, lat): session = Session() if not self.get_place(place_id): geo = GeoInfos(place_id = place_id, name = name, city = city, address = address, lng = lng, lat = lat) session.add(geo) session.commit() print '<GEOINFO>', place_id, name, 'saved successfully!' session.close()
def get_place_name(self, place_id): try: session = Session() row = session.query(GeoInfos).filter(GeoInfos.place_id == place_id).first() # if row: print '<GEOINFO>', row, 'exists.' return row.name except Exception as e: print e finally: session.close()
def save_news(self, news_id, title, content): session = Session() if not self.get_news_by_id(news_id): print news_id news = News(link=news_id, title=title, content=content, crawl_time=datetime.datetime.now()) session.add(news) session.commit() session.close()
def get_comment_by_id(self, comment_id): try: session = Session() row = session.query(comments) \ .filter(comments.ID == comment_id) \ .first() return row except Exception as e: print e finally: session.close()
def criar(item): try: session = Session() session.add(item) session.commit() return item except Exception as e: return internal_error(e), 500 finally: session.close()
def deletar(id): try: session = Session() item = session.query(Item).get(id) session.delete(item) session.commit() except Exception as e: return internal_error(e), 500 finally: session.close()
def get_movie_by_title(self, movie_title): try: session = Session() row = session.query(movie_info) \ .filter(movie_info.title == movie_title) \ .first() return row except Exception as e: print e finally: session.close()
def session(connection): '''We need to ensure that there is a fresh session per test run''' transaction = connection.begin() session = Session(bind=connection) # SourceFactory._meta.sqlalchemy_session = session # TargetFactory._meta.sqlalchemy_session = session yield session session.close() transaction.rollback()
def update_comment_info(self, productno, comment_writer, comment_grade): session = Session() update_comment = session.query(Comment)\ .filter(Comment.ProductNo == productno, Comment.Writer == comment_writer)\ .one() if update_comment.Grade != comment_grade: update_comment.Grade = comment_grade update_comment.Enrolltime = crawltime session.commit() session.close()
def save_ratings(self, place_id, gg_reviews, gg_ratings, ta_reviews, ta_ratings): session = Session() if not self.get_ratings(place_id): rate = Ratings(place_id = place_id, gg_reviews = gg_reviews, gg_ratings = gg_ratings, ta_reviews = ta_reviews, ta_ratings = ta_ratings) session.add(rate) session.commit() print '<RATING>', place_id, 'saved successfully!' session.close()
def get_news_id(self, news_url): session = Session() find_news = session.query(NewsArticle).filter(NewsArticle.Link == news_url).all() if len(find_news) >= 1: return False else: return True session.close()
def save_comment_info(self, productno, comment_writer, comment_grade): if self.get_comment_info(productno, comment_writer): session = Session() insert_comment = Comment(ProductNo=productno, Writer=comment_writer, Grade=comment_grade, Enrolltime=crawltime) session.add(insert_comment) session.commit() session.close() else: self.update_comment_info(productno, comment_writer, comment_grade)
def get_comments_id(self, coId): session = Session() find_comment = session.query(CommentList).filter( CommentList.Id == coId).all() if len(find_comment) >= 1: return False else: return True session.close()
def get_news_content(self, news_id): try: session = Session() result = session.query(News)\ .filter(News.link == news_id).first() return result.content except Exception as e: print '21', e finally: session.close()
def get_news_by_id(self, news_id): try: session = Session() row = session.query(News) \ .filter(News.link == news_id) \ .first() return row except Exception as e: print e finally: session.close()
def save_comment(self, id, news_id, content, written_time, sympathy_count, antipathy_count): session = Session() if not self.get_comment_by_id(id): print content comment = Comment(id = id, news_id = news_id, content = content, written_time = written_time, sympathy_count = sympathy_count, antipathy_count = antipathy_count, crawl_time = datetime.datetime.now()) session.add(comment) session.commit() session.close()
def get_news_by_id(self, link): try: session = Session() result = session.query(News)\ .filter(News.link == link).first() return result except Exception as e: print '11', e finally: session.close()