def save_types(self, place_id, types): session = Session() if not self.get_types(place_id, types): type = Types(place_id = place_id, types = types) session.add(type) session.commit() print '<TYPE>', place_id, types, 'saved successfully!' session.close()
def save_users(self, user_id, place_id): session = Session() if not self.get_users(user_id, place_id): user = Users(user_id = user_id, place_id = place_id) session.add(user) session.commit() print '<USER>', user_id, place_id, 'saved successfully.' session.close()
def save_place(self, place_id, city, name, address, lng, lat): session = Session() if not self.get_place(place_id): geo = GeoInfos(place_id = place_id, name = name, city = city, address = address, lng = lng, lat = lat) session.add(geo) session.commit() print '<GEOINFO>', place_id, name, 'saved successfully!' session.close()
def criar(item): try: session = Session() session.add(item) session.commit() return item except Exception as e: return internal_error(e), 500 finally: session.close()
def save_ratings(self, place_id, gg_reviews, gg_ratings, ta_reviews, ta_ratings): session = Session() if not self.get_ratings(place_id): rate = Ratings(place_id = place_id, gg_reviews = gg_reviews, gg_ratings = gg_ratings, ta_reviews = ta_reviews, ta_ratings = ta_ratings) session.add(rate) session.commit() print '<RATING>', place_id, 'saved successfully!' session.close()
def save_news(self, news_id, title, content): session = Session() if not self.get_news_by_id(news_id): print news_id news = News(link=news_id, title=title, content=content, crawl_time=datetime.datetime.now()) session.add(news) session.commit() session.close()
def save_comment(self, id, news_id, content, written_time, sympathy_count, antipathy_count): session = Session() if not self.get_comment_by_id(id): print content comment = Comment(id = id, news_id = news_id, content = content, written_time = written_time, sympathy_count = sympathy_count, antipathy_count = antipathy_count, crawl_time = datetime.datetime.now()) session.add(comment) session.commit() session.close()
def save_comment_info(self, productno, comment_writer, comment_grade): if self.get_comment_info(productno, comment_writer): session = Session() insert_comment = Comment(ProductNo=productno, Writer=comment_writer, Grade=comment_grade, Enrolltime=crawltime) session.add(insert_comment) session.commit() session.close() else: self.update_comment_info(productno, comment_writer, comment_grade)
def save_movie_info(self, title, genre): session = Session() if not self.get_movie_by_title(title): print title print 'genre: ' + genre print '=' * 70 data = movie_info(title=title, genre=genre) session.add(data) session.commit() return True else: return False session.close()
class MainExecutor(): def __init__(self,id): self.id = id self.Session = Session self.session = Session() self.process = self.session.query(models.Process).options(joinedload('queues').joinedload('queues_tasks').joinedload('task')).filter(models.Process.id == self.id).first() if not self.process: raise Exception(f"Can't find process with id ='{self.id}'") with open('./config.yaml','r') as config: settings = yaml.safe_load(config) self.logger = create_logger('main', os.path.join(settings['logs']['test']['logpath'],self.process.name, dt.datetime.now().strftime("%Y%m%d_%H%M%S"))) self.logger.info(f"Executor initiated for process with id = '{self.id}'") self.logger.debug(self.process) self.threads = [] self.threads_with_errors = [] def prepare(self): self.logger.info('Creating Run instance.') self.run = self.process.create_run() self.session.add(self.run) self.session.commit() self.logger.debug(self.run) self.logger.info('Run saved in db.') def execute(self): error = False try: self.logger.info("Executing.") splitted_queues = split_by_order(self.process.queues) if not splitted_queues: self.logger.error("Splitted queues is empty.") raise Exception("Splitted queues is empty.") for key,val in splitted_queues.items(): blockers = [] for queue_item in val: QT = QueueThread('queue-' + str(queue_item.id),queue_item,self.run,self.Session,self.logger) QT.start() if queue_item.blocking: self.logger.info(f" Queue.id: '{str(queue_item.id)}' added to blocking in same run order.") blockers.append(QT) else: self.logger.info(f" Queue.id: '{str(queue_item.id)}' not blocking anything. Adding it to left threads.") self.threads.append(QT) [ (x.join(),self.logger.info(f"Joined Queue.id: '{str(x.queue.id)}'")) for x in blockers] [x.join() for x in self.threads] print('end') except Exception as e: finally: self.run.end_date = dt.datetime.now() self.session.commit()
def save_product_info(self, productno, link, title, category): if self.get_product_id(productno): session = Session() insert_product = Product(ProductNo=productno, Link=link, Name=title, Category=category, TrainTest='train', Enrolltime=crawltime) session.add(insert_product) session.commit() session.close() else: self.update_product_info(productno, link, title, category)
def save_comments(self, coId, url, mcontent, user, sympathy, antipathy, enrolltime): if self.get_comments_id(coId): session = Session() insert_comment = CommentList(Id=coId, Link=url, Mcontent=mcontent, User=user, Sympathy=sympathy, Antipathy=antipathy, Enrolltime=enrolltime) session.add(insert_comment) session.commit() session.close()
def save_news(self, news_id, title, content, written_clock): saved = False session = Session() if not self.get_news_by_id(news_id): print news_id news = News(link=news_id, title=title, contents=content, written_time=written_clock, crawl_time=dt.datetime.now()) session.add(news) session.commit() saved = True session.close() return saved
def store_similarity(self, user_id1, user_id2, similarity): session = Session() try: sim = Similarity(user_id1=user_id1, user_id2=user_id2, similarity=similarity) session.add(sim) session.commit() except Exception as e: print e session.rollback() finally: session.close()
def make_user_list(self): session = Session() for row in session.query(Comment.Writer, func.count(Comment.Id))\ .group_by(Comment.Writer)\ .all(): if not self.get_userid(row[0]): insert_user = User(UserId=row[0], TrainTest='train', Written=int(row[1])) session.add(insert_user) session.commit() print row[0], 'train', row[1] else: self.update_user_list(row[0], int(row[1])) session.close()
def save_news(self, link, title, content, written_time): saved = False session = Session() if not self.get_news_by_id(link): #print link news = News(link=link, title=title, content=content, written_time=written_time, crawl_time=datetime.datetime.now()) session.add(news) session.commit() saved = True session.close() return saved
def save_comment(self, id, title, point, comment): session = Session() if not self.get_comment_by_id(id): print id print title print 'point: ' + point print comment print '=' * 70 data = comments(ID=id, title=title.encode('utf-8'), point=point, comment=comment.encode('utf-8')) session.add(data) session.commit() return True else: return False session.close()
def save_news(self, news_url, news_title, news_contents, news_company, news_reporter_email, news_date): if self.get_news_id(news_url): session = Session() insert_news = NewsArticle(Link = news_url, Title = news_title, Content = news_contents, NewsCompany = news_company, ReporterEmail = news_reporter_email, ReportDate = news_date) session.add(insert_news) session.commit() session.close() else: session = Session() update_news = session.query(NewsArticle).filter(NewsArticle.Link == news_url).one() update_news.Title = news_title update_news.Content = news_contents update_news.NewsCompany = news_company update_news.ReporterEmail = news_reporter_email update_news.ReportDate = news_date session.commit() session.close()
def post(self): status = 200; if validationUtil.is_json(request.data): args = self.reqparse.parse_args(strict=True, custom=True) if args['error']: genericError = GenericError(0, list(args['body'].values())) res = genericError.__dict__ else: name = request.json['name'] description = request.json['description'] errors = [] if not name: errors.append('Name is empty') if not description: errors.append('Description is empty') if len(errors) <= 0: session = Session() newStock = Stock(request.json['name'], request.json['description']) session.add(newStock) session.flush() res = newStock._asdict() session.commit() session.close() else: genericError = GenericError(0, errors) res = genericError.__dict__ status = 400 else: errors.append('Not a valid JSON format') res = GenericError(0, errors).__dict__ status = 400 return res, status