def save_types(self, place_id, types):
session = Session()
if not self.get_types(place_id, types):
type = Types(place_id = place_id, types = types)
session.add(type)
session.commit()
print '<TYPE>', place_id, types, 'saved successfully!'
session.close()
def save_users(self, user_id, place_id):
session = Session()
if not self.get_users(user_id, place_id):
user = Users(user_id = user_id, place_id = place_id)
session.add(user)
session.commit()
print '<USER>', user_id, place_id, 'saved successfully.'
session.close()
def save_place(self, place_id, city, name, address, lng, lat):
session = Session()
if not self.get_place(place_id):
geo = GeoInfos(place_id = place_id, name = name, city = city, address = address, lng = lng, lat = lat)
session.add(geo)
session.commit()
print '<GEOINFO>', place_id, name, 'saved successfully!'
session.close()
def criar(item):
try:
session = Session()
session.add(item)
session.commit()
return item
except Exception as e:
return internal_error(e), 500
finally:
session.close()
def save_ratings(self, place_id, gg_reviews, gg_ratings, ta_reviews, ta_ratings):
session = Session()
if not self.get_ratings(place_id):
rate = Ratings(place_id = place_id, gg_reviews = gg_reviews, gg_ratings = gg_ratings,
ta_reviews = ta_reviews, ta_ratings = ta_ratings)
session.add(rate)
session.commit()
print '<RATING>', place_id, 'saved successfully!'
session.close()
def save_news(self, news_id, title, content):
session = Session()
if not self.get_news_by_id(news_id):
print news_id
news = News(link=news_id,
title=title,
content=content,
crawl_time=datetime.datetime.now())
session.add(news)
session.commit()
session.close()
def save_comment(self, id, news_id, content, written_time, sympathy_count, antipathy_count):
session = Session()
if not self.get_comment_by_id(id):
print content
comment = Comment(id = id, news_id = news_id, content = content,
written_time = written_time, sympathy_count = sympathy_count,
antipathy_count = antipathy_count, crawl_time = datetime.datetime.now())
session.add(comment)
session.commit()
session.close()
def save_comment_info(self, productno, comment_writer, comment_grade):
if self.get_comment_info(productno, comment_writer):
session = Session()
insert_comment = Comment(ProductNo=productno,
Writer=comment_writer,
Grade=comment_grade,
Enrolltime=crawltime)
session.add(insert_comment)
session.commit()
session.close()
else:
self.update_comment_info(productno, comment_writer, comment_grade)
def save_movie_info(self, title, genre):
session = Session()
if not self.get_movie_by_title(title):
print title
print 'genre: ' + genre
print '=' * 70
data = movie_info(title=title, genre=genre)
session.add(data)
session.commit()
return True
else:
return False
session.close()
class MainExecutor():
def __init__(self,id):
self.id = id
self.Session = Session
self.session = Session()
self.process = self.session.query(models.Process).options(joinedload('queues').joinedload('queues_tasks').joinedload('task')).filter(models.Process.id == self.id).first()
if not self.process:
raise Exception(f"Can't find process with id ='{self.id}'")
with open('./config.yaml','r') as config:
settings = yaml.safe_load(config)
self.logger = create_logger('main', os.path.join(settings['logs']['test']['logpath'],self.process.name, dt.datetime.now().strftime("%Y%m%d_%H%M%S")))
self.logger.info(f"Executor initiated for process with id = '{self.id}'")
self.logger.debug(self.process)
self.threads = []
self.threads_with_errors = []
def prepare(self):
self.logger.info('Creating Run instance.')
self.run = self.process.create_run()
self.session.add(self.run)
self.session.commit()
self.logger.debug(self.run)
self.logger.info('Run saved in db.')
def execute(self):
error = False
try:
self.logger.info("Executing.")
splitted_queues = split_by_order(self.process.queues)
if not splitted_queues:
self.logger.error("Splitted queues is empty.")
raise Exception("Splitted queues is empty.")
for key,val in splitted_queues.items():
blockers = []
for queue_item in val:
QT = QueueThread('queue-' + str(queue_item.id),queue_item,self.run,self.Session,self.logger)
QT.start()
if queue_item.blocking:
self.logger.info(f" Queue.id: '{str(queue_item.id)}' added to blocking in same run order.")
blockers.append(QT)
else:
self.logger.info(f" Queue.id: '{str(queue_item.id)}' not blocking anything. Adding it to left threads.")
self.threads.append(QT)
[ (x.join(),self.logger.info(f"Joined Queue.id: '{str(x.queue.id)}'")) for x in blockers]
[x.join() for x in self.threads]
print('end')
except Exception as e:
finally:
self.run.end_date = dt.datetime.now()
self.session.commit()
def save_product_info(self, productno, link, title, category):
if self.get_product_id(productno):
session = Session()
insert_product = Product(ProductNo=productno,
Link=link,
Name=title,
Category=category,
TrainTest='train',
Enrolltime=crawltime)
session.add(insert_product)
session.commit()
session.close()
else:
self.update_product_info(productno, link, title, category)
def save_comments(self, coId, url, mcontent, user, sympathy, antipathy,
enrolltime):
if self.get_comments_id(coId):
session = Session()
insert_comment = CommentList(Id=coId,
Link=url,
Mcontent=mcontent,
User=user,
Sympathy=sympathy,
Antipathy=antipathy,
Enrolltime=enrolltime)
session.add(insert_comment)
session.commit()
session.close()
def save_news(self, news_id, title, content, written_clock):
saved = False
session = Session()
if not self.get_news_by_id(news_id):
print news_id
news = News(link=news_id, title=title, contents=content,
written_time=written_clock, crawl_time=dt.datetime.now())
session.add(news)
session.commit()
saved = True
session.close()
return saved
def store_similarity(self, user_id1, user_id2, similarity):
session = Session()
try:
sim = Similarity(user_id1=user_id1,
user_id2=user_id2,
similarity=similarity)
session.add(sim)
session.commit()
except Exception as e:
print e
session.rollback()
finally:
session.close()
def make_user_list(self):
session = Session()
for row in session.query(Comment.Writer, func.count(Comment.Id))\
.group_by(Comment.Writer)\
.all():
if not self.get_userid(row[0]):
insert_user = User(UserId=row[0],
TrainTest='train',
Written=int(row[1]))
session.add(insert_user)
session.commit()
print row[0], 'train', row[1]
else:
self.update_user_list(row[0], int(row[1]))
session.close()
def save_news(self, link, title, content, written_time):
saved = False
session = Session()
if not self.get_news_by_id(link):
#print link
news = News(link=link,
title=title,
content=content,
written_time=written_time,
crawl_time=datetime.datetime.now())
session.add(news)
session.commit()
saved = True
session.close()
return saved
def save_comment(self, id, title, point, comment):
session = Session()
if not self.get_comment_by_id(id):
print id
print title
print 'point: ' + point
print comment
print '=' * 70
data = comments(ID=id,
title=title.encode('utf-8'),
point=point,
comment=comment.encode('utf-8'))
session.add(data)
session.commit()
return True
else:
return False
session.close()
def save_news(self, news_url, news_title, news_contents, news_company, news_reporter_email, news_date):
if self.get_news_id(news_url):
session = Session()
insert_news = NewsArticle(Link = news_url, Title = news_title, Content = news_contents, NewsCompany = news_company, ReporterEmail = news_reporter_email, ReportDate = news_date)
session.add(insert_news)
session.commit()
session.close()
else:
session = Session()
update_news = session.query(NewsArticle).filter(NewsArticle.Link == news_url).one()
update_news.Title = news_title
update_news.Content = news_contents
update_news.NewsCompany = news_company
update_news.ReporterEmail = news_reporter_email
update_news.ReportDate = news_date
session.commit()
session.close()
def post(self):
status = 200;
if validationUtil.is_json(request.data):
args = self.reqparse.parse_args(strict=True, custom=True)
if args['error']:
genericError = GenericError(0, list(args['body'].values()))
res = genericError.__dict__
else:
name = request.json['name']
description = request.json['description']
errors = []
if not name:
errors.append('Name is empty')
if not description:
errors.append('Description is empty')
if len(errors) <= 0:
session = Session()
newStock = Stock(request.json['name'], request.json['description'])
session.add(newStock)
session.flush()
res = newStock._asdict()
session.commit()
session.close()
else:
genericError = GenericError(0, errors)
res = genericError.__dict__
status = 400
else:
errors.append('Not a valid JSON format')
res = GenericError(0, errors).__dict__
status = 400
return res, status