def get_post(post_id): session = Session() try: post = session.query(Post).get(post_id) if post == None: return not_found else: image_path = image_format(post.id) if os.path.isfile(image_path): image_path = image_host + '/' + image_path else: image_path = stub_image return successful({ 'author_id': post.author_id, 'title': post.title, 'description': post.description, 'body': post.body, 'created_at': post.created_at, 'image': image_path }) except: return internal_server_error finally: session.close()
def get_articles(gvket, date): session = Session() q = session.query(CompanyArticle) \ .join(Articles, Articles.id == CompanyArticle.article_id) \ .filter(CompanyArticle.gvkey == gvkey) \ .filter(Articles.PD == date) return q.all()
def get_posts(limit, offset): session = Session() try: result = [] for post in session.query( Post.id, Post.author_id, Post.title, Post.description, Post.created_at).limit(limit).offset(offset).all(): image_path = image_format(post.id) if os.path.isfile(image_path): image_path = image_host + '/' + image_path else: image_path = stub_image result.append({ 'id': post.id, 'author_id': post.author_id, 'title': post.title, 'description': post.description, 'created_at': post.created_at, 'image': image_path }) return successful(result) except: return internal_server_error finally: session.close()
def main_worker(article): co_list = article.CO.split('|') if len(co_list) > 0: for organization in co_list: inserted = insert_comp_art(organization, article) else: logger.warning('Could not get company codes for article %s', article.id) Session.remove()
def process_file(fname): file_location = os.path.join(settings.RTF_DIR, fname) logger.info('Opening file %s...', fname) session = Session() if fname.startswith('~$'): return None try: with open(file_location, 'rb') as rtf_file: txt = rtf_file.read() except Exception: logger.warning('Cannot read from file %s', fname) clean_text = striprtf(txt) dicts = parser(clean_text, fname) if len(dicts) == 0: logger.error('Cannot extract articles from file %s', fname) return None logger.info('Found %d articles in file %s', len(dicts), fname) for dict_item in dicts: article = session.query(Articles).filter_by(id=dict_item['id']).first() if article: logger.info('Article %s already exists in database', article.id) else: article = Articles(**dict_item) session.add(article) session.commit() session.close() logger.info('Finished parsing file %s...', fname)
def get_company_by_code(code): try: session = Session() q = session.query(Company).filter(Company.factiva_code == code).first() if q is None: logger.warning('Could not match company with code %s', code) return None return q except Exception: logger.exception('message') raise
def try_get_user_id(user_token): session = Session() try: user = session.query(User).filter(User.token==user_token).scalar() if user == None: return False, 0 else: return True, user.id except: return False, 0 finally: session.close()
def get_unmatched_articles(): # session = Session() try: Session = sessionmaker(bind=db_engine, autoflush=False) session = Session() q = session.query(Articles) \ .outerjoin(CompanyArticle, CompanyArticle.article_id == Articles.id)\ .filter(CompanyArticle.article_id.is_(None)) return (r for r in q.execution_options(stream_results=True)) except Exception: logger.exception('message') raise
def create_post(author_id, title, description, body): session = Session() try: post = Post(author_id, title, description, body, strftime("%Y-%m-%dT%H:%M:%S", gmtime())) session.add(post) session.commit() return successful(post.id) except: session.rollback() return internal_server_error finally: session.close()
def insert_comp_art(organization, article): try: session = Session() org_list = organization.split(':') code = org_list[0].strip().upper() company = get_company_by_code(code) if company is None: return None com_art = CompanyArticle(gvkey=company.gvkey, article_id=article.id) if article.NS is not None: ns_list = article.NS.split('|') else: ns_list = [] i = 0 for cat in ns_list: cat_list = cat.split(':') match_cnum = re.match('c\d+', cat_list[0].strip()) if i > 2 or match_cnum is None: continue i += 1 if i == 1: com_art.main_category = cat_list[1].strip() if i == 2: com_art.sub_category = cat_list[1].strip() session.add(com_art) session.commit() logger.info('Matched company %s to article %s', company.factiva_name, com_art.article_id) session.close() except Exception: logger.exception('message') raise
def update_post(title, description, body, post_id): session = Session() try: session.query(Post).filter(Post.id == post_id).update({ 'title': title, 'description': description, 'body': body }) session.commit() return successful('Post {} updated'.format(post_id)) except: session.rollback() return internal_server_error finally: session.close()
def new_user(email, name, password_encrypted, token): session = Session() try: user = User(email, name, password_encrypted, token) session.add(user) session.commit() return successful({ 'id': user.id, 'email': user.email, 'password_encrypted': user.password_encrypted, 'token': user.token, }) except: session.rollback() return internal_server_error finally: session.close()
def get_user(email): session = Session() try: user = session.query(User).filter(User.email == email).scalar() if user == None: return not_found else: return successful({ 'id': user.id, 'email': user.email, 'password_encrypted': user.password_encrypted, 'token': user.token, }) except: return internal_server_error finally: session.close()
def delete_post(post_id): session = Session() try: session.query(Post).filter(Post.id == post_id).delete() session.commit() image_path = image_format(post_id) if os.path.isfile(image_path): os.remove(image_path) return successful('Post {} deleted'.format(post_id)) except: session.rollback() return internal_server_error finally: session.close()
def get_articles(): session = Session() articles = session.query(Articles).all() for article in articles: analysed = session.query(Analysis).filter_by(id=article.id).first() if analysed: continue logger.info('Analysing %s', article.id) text = article.text.rstrip('None') analyzed_dict = get_data(text) analyzed_dict['id'] = article.id analyzed_dict.pop('doc_size') a = Analysis(**analyzed_dict) session.add(a) session.commit()
def import_data(path): with open(path) as f: print(1) line_count = 0 reader = csv.reader(f, delimiter=';') session = Session() for row in reader: if line_count == 0: print(f'Column names are {", ".join(row)}') line_count += 1 else: company = Company( gvkey=row[0], name=row[1], factiva_name=row[2], factiva_code=row[3], ) session.add(company) session.commit() print(f'Processed {line_count} lines.')
def get_analysis(article_id): session = Session() q = session.query(Analysis).filter(Analysis.id == article_id) if q is not None: return q.first() return None