def get_post(post_id):
session = Session()
try:
post = session.query(Post).get(post_id)
if post == None:
return not_found
else:
image_path = image_format(post.id)
if os.path.isfile(image_path):
image_path = image_host + '/' + image_path
else:
image_path = stub_image
return successful({
'author_id': post.author_id,
'title': post.title,
'description': post.description,
'body': post.body,
'created_at': post.created_at,
'image': image_path
})
except:
return internal_server_error
finally:
session.close()
def get_articles(gvket, date):
session = Session()
q = session.query(CompanyArticle) \
.join(Articles, Articles.id == CompanyArticle.article_id) \
.filter(CompanyArticle.gvkey == gvkey) \
.filter(Articles.PD == date)
return q.all()
def get_posts(limit, offset):
session = Session()
try:
result = []
for post in session.query(
Post.id, Post.author_id, Post.title, Post.description,
Post.created_at).limit(limit).offset(offset).all():
image_path = image_format(post.id)
if os.path.isfile(image_path):
image_path = image_host + '/' + image_path
else:
image_path = stub_image
result.append({
'id': post.id,
'author_id': post.author_id,
'title': post.title,
'description': post.description,
'created_at': post.created_at,
'image': image_path
})
return successful(result)
except:
return internal_server_error
finally:
session.close()
def main_worker(article):
co_list = article.CO.split('|')
if len(co_list) > 0:
for organization in co_list:
inserted = insert_comp_art(organization, article)
else:
logger.warning('Could not get company codes for article %s',
article.id)
Session.remove()
def process_file(fname):
file_location = os.path.join(settings.RTF_DIR, fname)
logger.info('Opening file %s...', fname)
session = Session()
if fname.startswith('~$'):
return None
try:
with open(file_location, 'rb') as rtf_file:
txt = rtf_file.read()
except Exception:
logger.warning('Cannot read from file %s', fname)
clean_text = striprtf(txt)
dicts = parser(clean_text, fname)
if len(dicts) == 0:
logger.error('Cannot extract articles from file %s', fname)
return None
logger.info('Found %d articles in file %s', len(dicts), fname)
for dict_item in dicts:
article = session.query(Articles).filter_by(id=dict_item['id']).first()
if article:
logger.info('Article %s already exists in database', article.id)
else:
article = Articles(**dict_item)
session.add(article)
session.commit()
session.close()
logger.info('Finished parsing file %s...', fname)
def get_company_by_code(code):
try:
session = Session()
q = session.query(Company).filter(Company.factiva_code == code).first()
if q is None:
logger.warning('Could not match company with code %s', code)
return None
return q
except Exception:
logger.exception('message')
raise
def try_get_user_id(user_token):
session = Session()
try:
user = session.query(User).filter(User.token==user_token).scalar()
if user == None:
return False, 0
else:
return True, user.id
except:
return False, 0
finally:
session.close()
def get_unmatched_articles():
# session = Session()
try:
Session = sessionmaker(bind=db_engine, autoflush=False)
session = Session()
q = session.query(Articles) \
.outerjoin(CompanyArticle, CompanyArticle.article_id == Articles.id)\
.filter(CompanyArticle.article_id.is_(None))
return (r for r in q.execution_options(stream_results=True))
except Exception:
logger.exception('message')
raise
def create_post(author_id, title, description, body):
session = Session()
try:
post = Post(author_id, title, description, body,
strftime("%Y-%m-%dT%H:%M:%S", gmtime()))
session.add(post)
session.commit()
return successful(post.id)
except:
session.rollback()
return internal_server_error
finally:
session.close()
def insert_comp_art(organization, article):
try:
session = Session()
org_list = organization.split(':')
code = org_list[0].strip().upper()
company = get_company_by_code(code)
if company is None:
return None
com_art = CompanyArticle(gvkey=company.gvkey, article_id=article.id)
if article.NS is not None:
ns_list = article.NS.split('|')
else:
ns_list = []
i = 0
for cat in ns_list:
cat_list = cat.split(':')
match_cnum = re.match('c\d+', cat_list[0].strip())
if i > 2 or match_cnum is None:
continue
i += 1
if i == 1:
com_art.main_category = cat_list[1].strip()
if i == 2:
com_art.sub_category = cat_list[1].strip()
session.add(com_art)
session.commit()
logger.info('Matched company %s to article %s', company.factiva_name,
com_art.article_id)
session.close()
except Exception:
logger.exception('message')
raise
def update_post(title, description, body, post_id):
session = Session()
try:
session.query(Post).filter(Post.id == post_id).update({
'title': title,
'description': description,
'body': body
})
session.commit()
return successful('Post {} updated'.format(post_id))
except:
session.rollback()
return internal_server_error
finally:
session.close()
def new_user(email, name, password_encrypted, token):
session = Session()
try:
user = User(email, name, password_encrypted, token)
session.add(user)
session.commit()
return successful({
'id': user.id,
'email': user.email,
'password_encrypted': user.password_encrypted,
'token': user.token,
})
except:
session.rollback()
return internal_server_error
finally:
session.close()
def get_user(email):
session = Session()
try:
user = session.query(User).filter(User.email == email).scalar()
if user == None:
return not_found
else:
return successful({
'id': user.id,
'email': user.email,
'password_encrypted': user.password_encrypted,
'token': user.token,
})
except:
return internal_server_error
finally:
session.close()
def delete_post(post_id):
session = Session()
try:
session.query(Post).filter(Post.id == post_id).delete()
session.commit()
image_path = image_format(post_id)
if os.path.isfile(image_path):
os.remove(image_path)
return successful('Post {} deleted'.format(post_id))
except:
session.rollback()
return internal_server_error
finally:
session.close()
def get_articles():
session = Session()
articles = session.query(Articles).all()
for article in articles:
analysed = session.query(Analysis).filter_by(id=article.id).first()
if analysed:
continue
logger.info('Analysing %s', article.id)
text = article.text.rstrip('None')
analyzed_dict = get_data(text)
analyzed_dict['id'] = article.id
analyzed_dict.pop('doc_size')
a = Analysis(**analyzed_dict)
session.add(a)
session.commit()
def import_data(path):
with open(path) as f:
print(1)
line_count = 0
reader = csv.reader(f, delimiter=';')
session = Session()
for row in reader:
if line_count == 0:
print(f'Column names are {", ".join(row)}')
line_count += 1
else:
company = Company(
gvkey=row[0],
name=row[1],
factiva_name=row[2],
factiva_code=row[3],
)
session.add(company)
session.commit()
print(f'Processed {line_count} lines.')
def get_analysis(article_id):
session = Session()
q = session.query(Analysis).filter(Analysis.id == article_id)
if q is not None:
return q.first()
return None