def parse_currency(currency_key, directory, name): # Type : 1 - Currency
print("Currency")
col = Mongo().create_collection("Currency", FDC.get_index_models())
with open(directory) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
print(currency_key)
hour = -1
fd = None
for row in csv_reader:
if len(row) < 2: # Check Data
continue
add_value = 0
if currency_key == "EURUSD":
date = DateHelper.str2date(row[0])
add_value = -1
else:
date = DateHelper.str2date(row[0]+row[1])
if hour != date.hour:
hour = date.hour
if fd is not None:
try:
col.insert(fd.get_currency())
except:
Logger().get_logger().error('Insert Error', exc_info=True)
fd = FinancialData(name, currency_key, date,
row[FDLocations.Currency_Open.value + add_value],
row[FDLocations.Currency_High.value + add_value],
row[FDLocations.Currency_Low.value + add_value],
row[FDLocations.Currency_Close.value + add_value])
else:
fd.add(row[FDLocations.Currency_High.value + add_value],
row[FDLocations.Currency_Low.value + add_value],
row[FDLocations.Currency_Close.value + add_value])
def parse_index_datetime(currency_key, directory, name, interval): # Type : 4 - Index
col = Mongo().create_collection("Index")
with open(directory) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
line_count = 0
print(currency_key)
hour = -1
hour_count = 0
fd = None
for row in csv_reader:
if len(row) < 2: # Check Data
continue
date = DateHelper.str2date(row[0] + row[1])
if hour != date.hour:
hour = date.hour
hour_count = 0
if fd is not None:
print(fd)
try:
col.insert(fd.get_index())
except:
Logger().get_logger().error('Insert Error', exc_info=True)
fd = FinancialData(name, currency_key, date,
row[FDLocations.IndexDateTime_Open.value],
row[FDLocations.IndexDateTime_High.value],
row[FDLocations.IndexDateTime_Low.value],
row[FDLocations.IndexDateTime_Close.value])
else:
fd.add(row[FDLocations.IndexDateTime_High.value],
row[FDLocations.IndexDateTime_Low.value],
row[FDLocations.IndexDateTime_Close.value])
hour_count += 1
line_count += 1
print(f'Processed {line_count} lines.')
def sentiment_analysis_test(date=None, hashtags=None):
nlp = pipeline('sentiment-analysis')
if date is None:
date = DateHelper.str2date("2015-05-12T16:07:40Z")
if hashtags is None:
hashtags = ["oil", "crude", "crude oil"]
tweets = TwitterForecast.get_tweets_before_date_from_elastic_search(
date, hashtags, days=5, maxsize=10000)
total_tweets = tweets["hits"]["total"]["value"]
if total_tweets == 0:
print("No Tweet Found")
else:
for es_tweet in tweets["hits"]["hits"]:
tweet = es_tweet["_source"]
try:
text = tweet["tweet_text"].replace("\n", "")
username = tweet['tweet_user_name']
sentiment = nlp(text)[0]
if sentiment['score'] > 0.98:
if tweet["tweet_user_verified"]:
print('[%s-%s] - %s (%s)' %
(u"\U0001F44D", sentiment['label'], text,
username))
else:
print('[%s] - %s (%s)' %
(sentiment['label'], text, username))
except Exception as exception:
print(exception)
traceback.print_exc()
async def __price_handler(self, request):
request = await request.json()
date = DateHelper.str2date(request['news_date'])
info = self.get_price_before_date(self.db, request['collection'],
request['key'], date,
request['range'])
date_list = []
open_list = []
high_list = []
low_list = []
close_list = []
volume_list = []
for a in info:
date_list.append(str(a.get('Date')))
open_list.append(a.get('Open'))
high_list.append(a.get('High'))
low_list.append(a.get('Low'))
close_list.append(a.get('Close'))
volume_list.append(a.get('Volume'))
res = {
'Title': request['collection'] + " - " + request['key'],
'PriceDate': date_list,
'OpenPrice': open_list,
'HighPrice': high_list,
'LowPrice': low_list,
'ClosePrice': close_list,
'Volume': volume_list
}
res = JSONEncoder().encode(res)
return web.json_response(res)
def get_date(news):
date = news['Date']
rss_date = news['RSS_Date']
selected_date = rss_date
if date:
if DateHelper.is_time_of_date_exist(date):
try:
if date > rss_date:
selected_date = rss_date
else:
selected_date = date
except:
selected_date = date
elif rss_date:
selected_date = rss_date
else:
try:
metadata = news['Meta_Data'].get("pubdate")
if metadata:
return DateHelper.str2date(metadata)
else:
html = news['HTML']
sub_index = html.find('publishDate')
print(news['Meta_Data']["pubdate"])
if sub_index > 0:
date = html[sub_index:(sub_index + 100)]
result = re.search('publishDate":"(.*?)",', date)
if result:
print(result.group(1))
selected_date = DateHelper.str2date(
result.group(1))
else:
return None
else:
return None
except Exception:
return None
return selected_date
def collect(self):
db = Mongo()
conn = sqlite3.connect(self.SQL_LOCATION)
c = conn.cursor()
c.execute(
'SELECT title, author, date, publication, category, digital, section, url FROM longform'
)
line_count = 0
date_count = 0
newslist = []
for row in c:
url = row[self.Url]
date = DateHelper.str2date(row[self.Date])
title = row[self.Title]
if url == "" or url is None or date == "": # Is There Url Or Date
continue
if db.is_title_url_exists(title, url):
continue
allUrls = FileCollector.extract_url_from_text(url)
article = Article(allUrls[1])
category = row[self.Category]
section = row[self.Section]
newslist.append(
News.RssNews(title=title,
time=date,
summery='',
category=FileCollector.get_category(
category, section),
tags='',
url=allUrls[1],
iaurl=allUrls[0],
article=article))
print(line_count)
if len(newslist) == 20:
pool = NewsPool()
pool.set(newslist)
pool.join()
newslist = []
line_count += 1
print(f'\t{line_count}')
print(f'\t{len(newslist)}')
def parse_stock(currency_key, directory, name, interval): # Type : 3 - Stock
print("Stock")
col = Mongo().create_collection("Stock", FDC.get_index_models())
with open(directory) as csv_file:
csv_reader = csv.reader(csv_file, delimiter=',')
print(currency_key)
for row in csv_reader:
if len(row) < 2: # Check Data
continue
date = DateHelper.str2date(row[0])
if interval == 60:
fd = FinancialData(name, currency_key, date,
row[FDLocations.Stock_Open.value],
row[FDLocations.Stock_High.value],
row[FDLocations.Stock_Low.value],
row[FDLocations.Stock_Close.value],
row[FDLocations.Stock_Volume.value],
row[FDLocations.Stock_Trade.value],
row[FDLocations.Stock_Avg.value])
col.insert(fd.get_stock())
else:
print("Not Handled !!!")