def parse_page(responce_get: str) -> list:
conn = h2('minfin.com.ua:443')
conn.request('GET',
'/news/' + urls_dict['currency'] + '/',
headers=headers)
resp = conn.get_response()
if resp.status != 200:
logger.error(
'could not fetch news from minfin.com.ua, resp.status= {}'.format(
resp.status))
return []
text = resp.read()
soup = BeautifulSoup(text, 'html.parser')
current_date = current_datetime_tz()
data = []
news_table = soup.body.find(attrs={'class': ['all-news', 'news-all']})
for news in news_table.find_all(attrs={'class': ['item']}):
logger.debug('news= {}'.format(news))
if news.div != None:
continue
dic = {
'time': news.span['content'],
'href': 'http://minfin.com.ua' + news.a['href'],
'headline': news.a.get_text(),
'source': 'm'
}
dic['time'] = datetime.strptime(dic['time'], '%Y-%m-%d %H:%M:%S')
dic['time'] = kyiv_tz.localize(dic['time']).astimezone(timezone.utc)
data.append(dic)
logger.debug('minfin_headlines data= {}'.format(data))
return data
def data_api_minfin(fn):
def convertor_minfin(dic: dict, current_date: datetime, id: int) -> dict:
dic['bid'] = dic['id']
del dic['id']
# dic['id'] = id
dic['currency'] = dic['currency'].upper()
dic['location'] = location_orig
dic['source'] = 'm'
dic['session'] = False
time = dic['time'].split(':')
dic['time'] = current_date.replace(hour=int(time[0]), minute=int(time[1]), second=0, microsecond=0)
dic['rate'] = float(dic['rate'])
dic['amount'] = int(''.join(filter(lambda x: x.isdigit(), dic['amount'])))
# TODO: remove below strings
# if dic['time'] > current_date:
# dic['time'] = dic['time'] - timedelta(days=1)
return dic
data = {}
sessions = []
for cur in ['RUB', 'EUR', 'USD']:
for oper in ['sell', 'buy']:
fn_result = fn(cur.lower(), oper)
if len(fn_result) < 1:
logger.error('empty result for currency={}, operation= {}'.format(cur, oper))
continue
data.update(fn_result[0])
sessions.append({'currency': cur.upper(), 'source': 'm', 'operation': oper,
'url': fn_result[1]['url'], 'cookies': fn_result[1]['cookies'],
'session': True})
current_date = kyiv_tz.localize(datetime.now())
return [convertor_minfin(value, current_date, index) for index, value in enumerate(data.values())] + sessions
def data_api_minfin(fn):
def convertor_minfin(dic: dict, current_date: datetime, id: int) -> dict:
dic['bid'] = dic['id']
del dic['id']
# dic['id'] = id
dic['currency'] = dic['currency'].upper()
dic['location'] = location_orig
dic['source'] = 'm'
dic['session'] = False
time = dic['time'].split(':')
dic['time'] = current_date.replace(hour=int(time[0]), minute=int(time[1]), second=0, microsecond=0)
dic['rate'] = float(dic['rate'])
dic['amount'] = int(''.join(filter(lambda x: x.isdigit(), dic['amount'])))
# TODO: remove below strings
# if dic['time'] > current_date:
# dic['time'] = dic['time'] - timedelta(days=1)
return dic
data = {}
sessions = []
for cur in ['RUB', 'EUR', 'USD']:
for oper in ['sell', 'buy']:
fn_result = fn(cur.lower(), oper)
if len(fn_result) < 1:
logger.error('empty result for currency={}, operation= {}'.format(cur, oper))
continue
data.update(fn_result[0])
sessions.append({'currency': cur.upper(), 'source': 'm', 'operation': oper,
'url': fn_result[1]['url'], 'cookies': fn_result[1]['cookies'],
'session': True})
current_date = kyiv_tz.localize(datetime.now())
return [convertor_minfin(value, current_date, index) for index, value in enumerate(data.values())] + sessions
def parse_page(responce_get: str) -> list:
conn = h2('minfin.com.ua:443')
conn.request('GET', '/news/'+ urls_dict['currency'] + '/', headers=headers)
resp = conn.get_response()
if resp.status != 200:
logger.error('could not fetch news from minfin.com.ua, resp.status= {}'.format(resp.status))
return []
text = resp.read()
soup = BeautifulSoup(text, 'html.parser')
current_date = current_datetime_tz()
data = []
news_table = soup.body.find(attrs={'class': ['all-news', 'news-all']})
for news in news_table.find_all(attrs={'class': ['item']}):
logger.debug('news= {}'.format(news))
if news.div != None:
continue
dic = {'time': news.span['content'],
'href': 'http://minfin.com.ua' + news.a['href'],
'headline': news.a.get_text(),
'source': 'm'}
dic['time'] = datetime.strptime(dic['time'], '%Y-%m-%d %H:%M:%S')
dic['time'] = kyiv_tz.localize(dic['time']).astimezone(timezone.utc)
data.append(dic)
logger.debug('minfin_headlines data= {}'.format(data))
return data
def from_form(self, in_doc: dict):
for key in self._mandatory:
try:
setattr(self, key, in_doc[key])
except KeyError as e:
setattr(self, key, None)
for key in self._optional:
if in_doc.get(key) not in ('', 'None', None):
setattr(self, key, in_doc[key])
# specific keys handling
# add tz to naive datetime
all_attrs = {**self._mandatory, **self._optional}
for key in [i for i in all_attrs if all_attrs[i] == datetime]:
if getattr(self, key, None) is not None:
try:
# expected recive naive time in Kyiv tz
setattr(self, key, kyiv_tz.localize(self.__dict__[key]))
except ValueError:
# Not naive datetime (tzinfo is already set)
setattr(self, key, self.__dict__[key].astimezone(kyiv_tz))
# convert string to list
for key in [i for i in all_attrs if all_attrs[i] == list]:
if getattr(self, key, None) is not None:
setattr(self, key, self.__dict__[key].replace(' ', '').split(','))
setattr(self, key, list(set(self.__dict__[key])))
return {key: val for key, val in self.__dict__.items()
if not key.startswith('_') and val is not None and val != ''}
def from_form(self, in_doc: dict):
for key in self._mandatory:
try:
setattr(self, key, in_doc[key])
except KeyError as e:
setattr(self, key, None)
for key in self._optional:
if in_doc.get(key) not in ('', 'None', None):
setattr(self, key, in_doc[key])
# specific keys handling
# add tz to naive datetime
all_attrs = {**self._mandatory, **self._optional}
for key in [i for i in all_attrs if all_attrs[i] == datetime]:
if getattr(self, key, None) is not None:
try:
# expected recive naive time in Kyiv tz
setattr(self, key, kyiv_tz.localize(self.__dict__[key]))
except ValueError:
# Not naive datetime (tzinfo is already set)
setattr(self, key, self.__dict__[key].astimezone(kyiv_tz))
# convert string to list
for key in [i for i in all_attrs if all_attrs[i] == list]:
if getattr(self, key, None) is not None:
setattr(self, key, self.__dict__[key].replace(' ',
'').split(','))
setattr(self, key, list(set(self.__dict__[key])))
return {
key: val
for key, val in self.__dict__.items()
if not key.startswith('_') and val is not None and val != ''
}
def from_lists(self, doc: dict):
self.contract_currency = doc['currency']
self.contract_rate = doc['rate']
self.contract_phones = doc['phone']
self.contract_time = kyiv_tz.localize(datetime.now())
self.contract_comments = doc['comment']
return self.as_dict()
def from_lists(self, doc: dict):
self.contract_currency = doc['currency']
self.contract_rate = doc['rate']
self.contract_phones = doc['phone']
self.contract_time = kyiv_tz.localize(datetime.now())
self.contract_comments = doc['comment']
return self.as_dict()
def convertor_berlox(id: int, dic: dict, current_date: datetime) -> dict:
out_dic = {}
for key, item in dic.items():
out_dic[conv_dict_orig[key]] = item
out_dic['operation'] = conv_operation_orig[out_dic['operation']]
# out_dic['id'] = id
out_dic['source'] = 'b'
try:
out_dic['time'] = datetime.strptime(out_dic['time'], '%Y-%m-%dT%H:%M:%S')
out_dic['time'] = kyiv_tz.localize(out_dic['time'])
except ValueError:
try:
out_dic['time'] = datetime.strptime(out_dic['time'], '%Y-%m-%dT%H:%M:%S.%f%z')
except ValueError:
out_dic['time'] = datetime.strptime(out_dic['time'][:19], '%Y-%m-%dT%H:%M:%S')
out_dic['time'] = kyiv_tz.localize(out_dic['time'])
return out_dic
def convertor_berlox(id: int, dic: dict, current_date: datetime) -> dict:
out_dic = {}
for key, item in dic.items():
out_dic[conv_dict_orig[key]] = item
out_dic['operation'] = conv_operation_orig[out_dic['operation']]
# out_dic['id'] = id
out_dic['source'] = 'b'
try:
out_dic['time'] = datetime.strptime(out_dic['time'],
'%Y-%m-%dT%H:%M:%S')
out_dic['time'] = kyiv_tz.localize(out_dic['time'])
except ValueError:
try:
out_dic['time'] = datetime.strptime(out_dic['time'],
'%Y-%m-%dT%H:%M:%S.%f%z')
except ValueError:
out_dic['time'] = datetime.strptime(out_dic['time'][:19],
'%Y-%m-%dT%H:%M:%S')
out_dic['time'] = kyiv_tz.localize(out_dic['time'])
return out_dic
def create_contract(_id='new'):
"""
create new contract or edit existing
:param _id:
:return:
"""
# if request.method == 'GET':
if _id == 'new':
# create new contract
title = 'Create contract manualy'
form = SaveContract()
else:
# edit contract
title = 'Edit contract -- ' + _id
doc = contracts.find_one({'_id': ObjectId(_id)})
if doc is None:
web_logging.error('cotract not found, _id= {}'.format(_id))
flash('cotract not found, _id= {}'.format(_id))
return redirect(url_for('list_contracts'))
contract = Contract()
contract.done_time = kyiv_tz.localize(datetime.now())
contract.user = current_user.username
form = SaveContract(**contract.to_form(doc))
if request.method == 'POST':
if form.validate_on_submit():
contract = Contract()
contract.from_form(form.data)
if _id == 'new':
result = contract.mongo_insert()
flash("inserted doc={}, _id={}".format(contract.as_dict(),
result))
else:
result = contract.mongo_update(_id)
flash("modified_count= {}, updated doc= {}, ".format(
result, contract.as_dict()))
return redirect(url_for('list_contracts'))
return render_template('create_edit_contract.html', title=title, form=form)
def create_contract(_id='new'):
"""
create new contract or edit existing
:param _id:
:return:
"""
# if request.method == 'GET':
if _id == 'new':
# create new contract
title = 'Create contract manualy'
form = SaveContract()
else:
# edit contract
title = 'Edit contract -- ' + _id
doc = contracts.find_one({'_id': ObjectId(_id)})
if doc is None:
web_logging.error('cotract not found, _id= {}'.format(_id))
flash('cotract not found, _id= {}'.format(_id))
return redirect(url_for('list_contracts'))
contract = Contract()
contract.done_time = kyiv_tz.localize(datetime.now())
contract.user = current_user.username
form = SaveContract(**contract.to_form(doc))
if request.method == 'POST':
if form.validate_on_submit():
contract = Contract()
contract.from_form(form.data)
if _id == 'new':
result = contract.mongo_insert()
flash("inserted doc={}, _id={}".format(contract.as_dict(), result))
else:
result = contract.mongo_update(_id)
flash("modified_count= {}, updated doc= {}, ".format(result, contract.as_dict()))
return redirect(url_for('list_contracts'))
return render_template('create_edit_contract.html', title=title, form=form)
def test_3parse_announcement_ovdp(self, requests_fun):
result = announcement_ovdp()
result = [{**doc, **{'time': kyiv_tz.localize(datetime(2017, 3, 19, 19, 32, 34, 319882))}}
for doc in result] # replace today date on date from etalon
announcement_ovdp_data = [{**doc, **{'headline': doc['headline'] + ' in a same_date', 'flag': 'same_date'}}
if doc['time_auction'].date() == datetime.today().date() else doc
for doc in self.announcement_ovdp_data]
# print(announcement_ovdp_data, result)
self.assertEqual(announcement_ovdp_data, result,
'error in parse_announcement_ovdp')
# mongo tests
current_time = datetime.utcnow()
mongo_result = parent([(minfin.announcement_ovdp, ), ], 'news')
self.assertEqual(mongo_result, (len(self.announcement_ovdp_data), 0), 'announcement_ovdp wrong return')
self.assertEqual(news.find({}).count(), len(self.announcement_ovdp_data), 'wrong news number in DB')
test_doc = news.find_one({'href_announce': 'http://www.minfin.gov.ua/uploads/redactor/files/48-52%20оголошення-сайт.doc'})
if datetime.today().date() != datetime(2017, 3, 21).date():
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017', 'quick check NAME of doc')
else:
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017 in a same_date', 'quick check NAME of doc')
self.assertEqual(test_doc['time_auction'], datetime(2017, 3, 21, 0, 0), 'time_auction should be same')
self.assertAlmostEqual(test_doc['time'], current_time, delta=timedelta(seconds=1),
msg='time should be same as insert data')
# mongo check duplicate
mongo_result = parent([(minfin.announcement_ovdp, ), ], 'news')
self.assertEqual(mongo_result, (0, len(self.announcement_ovdp_data)), 'announcement_ovdp wrong return')
self.assertEqual(news.find({}).count(), len(self.announcement_ovdp_data), 'wrong news number in DB')
test_doc = news.find_one({'href_announce': 'http://www.minfin.gov.ua/uploads/redactor/files/48-52%20оголошення-сайт.doc'})
if datetime.today().date() != datetime(2017, 3, 21).date():
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017', 'quick check NAME of doc')
else:
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017 in a same_date', 'quick check NAME of doc')
self.assertEqual(test_doc['time_auction'], datetime(2017, 3, 21, 0, 0), 'time_auction should be same')
self.assertAlmostEqual(test_doc['time'], current_time, delta=timedelta(seconds=1),
msg='time should be same as INSERT data')
def test_3parse_announcement_ovdp(self, requests_fun):
result = announcement_ovdp()
result = [{
**doc,
**{
'time': kyiv_tz.localize(
datetime(2017, 3, 19, 19, 32, 34, 319882))
}
} for doc in result] # replace today date on date from etalon
announcement_ovdp_data = [{
**doc,
**{
'headline': doc['headline'] + ' in a same_date',
'flag': 'same_date'
}
} if doc['time_auction'].date() == datetime.today().date() else doc
for doc in self.announcement_ovdp_data]
# print(announcement_ovdp_data, result)
self.assertEqual(announcement_ovdp_data, result,
'error in parse_announcement_ovdp')
# mongo tests
current_time = datetime.utcnow()
mongo_result = parent([
(minfin.announcement_ovdp, ),
], 'news')
self.assertEqual(mongo_result, (len(self.announcement_ovdp_data), 0),
'announcement_ovdp wrong return')
self.assertEqual(
news.find({}).count(), len(self.announcement_ovdp_data),
'wrong news number in DB')
test_doc = news.find_one({
'href_announce':
'http://www.minfin.gov.ua/uploads/redactor/files/48-52%20оголошення-сайт.doc'
})
if datetime.today().date() != datetime(2017, 3, 21).date():
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017',
'quick check NAME of doc')
else:
self.assertEqual(test_doc['headline'],
'OVDP 21.03.2017 in a same_date',
'quick check NAME of doc')
self.assertEqual(test_doc['time_auction'], datetime(2017, 3, 21, 0, 0),
'time_auction should be same')
self.assertAlmostEqual(test_doc['time'],
current_time,
delta=timedelta(seconds=1),
msg='time should be same as insert data')
# mongo check duplicate
mongo_result = parent([
(minfin.announcement_ovdp, ),
], 'news')
self.assertEqual(mongo_result, (0, len(self.announcement_ovdp_data)),
'announcement_ovdp wrong return')
self.assertEqual(
news.find({}).count(), len(self.announcement_ovdp_data),
'wrong news number in DB')
test_doc = news.find_one({
'href_announce':
'http://www.minfin.gov.ua/uploads/redactor/files/48-52%20оголошення-сайт.doc'
})
if datetime.today().date() != datetime(2017, 3, 21).date():
self.assertEqual(test_doc['headline'], 'OVDP 21.03.2017',
'quick check NAME of doc')
else:
self.assertEqual(test_doc['headline'],
'OVDP 21.03.2017 in a same_date',
'quick check NAME of doc')
self.assertEqual(test_doc['time_auction'], datetime(2017, 3, 21, 0, 0),
'time_auction should be same')
self.assertAlmostEqual(test_doc['time'],
current_time,
delta=timedelta(seconds=1),
msg='time should be same as INSERT data')