def get_all_user_data():
all_users = mysql_connector.get_all_users()
for user in all_users:
user_id = str(user[0])
if user_id == '':
continue
user_channels = mysql_connector.get_user_channels(user_id=user_id)
print(user_channels)
for user_channel in user_channels:
channel = user_channel[0]
channel_type = user_channel[1]
last_aggregation = mysql_connector.get_last_aggregation_for_user_channel(
user_id, channel)
if channel_type == "api":
try:
api_scraper.fetch_financial_data(
pre_last_date=last_aggregation,
user_id=user_id,
channel=channel)
except CantAggragateError as e:
print(e)
elif channel_type == "website":
try:
selenium_scraper.fetch_financial_data(
pre_last_date=last_aggregation,
user_id=user_id,
channel=channel)
except CantAggragateError as e:
print(e)
# after getting all channels data, get all statements
statement_handler.get_statments(user_id)
def test_small_aggregation_interval(self):
with self.assertRaises(CantAggragateError):
last_date = datetime.datetime.now()
api_scraper.fetch_financial_data(
pre_last_date=last_date,
user_id="061509949",
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
def test_fetch_transactions_all_params(self):
results = api_scraper.fetch_financial_data(
last_date,
user_id="061509949",
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
self.assertTrue(type(results) == type({}))
self.assertTrue(results.get('balance'))
self.assertTrue(results.get('transactions'))
def test_incorrect_input(self):
# wrong last_date format
with self.assertRaises(ValueError):
api_scraper.fetch_financial_data(
pre_last_date="2019",
user_id="061509949",
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
# wrong last_date type
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(
pre_last_date=2019,
user_id="061509949",
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
# wrong user_id type
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(
pre_last_date=last_date,
user_id=False,
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
# wrong channel type
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(pre_last_date=last_date,
user_id="061509949",
channel=False,
test=True)
# invalid url
with self.assertRaises(Exception):
api_scraper.fetch_financial_data(
pre_last_date=last_date,
user_id="061509949",
channel="https://nonexisting.url.com/senior-test",
test=True)
def on_demand_data_aggregation():
request_json = request.get_json()
user_id = request_json["user_id"]
username = request_json["username"]
last_aggregation = request_json["last_aggregation"]
channel = request_json["channel"]
transactions = []
channel_type = mysql_connector.get_channel_type(channel)
if channel_type != []:
if channel_type == "api":
transactions = api_scraper.fetch_financial_data(pre_last_date=last_aggregation, test=True, user_id=user_id, channel=channel)
elif channel_type == "website":
transactions = selenium_scraper.fetch_financial_data(pre_last_date=last_aggregation, test=True, user_id=user_id, channel=channel)
# if channel type comes back empty from the database, assume it's a scanned statement
else:
transactions = statement_handler.get_statments(user_id)
return str(transactions)
def test_fetch_transactions_missing_params(self):
# no params
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data()
# missing user id
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(
last_date=last_date,
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
test=True)
# missing channel
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(last_date=last_date,
user_id="061509949",
test=True)
# missing last_date
with self.assertRaises(TypeError):
api_scraper.fetch_financial_data(
channel=
"https://qndxqxuz35.execute-api.us-west-2.amazonaws.com/senior-test",
user_id="061509949",
test=True)
def get_user_data():
request_json = request.get_json()
user_id = request_json["user_id"]
username = request_json["username"]
last_aggregation = request_json["last_aggregation"]
user_channels = mysql_connector.get_user_channels(user_id)
response = []
for user_channel in user_channels:
channel = user_channel[0]
channel_type = user_channel[1]
if channel_type == "api":
try:
response.append(api_scraper.fetch_financial_data(pre_last_date=last_aggregation, user_id=user_id, channel=channel))
except CantAggragateError as e:
print(e)
elif channel_type == "website":
try:
response.append(selenium_scraper.fetch_financial_data(pre_last_date=last_aggregation, user_id=user_id, channel=channel))
except CantAggragateError as e:
print(e)
# after getting all channels data, get all statements
response.append(statement_handler.get_statments(user_id))
return str(response)