def get_history():
"""Download historical vessel schedules."""
src_dir = get_src_dir_path(__file__)
data_dir = src_dir.parent / "data"
start_date = datetime.date(year=2020, month=1, day=1)
end_date = datetime.date(year=2020, month=1, day=10)
time_delta = datetime.timedelta(days=1)
file_paths = []
jocasta = Spider()
while start_date <= end_date:
time.sleep(random.randrange(0, 5, 1))
file_path = jocasta.crawl(data_dir, date=start_date)
file_paths.append(file_path)
start_date += time_delta
for file_path in file_paths:
if file_path.endswith('.xlsx'):
print("unhandled xlsx file!")
elif file_path.endswith('.pdf'):
tables = Wrangler.parse_pdf(file_path)
else:
raise ValueError(f"Unexpected file type: {file_path}")
return None
def test_wrangle_success(self):
with open('./tests/fixtures/test_config.yaml') as file:
config = yaml.load(file, Loader=yaml.FullLoader)
Wrangler(config=config).wrangle()
with open('./tests/fixtures/test_output.csv', 'r') as output:
with open('./tests/fixtures/wrangler_success_output.csv', 'r') as fixture:
# Assert output file matches expected
diff = set(output).difference(fixture)
self.assertEqual(len(diff), 0)
os.remove('./tests/fixtures/test_output.csv')
def get(self, position_index):
""" Return the health of the collateral for a loan, given its loan number."""
w = Wrangler(
config=config,
web3_client=w3,
current_net=CURRENT_NET
)
health, errors = w.get_loan_health(position_index)
if len(errors):
abort(400, {"error": errors})
return { 'data': health }, 201
def post(self):
""" Approve a loan request."""
w = Wrangler(
config=config,
web3_client=w3,
current_net=CURRENT_NET
)
loan, approval, errors = w.approve_loan(request.get_json(force=True))
if len(errors):
abort(400, {"error": errors})
return { 'data': loan, 'approval': approval }, 201
def get(self, prover, txHash):
""" Return the health of the collateral for a loan, given its loan number."""
w = Wrangler(
config=config,
web3_client=w3,
current_net=CURRENT_NET
)
is_valid_sender = w.is_valid_protocol_transaction_sender(prover, txHash)
print('\n\nis_valid_sender: {0}\n\n'.format(is_valid_sender))
if not is_valid_sender:
abort(400)
return {}, 200
def main():
"""Main spider program processes the latest vessel schedule."""
src_dir = get_src_dir_path(__file__)
data_dir = src_dir.parent / "data"
# TODO: testing on Nov 11, 2020
file_path = Spider().crawl(data_dir, date=datetime.date(year=2020, month=11, day=11))
# file_path = Spider().crawl(data_dir)
if file_path.endswith('.xlsx'):
print("unhandled xlsx file!")
elif file_path.endswith('.pdf'):
tables = Wrangler.parse_pdf(file_path)
breakpoint()
else:
raise ValueError(f"Unexpected file type: {file_path}")
return None
correct += 1
accuracy = (correct / float(len(shell.y_test))) * 100.0
return accuracy
"""Test Shell on normal test data and classifiers"""
classifier = KNNClassifier(k=5)
iris = datasets.load_iris()
shell = Shell(iris.data, iris.target, classifier)
shell.cross_val_custom(10)
# wrangler holds all three datasets as member variables
w = Wrangler()
# cars dataset
shell_cars = Shell(w.car_data.data.values, w.car_data.targets.values, classifier)
shell_cars.cross_val_custom(10)
# diabetes dataset
shell_d = Shell(w.diabetes_data.data.values, w.diabetes_data.targets.values, classifier)
shell_d.cross_val_custom(10)
# mpg dataset
shell_m = Shell(w.mpg_data.data.values, w.mpg_data.targets.values, classifier)
shell_m.cross_val_custom(10, 3)
# ins are income, outs are spends, unless they co-occur, in which case, like kind or self transfer
# keep track of the lots as they dissolve
import copy
import json
from config import *
import os
from prices import prices
from hashlib import sha3_256 as sha
from config import logger
from wrangler import Wrangler
anomalies = []
target_currencies = ['ZET']
wrangler = Wrangler()
#queues = { 'BTC': [], 'ETH': [], 'DASH': [], 'BCH': [] }
# queues = {}
# balances = {}
#
# my_addresses = []
def initialize():
global queues
global balances
global cost_bases
global my_addresses
global tx_id
global spend_queues