def test_integration(self):
loader = CSVTradeLoader("trade_file", source=os.path.dirname(__file__))
loader.run()
with open(os.path.join(tempfile.gettempdir(), loader.error_file),
'r') as f:
expect = 'Quantity,Price,Action,Trade Date,Instrument,Fund,TradeType,Trade ID,Msg Type,Status\n\n'
expect += "333,112,Buy,2015-03-03,BAC.N,Fund1,,FF235,New,'TradeType': 'Valid values for TradeType are [Vanilla]'"
self.assertEquals(
expect.replace("\n", "").replace("\r", ""),
f.read().replace("\n", "").replace("\r", ""))
def test_write_csv(self):
loader = CSVTradeLoader("some_csv")
headers = ['F1', 'F2']
data = [{'F1': 'V1'}, {'F1': 'V1', 'F2': 'V2'}]
csv_file = os.path.join(tempfile.gettempdir(), 'some_csv.csv')
loader.write_csv(csv_file, headers, data)
with open(csv_file, 'r') as f:
expect = 'F1,F2\n\nV1,\n\nV1,V2\n'
self.assertEquals(
expect.replace("\n", "").replace("\r", ""),
f.read().replace("\n", "").replace("\r", ""))
def test_load_trade_file(self):
loader = CSVTradeLoader("trade_file", source=os.path.dirname(__file__))
fields, trades = loader.load_trade_file()
expect = [
'Quantity', 'Price', 'Action', 'Trade Date', 'Instrument', 'Fund',
'TradeType', "Trade ID", 'Msg Type'
]
self.assertEquals(expect, fields)
self.assertEquals(3, len(trades))
expect = {
'Action': 'Buy',
'Fund': 'Fund1',
'Instrument': 'BAC.N',
'Price': '22',
'Quantity': '200',
'Trade Date': '2015-03-03',
'TradeType': 'Vanilla',
'Trade ID': 'FF123',
'Msg Type': 'New'
}
self.assertEquals(expect, trades[0])
def build(self, file_name, number, msg_type="new"):
ID_BASE = hash(random.random())
header = default_record.keys()
records = []
trade_ids = None
if msg_type != "new":
s = Session()
trade_ids = [t.trade_id for t in s.query(Trade).limit(number)]
s.close()
for i in range(number):
rec = default_record.copy()
rnd = Decimal("{:5f}".format(random.random()))
rec['Price'] = round(rec['Price'] * rnd * 2, 5)
rec['Quantity'] = round(rec['Quantity'] * rnd * 2 + 1, 0)
rec['Instrument'] = INSTRUMENTS[int(rnd * 100) % len(INSTRUMENTS)]
if trade_ids:
rec['Trade ID'] = trade_ids[i]
else:
rec['Trade ID'] = "{}-{}".format(ID_BASE, i)
rec["Msg Type"] = msg_type
records.append(rec)
CSVTradeLoader.write_csv(file_name, header, records)
def build(self, file_name, number, msg_type ="new"):
ID_BASE = hash(random.random())
header = default_record.keys()
records = []
trade_ids = None
if msg_type != "new":
s = Session()
trade_ids = [t.trade_id for t in s.query(Trade).limit(number)]
s.close()
for i in range(number):
rec = default_record.copy()
rnd = Decimal("{:5f}".format( random.random() ))
rec['Price'] = round(rec['Price'] * rnd*2, 5)
rec['Quantity'] = round(rec['Quantity'] * rnd*2 +1, 0)
rec['Instrument'] = INSTRUMENTS[int(rnd*100)%len(INSTRUMENTS)]
if trade_ids:
rec['Trade ID'] = trade_ids[i]
else:
rec['Trade ID'] = "{}-{}".format(ID_BASE, i)
rec["Msg Type"] = msg_type
records.append(rec)
CSVTradeLoader.write_csv(file_name, header, records)
def run_once(cnt):
# DATA_CACHE.clear()
# import gc
# gc.collect() # don't care about stuff that would be garbage collected properly
# # tr.print_diff()
# # all_objects = muppy.get_objects()
# # print(len(all_objects))
# import objgraph
# growth = objgraph.show_growth(limit=10)
# if growth:
# print ("I leaked{}".format( growth ) )
# print(objgraph.show_most_common_types())
start = time.time()
loader = CSVTradeLoader.create_from_path(file_name)
status = loader.run()
# loaders.append(loader)
print ("{}. Took {}. {}".format(cnt, time.time() - start, status))
def run_once(cnt):
# DATA_CACHE.clear()
# import gc
# gc.collect() # don't care about stuff that would be garbage collected properly
# # tr.print_diff()
# # all_objects = muppy.get_objects()
# # print(len(all_objects))
# import objgraph
# growth = objgraph.show_growth(limit=10)
# if growth:
# print ("I leaked{}".format( growth ) )
# print(objgraph.show_most_common_types())
start = time.time()
loader = CSVTradeLoader.create_from_path(file_name)
status = loader.run()
# loaders.append(loader)
print("{}. Took {}. {}".format(cnt, time.time() - start, status))
def process(self, conn):
bucket = conn.get_bucket(self.bucket)
trade_file_key = bucket.get_key(self.file_name)
if str(trade_file_key.get_metadata('user_upload')).lower() !='true':
return ("Skipping {}. Not user uploaded file".format(self.file_name))
trade_file = os.path.join(tempfile.gettempdir(),self.file_name)
trade_file_key.get_contents_to_filename(trade_file)
loader = CSVTradeLoader.create_from_path(trade_file)
loader.run()
error_file_key = bucket.new_key(loader.error_file)
error_file_key.set_contents_from_filename(loader.error_filepath)
os.remove(trade_file)
os.remove(loader.error_filepath)
return ("Trade file processed. {} filed produced".format( loader.error_file ))
def process(self, conn):
bucket = conn.get_bucket(self.bucket)
trade_file_key = bucket.get_key(self.file_name)
if str(trade_file_key.get_metadata('user_upload')).lower() !='true':
return ("Skipping {}. Not user uploaded file".format(self.file_name))
trade_file = os.path.join(tempfile.gettempdir(),self.file_name)
trade_file_key.get_contents_to_filename(trade_file)
loader = CSVTradeLoader.create_from_path(trade_file)
loader.run()
error_file_key = bucket.new_key(loader.error_file)
error_file_key.set_contents_from_filename(loader.error_filepath)
os.remove(trade_file)
os.remove(loader.error_filepath)
return ("Trade file processed. {} filed produced".format( loader.error_file ))
def test_create_from_path(self):
file_name = '/foo/bar/test.csv'
loader = CSVTradeLoader.create_from_path(file_name)
self.assertEquals('test', loader.file_root)
self.assertEquals('/foo/bar', loader.source)
self.assertEquals('/foo/bar', loader.output)
def test_trade_file(self):
loader = CSVTradeLoader("file_root")
self.assertEquals('file_root.csv', loader.trade_file)
def test_error_filepath(self):
loader = CSVTradeLoader("file_root", output='/foo/bar')
self.assertEquals(os.path.join('/foo/bar', 'file_root_errors.csv'),
loader.error_filepath)
def test_error_file(self):
loader = CSVTradeLoader("file_root", output='/foo/bar')
self.assertEquals(r'file_root_errors.csv', loader.error_file)
def build_csv(cls, file_name):
models = RecordProcessor.model_list()
fields = RecordProcessor.FIELD_MAP
records = cls.get_data_table(fields, models)
CSVTradeLoader.write_csv(file_name, cls.get_headers(models), records)