def __init__(self,
secondaryDataFilename,
doNotPrintOutput=False,
isVerbose=False):
'''
Constructs an instance of Notifyer which computes the secndary data.
:param filename: name of the file to write in.
:param doNotPrintOutput: used to indicate that the computed secondary data must not be printed
in the console. Used when C2 is started in RT mode with a duration specified. In this
case, a count down string is output in the console and this would interfere with
the output of secondary data.
:param isVerbose: if True, outputs received data in console
'''
#creating the output csv file and initializing it with the column titles
self.archiver = Archiver(secondaryDataFilename,
Archiver.SECONDARY_DATA_CSV_ROW_HEADER,
isVerbose=False)
self.doNotPrintOutput = doNotPrintOutput
self.isVerbose = isVerbose
self.criterion = PriceVolumeCriterion()
self.isOneSecondIntervalReached = False
self.lastSecBeginTimestamp = 0
self.lastSecEndTimestamp = 0
self.lastSecVolume = 0
self.lastSecPriceVolumeTotal = 0
self.lastSecTradeNumber = 0
print('Time\t\tTrades\t\tVolume\t\tPrice')
def testUpdateRealTimeAfterOutputfileClose(self):
'''
Test the case which happens sometimes when C2 is started in mode RT for a specified duration.
When the duration is reached, all the Observers, namely the Archivers, are closed
preamptively, which sometimes causes an exception due to a tentative to write the last
received data into an already closed file. Now, the exception is caught and handled by printing
a message.
:return:
'''
csvFileName = "test.csv"
archiver = Archiver(csvFileName,
Archiver.PRIMARY_DATA_CSV_ROW_HEADER,
isVerbose=False)
archiver.update((1530201849627, 0.100402, 6103.0))
archiver.update((1530201851230, 0.03, 6103.99))
archiver.close()
savedStdout = sys.stdout
sys.stdout = capturedStdout = StringIO()
archiver.update((1530201851230, 6103.99, 0.03))
sys.stdout = savedStdout
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(['IDX\tTIMESTAMP (MS)\tVOLUME\tPRICE'],
next(csvReader))
self.assertEqual(['1\t1530201849627\t0.100402\t6103.00'],
next(csvReader))
self.assertEqual(['2\t1530201851230\t0.030000\t6103.99'],
next(csvReader))
os.remove(csvFileName)
self.assertEqual(
'Last real time data received after closing test.csv. Consequence: (1530201851230, 6103.99, 0.03) not saved/processed !\n',
capturedStdout.getvalue())
def testInitCloseCycle(self):
csvFileName = "test.csv"
archiver = Archiver(csvFileName, isVerbose=False)
archiver.close()
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(Archiver.CSV_ROW_HEADER, next(csvReader))
os.remove(csvFileName)
def __init__(self, secondaryDataFilename, isVerbose):
'''
Constructs an instance of Notifyer which computes the secndary data.
:param filename: name of the file to write in.
:param isVerbose: if True, outputs received data in console
'''
#creating the output csv file and initializing it with the column titles
self.archiver = Archiver(secondaryDataFilename, isVerbose=False)
self.isVerbose = isVerbose
self.criterion = PriceVolumeCriterion()
def testInitCloseCycle(self):
csvFileName = "test.csv"
archiver = Archiver(csvFileName,
Archiver.PRIMARY_DATA_CSV_ROW_HEADER,
isVerbose=False)
archiver.close()
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(['IDX\tTIMESTAMP (MS)\tVOLUME\tPRICE'],
next(csvReader))
os.remove(csvFileName)
def testInitCloseCycleOverwritingExistingFile(self):
#creating a csv file which will be overwritten
csvFileName = "test.csv"
with open(csvFileName, 'w') as csvFile:
csvWriter = csv.writer(csvFile)
csvWriter.writerow(DUMMY_HEADER)
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(DUMMY_HEADER, next(csvReader))
#now, instanciating the Archiver and checking the csv file was overwritten
archiver = Archiver(csvFileName, isVerbose=False)
archiver.close()
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(Archiver.CSV_ROW_HEADER, next(csvReader))
os.remove(csvFileName)
def testUpdate(self):
csvFileName = "test.csv"
archiver = Archiver(csvFileName, isVerbose=False)
archiver.update((1530201849627, 6103.0, 0.100402))
archiver.update((1530201851230, 6103.99, 0.03))
archiver.close()
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(Archiver.CSV_ROW_HEADER, next(csvReader))
self.assertEqual(
['1', '1530201849627', '6103.0', '0.100402', '-1', '-1', '-1'],
next(csvReader))
self.assertEqual(
['2', '1530201851230', '6103.99', '0.03', '-1', '-1', '-1'],
next(csvReader))
os.remove(csvFileName)
def testUpdateRealTime(self):
csvFileName = "test.csv"
archiver = Archiver(csvFileName,
Archiver.PRIMARY_DATA_CSV_ROW_HEADER,
isVerbose=False)
archiver.update((1530201849627, 0.100402, 6103.0))
archiver.update((1530201851230, 0.03, 6103.99))
archiver.close()
with open(csvFileName, 'r') as csvFile:
csvReader = csv.reader(csvFile)
self.assertEqual(['IDX\tTIMESTAMP (MS)\tVOLUME\tPRICE'],
next(csvReader))
self.assertEqual(['1\t1530201849627\t0.100402\t6103.00'],
next(csvReader))
self.assertEqual(['2\t1530201851230\t0.030000\t6103.99'],
next(csvReader))
os.remove(csvFileName)
class SecondaryDataAggregator(Observer):
'''
This class ...
:seqdiag_note Implements the Observer part in the Observable design pattern. Each tima its update(data) method is called, it adds this data to the current secondary aggreagated data and sends the secondary data when appropriate to the Criterion calling its check() method.
'''
def __init__(self, secondaryDataFilename, isVerbose):
'''
Constructs an instance of Notifyer which computes the secndary data.
:param filename: name of the file to write in.
:param isVerbose: if True, outputs received data in console
'''
#creating the output csv file and initializing it with the column titles
self.archiver = Archiver(secondaryDataFilename, isVerbose=False)
self.isVerbose = isVerbose
self.criterion = PriceVolumeCriterion()
def update(self, data):
recordIndex = ''
if len(data) == 4:
# data comming from archive file (mode simulation)
recordIndex, timestampMilliSec, priceFloat, volumeFloat = data
else:
# data comming from exchange (mode real time)
timestampMilliSec, priceFloat, volumeFloat = data
secondaryData = False
while not secondaryData:
secondaryData = self.aggregateSecondaryData(
timestampMilliSec, priceFloat, volumeFloat)
#calling the criterion to check if it should raise an alarm
criterionData = self.criterion.check(secondaryData)
# sending the secondary data to the archiver so that the sd are written in the
# sd file to enable viewing them in a price/volume chart. Note that the archiver
# takes care of implementing the secondary data record index.
self.archiver.update(secondaryData + criterionData)
if self.isVerbose:
print("SecondaryDataAggregator: {} {} {} {}".format(
recordIndex, timestampMilliSec, priceFloat, volumeFloat))
SeqDiagBuilder.recordFlow(
) # called to build the sequence diagram. Can be commented out later ...
def aggregateSecondaryData(self, timestampMilliSec, priceFloat,
volumeFloat):
'''
This method is called each time primary data are received. It aggregates the passed
primary data and returns None if the aggregation interval (typically 1 second9 is not
reached or the tuple timestampMilliSec, priceFloat, volumeFloat once data
for the aggregation interval was reached.
:seqdiag_note method to be implemented by Philippe
:param timestampMilliSec:
:param priceFloat:
:param volumeFloat:
:return: timestampMilliSec, priceFloat, volumeFloat
'''
SeqDiagBuilder.recordFlow(
) # called to build the sequence diagram. Can be commented out later ...
return timestampMilliSec, priceFloat, volumeFloat # temporally returning silly value !
def close(self):
'''
Called when the observed object is
closed and has stopped notifying all the
object's observers of the change.
'''
self.archiver.close()
print('SecondaryDataAggregator closed')
def start(self, commandLineArgs=None):
'''
Start the data stream.
:param commandLineArgs: used only for unit testing only
:return: error message if relevant
'''
isUnitTestMode = False
if commandLineArgs == None:
#here, we are not in unit test mode and we collect the parms entered
#by the user on the command line
commandLineArgs = sys.argv[1:]
else:
isUnitTestMode = True
executionMode, durationStr, primaryFileName, secondaryFileName, isVerbose = self.getCommandLineArgs(commandLineArgs)
localNow = arrow.now(LOCAL_TIME_ZONE)
if executionMode.upper() == 'R':
#C2 executing in real time mode ...
tradingPair = 'BTCUSDT'
print('Starting the Binance aggregate trade stream for pair {}. Type any key to stop the stream ...'.format(
tradingPair))
self.datasource = BinanceDatasource(tradingPair)
dateTimeStr = localNow.format(self.DATE_TIME_FORMAT_ARROW)
#adding an Archiver to store on disk the primary data
self.primaryDataFileName = self.buildPrimaryFileName(primaryFileName, dateTimeStr)
self.datasource.addObserver(Archiver(self.primaryDataFileName, Archiver.PRIMARY_DATA_CSV_ROW_HEADER, isVerbose))
#adding an Observer to compute the secondary data, store them on disk and send
#them to the Criterion
csvSecondaryDataFileName = "{}-{}.csv".format(secondaryFileName, dateTimeStr)
#forcing isVerbose to False to avoid interfering with Archiver verbosity !
self.datasource.addObserver(SecondaryDataAggregator(secondaryDataFilename=csvSecondaryDataFileName, doNotPrintOutput=durationStr != None, isVerbose=False))
try:
self.datasource.startDataReception()
except Exception as e:
print('\nERROR - Connection with RT datasource could not be established.\n')
self.stop()
sys.exit(1)
counter = None
if durationStr:
counter = ThreadedTimeCounter(ThreadedTimeCounter.MODE_COUNT_DOWN, \
intervalSecond=1, \
durationSecond=self.getDurationSeconds(durationStr), \
client=self)
if not isUnitTestMode or durationStr != None:
#here, we are not in unit test mode and we have to wait for the user to
#stop receiving the real time data
if counter:
counter.start()
while not self.stopped:
if not os.name == 'posix':
import msvcrt # only ok on Windows !
if msvcrt.kbhit():
if counter:
counter.stop()
print('\nStopping the stream ...')
else:
print('Stopping the stream ...')
self.stop()
sys.exit(0) # required for the program to exit !
else:
#C2 executing in simulation mode ...
if primaryFileName == self.DEFAULT_PRIMARY_FILENAME:
errorMsg = "ERROR - in simulation mode, a primary file name must be provided !"
print(errorMsg)
return errorMsg
csvSecondaryDataFileName = self.buildSecondaryFileNameFromPrimaryFileName(primaryFileName,
secondaryFileName)
try:
self.datasource = ArchivedDatasource(primaryFileName)
except FileNotFoundError as e:
errorMsg = "ERROR - specified file {} not found !".format(e.filename)
print(errorMsg)
return errorMsg
print('Starting C2 in simulation mode on primary data file {}.'.format(
primaryFileName))
self.datasource.addObserver(SecondaryDataAggregator(csvSecondaryDataFileName, isVerbose))
self.datasource.processArchivedData()
class SecondaryDataAggregator(Observer):
'''
This class ...
:seqdiag_note Implements the Observer part in the Observable design pattern. Each tima its update(data) method is called, it adds this data to the current secondary aggreagated data and sends the secondary data when appropriate to the Criterion calling its check() method.
'''
def __init__(self,
secondaryDataFilename,
doNotPrintOutput=False,
isVerbose=False):
'''
Constructs an instance of Notifyer which computes the secndary data.
:param filename: name of the file to write in.
:param doNotPrintOutput: used to indicate that the computed secondary data must not be printed
in the console. Used when C2 is started in RT mode with a duration specified. In this
case, a count down string is output in the console and this would interfere with
the output of secondary data.
:param isVerbose: if True, outputs received data in console
'''
#creating the output csv file and initializing it with the column titles
self.archiver = Archiver(secondaryDataFilename,
Archiver.SECONDARY_DATA_CSV_ROW_HEADER,
isVerbose=False)
self.doNotPrintOutput = doNotPrintOutput
self.isVerbose = isVerbose
self.criterion = PriceVolumeCriterion()
self.isOneSecondIntervalReached = False
self.lastSecBeginTimestamp = 0
self.lastSecEndTimestamp = 0
self.lastSecVolume = 0
self.lastSecPriceVolumeTotal = 0
self.lastSecTradeNumber = 0
print('Time\t\tTrades\t\tVolume\t\tPrice')
def update(self, data):
recordIndex = ''
if len(data) == 4:
# data comming from archive file (mode simulation)
recordIndexStr, timestampMilliSecStr, volumeFloatStr, priceFloatStr = data
else:
# data comming from exchange (mode real time)
timestampMilliSecStr, volumeFloatStr, priceFloatStr = data
timestampMilliSec = int(timestampMilliSecStr)
if self.lastSecBeginTimestamp == 0:
startTimestamp = self.calculateStartTimestamp(timestampMilliSec)
self.lastSecBeginTimestamp = startTimestamp
self.lastSecEndTimestamp = startTimestamp + 1000
self.lastSecTradeNumber = 0
self.lastSecVolume = 0
self.lastSecPriceVolumeTotal = 0
else:
priceFloat = float(priceFloatStr)
volumeFloat = float(volumeFloatStr)
lastNotifiedPriceFloat = priceFloat
lastNotifiedVolumeFloat = volumeFloat
secondary_data = self.aggregateSecondaryData(
timestampMilliSec, priceFloat, volumeFloat)
if not secondary_data:
return
catchUpSdTimestamp, catchUpSdTradesNumber, catchUpSdVolumeFloat, sdPricefloat = secondary_data
# calling the criterion to check if it should raise an alarm
criterionData = self.criterion.check(data)
if self.isOneSecondIntervalReached:
# sending the secondary data to the archiver so that the sd are written in the
# sd file to enable viewing them in a price/volume chart. Note that the archiver
# takes care of implementing the secondary data record index.
# self.archiver.update((sdTimestamp, sdTradesNumber, sdVolumeFloat, sdPricefloat) + criterionData)
if sdPricefloat and sdPricefloat > 0:
self.storeAndPrintSecondaryData(sdPricefloat,
catchUpSdTimestamp,
catchUpSdTradesNumber,
catchUpSdVolumeFloat)
wasTimeCaughtUp = False
catchUpSdTradesNumber = 0
catchUpSdVolumeFloat = 0
catchUpSdPricefloat = 0
while int(timestampMilliSec /
1000) * 1000 > self.lastSecEndTimestamp:
wasTimeCaughtUp = True
catchUpSdTimestamp = self.lastSecEndTimestamp
if self.lastSecVolume > 0:
# calculating the price to display for the 0 volume catchup secondary data
catchUpSdPricefloat = self.lastSecPriceVolumeTotal / self.lastSecVolume
self.storeAndPrintSecondaryData(catchUpSdPricefloat,
catchUpSdTimestamp,
catchUpSdTradesNumber,
catchUpSdVolumeFloat)
else:
# happens if no transaction were yet processed
catchUpSdPricefloat = 0
self.lastSecBeginTimestamp += 1000
self.lastSecEndTimestamp += 1000
if wasTimeCaughtUp:
self.lastSecVolume = lastNotifiedVolumeFloat
self.lastSecPriceVolumeTotal = lastNotifiedVolumeFloat * lastNotifiedPriceFloat
self.lastSecTradeNumber = 1
self.lastSecBeginTimestamp += 1000
self.lastSecEndTimestamp += 1000
self.isOneSecondIntervalReached = False
SeqDiagBuilder.recordFlow(
) # called to build the sequence diagram. Can be commented out later ...
def storeAndPrintSecondaryData(self, sdPricefloat, sdTimestamp,
sdTradesNumber, sdVolumeFloat):
self.archiver.update(
(sdTimestamp, sdTradesNumber, sdVolumeFloat, sdPricefloat))
if not self.doNotPrintOutput:
timeHHMMSS = datetime.fromtimestamp(sdTimestamp /
1000).strftime('%H:%M:%S')
print("{0}\t{1}\t\t{2:.6f}\t{3:.2f}".format(
timeHHMMSS, sdTradesNumber, sdVolumeFloat, sdPricefloat))
def calculateStartTimestamp(self, timestampMilliSec):
'''
:param timestampMilliSec:
:return:
'''
roundedTimestampMilliSec = int(timestampMilliSec / 1000) * 1000
return roundedTimestampMilliSec + 1000
def aggregateSecondaryData(self, timestampMilliSec, priceFloat,
volumeFloat):
'''
This method is called each time primary data are received. It aggregates the passed
primary data and returns None if the aggregation interval (typically 1 second9 is not
reached or the tuple timestampMilliSec, priceFloat, volumeFloat once data
for the aggregation interval was reached.
:seqdiag_note method to be implemented by Philippe
:param timestampMilliSec:
:param priceFloat:
:param volumeFloat:
:return: timestampMilliSec, priceFloat, volumeFloat
'''
SeqDiagBuilder.recordFlow(
) # called to build the sequence diagram. Can be commented out later ...
if timestampMilliSec >= self.lastSecBeginTimestamp and timestampMilliSec < self.lastSecEndTimestamp:
# here, the current primary data ts is within the current second frame
self.lastSecTradeNumber += 1
self.lastSecVolume += volumeFloat
self.lastSecPriceVolumeTotal += priceFloat * volumeFloat
return None, None, None, None # since we are still within the current second time frame !
elif timestampMilliSec >= self.lastSecEndTimestamp:
if timestampMilliSec < self.lastSecEndTimestamp + 1000:
# here, the current primary data ts is within the next second frame
lastSecBeginTimestamp = self.lastSecBeginTimestamp
lastSecTradeNumber = self.lastSecTradeNumber
lastSecVolume = self.lastSecVolume
if self.lastSecVolume > 0:
lastSecAvgPrice = self.lastSecPriceVolumeTotal / self.lastSecVolume
else:
# happens if no transaction were yet processed, at start of RT or simulation
lastSecAvgPrice = 0
self.lastSecTradeNumber = 1
self.lastSecVolume = volumeFloat
self.lastSecPriceVolumeTotal = priceFloat * volumeFloat
self.lastSecBeginTimestamp += 1000
self.lastSecEndTimestamp += 1000
self.isOneSecondIntervalReached = True
return lastSecBeginTimestamp, lastSecTradeNumber, lastSecVolume, lastSecAvgPrice
else:
# here, the current primary data ts is after the next second frame
lastSecBeginTimestamp = self.lastSecBeginTimestamp
lastSecTradeNumber = self.lastSecTradeNumber
lastSecVolume = self.lastSecVolume
if self.lastSecVolume > 0:
lastSecAvgPrice = self.lastSecPriceVolumeTotal / self.lastSecVolume
else:
# happens if no transaction were yet processed, at start of RT or simulation
lastSecAvgPrice = 0
self.isOneSecondIntervalReached = True
return lastSecBeginTimestamp, lastSecTradeNumber, lastSecVolume, lastSecAvgPrice
else:
# here, the current primary data ts is before the current second frame. This
# situation occurs at the very start of receiving the RT data or when processing
# the first lines of the primary data input file in simulation mode when the ts
# of those lines is before the ts calculated by the calculateStartTimestamp() method.
# In this case, the line must simply be ignored.
return None
def close(self):
'''
Called when the observed object is
closed and has stopped notifying all the
object's observers of the change.
'''
self.archiver.close()
print('SecondaryDataAggregator closed')