def doctor(doctorId):
_userId = 1
u = UserDispatcher()
_doctor = u.get(doctorId,False)
_d = Data(_userId)
_hm_hr,_hm_eda,_hm_temp = _d.getHeatMapSessions(True)
return render_template("doctor.html",_hm_hr = _hm_hr,_hm_eda=_hm_eda,_hm_temp= _hm_temp, _patientID = _userId, _doctor = _doctor)
def __init__(self,
logger,
data,
capital_allowed,
live_mode,
default_ticker=None,
default_period=None):
"""
- logger: a Logger instance linked to a bot
- data: a Data instance linked to a bot
- capital_allowed: float, percentage (0-100) of the capital this instance is able to trade with
- live_mode: bool, should we make real orders on the markets or simulate them
- default_ticker: string, the ticker formatted like that: ASSET1/ASSET2, optional
- default_period: string, the period resolution you want to get data in (1m, 5m, 1h...), optional
"""
self.logger = logger
self.data = data
self.capital_allowed = capital_allowed
self.default_ticker = default_ticker
self.default_period = default_period
self.live_mode = live_mode
self.fake_balance = []
self.fake_pnl = []
self.fake_current_price = None
self.fake_current_date = None
self.fake_orders = []
if (self.default_period and self.default_ticker):
self.cache = Data(
f"{self.default_period}-{''.join(self.default_ticker.split('/'))}-cache"
)
else:
self.cache = None
self.client = ccxt.binance(config.get_creds()['binance'])
def doctor_patient(doctorId,patientId):
u = UserDispatcher()
_doctor = u.get(doctorId,False)
_cpatient = u.get(patientId,False)
_d = Data(patientId)
_hm_hr,_hm_eda,_hm_temp = _d.getHeatMapSessions(True)
return render_template("doctor_patient.html",_hm_hr = _hm_hr,_hm_eda=_hm_eda,_hm_temp= _hm_temp, _patientID = patientId, _doctor = _doctor,_cpatient = _cpatient)
def _count_macd(self, date):
"""
Moving Average Convergence/Divergence
:link https://en.wikipedia.org/wiki/MACD
"""
period1, period2, period3 = self.params
if period1 > period2:
# period1 should always be less than period2
period1, period2 = period2, period2
# Obtaining the needed data
Count.count_once(
['EMA(' + str(period1) + ')', 'EMA(' + str(period2) + ')'],
self.ticker,
date=date)
data = Data.get(
['EMA(' + str(period1) + ')', 'EMA(' + str(period2) + ')'],
self.ticker,
date=date)
alpha = 2 / (period3 + 1)
initial_sum = 0
prev_signal = 0
for index, (dtime, ema1, ema2) in enumerate(data):
ema_diff = ema1 - ema2
if index < period3 - 1:
initial_sum += ema_diff
continue
elif index == period3 - 1:
signal = round((initial_sum + ema_diff) / period3)
else:
signal = round(ema_diff * alpha + prev_signal * (1 - alpha))
self.update(dtime, signal)
prev_signal = signal
self.commit()
def __init__(self, name, ticker, period, live_mode, periods_needed=200):
"""
- name: string, the name of the bot
- ticker: string, the ticker formatted like that: ASSET1/ASSET2
- period: string, the period on which the loop will be set, and the resolution of the candles
- live_mode: bool, should we launch the live loop and start trading live
- periods_needed: int, the number of candles you will get every loop, optional
"""
self.live_mode = live_mode
self.name = name
self.ticker = ticker
self.period_text = period
self.periods_needed = periods_needed
self.offset_seconds = 10
if (not self.name in config.get_config()):
print("❌ Cannot instantiate bot: no config entry")
exit(1)
self.config = config.get_config()[self.name]
if (not "capitalAllowed" in self.config):
print("❌ Cannot instantiate bot: no 'capitalAllowed' property")
exit(1)
try:
self.logger = Logger(self.name, live_mode)
except:
print("❌ Cannot connect to the log DB, are you sure it's running?")
raise
if (self.live_mode):
self.data = Data(self.name)
else:
self.data = Data(self.name + "-test")
self.exchange = Exchange(self.logger, self.data, self.config['capitalAllowed'], live_mode, self.ticker, self.period_text)
try:
self.period = period_matching[period]
except:
print("Available periods: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w")
raise
self.logger.log("ℹ️", f"Bot {self.name} started with a period of {period}")
self.logger.log("ℹ️", f"Capital allowed: {self.config['capitalAllowed']}%")
self.setup()
if (self.live_mode):
self.preloop()
def reset(self):
data_instance = Data.init_from_config_dic(config_dict)
self.model = Model(data_instance)
self.T = WARMUP_TIME_SECONDS
# run the warm up period
for t in range(self.T, self.T + 3600, INT_ASSIGN):
self.model.dispatch_at_time(t)
self.T = ANALYSIS_TIME_SECONDS
print("##########################")
print("##########################")
print("End of the warm up time ")
print("##########################")
print("##########################")
def count(indicators, ticker, date=None):
""" Count the requested set of indicators """
indicators = Count.__fix_indicators_list(indicators)
if date is not None and isinstance(date, str):
date = datetime.datetime.strptime(date, '%Y-%m-%d')
for indicator in indicators:
counter = Count.__get_indicator_counter(ticker, indicator)
if date is not None:
# Counting for one particular date
counter.count(date)
else:
# Counting indicator for all trade days
for date in Data.trade_dates(ticker):
counter.count(date)
return None
def _count_sma(self, date):
"""
Simple Moving Average
:link https://en.wikipedia.org/wiki/Moving_average#Simple_moving_average
"""
data = Data.get(['vwap'], self.ticker, date=date, market_hours=True)
period = self.params[0]
prices = []
for dtime, price in data:
prices.append(price)
if len(prices) > period:
del prices[0]
if len(prices) == period:
value = round(sum(prices) / period)
self.update(dtime, value)
self.commit()
def _count_ema(self, date):
"""
Exponential Moving Average
:link https://en.wikipedia.org/wiki/Moving_average#Exponential_moving_average
"""
data = Data.get(['vwap'], self.ticker, date=date, market_hours=True)
period = self.params[0]
alpha = 2 / (period + 1)
initial_sum = 0
prev_value = 0
for index, (dtime, price) in enumerate(data):
if index < period - 1:
initial_sum += price
continue
elif index == period - 1:
value = round((initial_sum + price) / period)
else:
value = round(price * alpha + prev_value * (1 - alpha))
self.update(dtime, value)
prev_value = value
self.commit()
def count_once(indicators, ticker, date=None):
""" Count the requested set of indicators if they were not counted previously """
indicators = Count.__fix_indicators_list(indicators)
# Getting data 1-minute slice from the very middle of the day to check which data presents and which is missing
if date is not None and isinstance(date, str):
date = datetime.datetime.strptime(date, '%Y-%m-%d')
date_mid = tradetime.daymid(date)
missing_indicators = []
data = list(
Data.get(indicators,
ticker,
dtime_from=date_mid,
dtime_to=date_mid,
market_hours=False))
if len(data) == 0:
missing_indicators = list(indicators)
else:
for index, value in enumerate(data[0]):
if index > 0 and value is None:
missing_indicators.append(indicators[index - 1])
if len(missing_indicators) > 0:
Count.count(missing_indicators, ticker, date)
def _count_chance(self, date):
"""
Relative price difference in the next N minutes
"""
data = Data.get(['open', 'vwap'],
self.ticker,
date=date,
market_hours=True,
order='desc')
period = self.params[0]
history = []
index = 0
for dtime, open, vwap in data:
index += 1
if index > period:
index = 1
if len(history) < index:
history.append(vwap)
else:
value = round((history[index - 1] - open) / open * 100, 2)
history[index - 1] = vwap
self.update(dtime, value)
self.commit()
def sessions(_userId): _d = Data(_userId) return _d.getSessions()
class Bot:
"""This class is an instance of a bot.
Every bot has a name, a ticker, and period.
The period can be 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w.
A bot can have a period_needed property that will specify how much past data you want at least
at every loop.
Example: a bot with a time period of 5m and a period_needed of 200 will receive at every loop the 200
last ticker, 1000 minutes.
To implement a bot, you just have to override the compute and setup function. Those two functions will be
called automatically by the timing system.
Compute will receive the last period_needed candles for the selected asset.
The data property is a Data object that allows you to store important and persistant information.
Every important variables or objects should be stored in data, in case the bot is restarted or if the server is down.
The logger property is an instance of Logger. It allows you to log information in the console and in the
database and the Dashboard. If you want to log custom metrics, use logger.custom. You will be able to create
visualizations in Grafana with this logs.
The config property is the dictionnary with the same name as the bot in the config.
The exchange property is an instance of Exchange. It allows you to interact with the actual markets.
The live_mode property indicates if the bot should loop and receive live data. Use it only to test your bot
live. If you want to backtrack or test your algorithm, leave live_mode = False.
When live_mode is False, the logger won't log to the DB, and the exchange actions will be simulated.
"""
def __init__(self, name, ticker, period, live_mode, periods_needed=200):
"""
- name: string, the name of the bot
- ticker: string, the ticker formatted like that: ASSET1/ASSET2
- period: string, the period on which the loop will be set, and the resolution of the candles
- live_mode: bool, should we launch the live loop and start trading live
- periods_needed: int, the number of candles you will get every loop, optional
"""
self.live_mode = live_mode
self.name = name
self.ticker = ticker
self.period_text = period
self.periods_needed = periods_needed
self.offset_seconds = 10
if (not self.name in config.get_config()):
print("❌ Cannot instantiate bot: no config entry")
exit(1)
self.config = config.get_config()[self.name]
if (not "capitalAllowed" in self.config):
print("❌ Cannot instantiate bot: no 'capitalAllowed' property")
exit(1)
try:
self.logger = Logger(self.name, live_mode)
except:
print("❌ Cannot connect to the log DB, are you sure it's running?")
raise
if (self.live_mode):
self.data = Data(self.name)
else:
self.data = Data(self.name + "-test")
self.exchange = Exchange(self.logger, self.data, self.config['capitalAllowed'], live_mode, self.ticker, self.period_text)
try:
self.period = period_matching[period]
except:
print("Available periods: 1m, 3m, 5m, 15m, 30m, 1h, 2h, 3h, 4h, 1d, 1w")
raise
self.logger.log("ℹ️", f"Bot {self.name} started with a period of {period}")
self.logger.log("ℹ️", f"Capital allowed: {self.config['capitalAllowed']}%")
self.setup()
if (self.live_mode):
self.preloop()
def preloop(self):
"""Waits for the selected period to begin. We use UTC time.
"""
while (1):
current_time = datetime.datetime.utcnow()
if (self.period < 60):
if (current_time.minute % self.period == 0 and current_time.second == self.offset_seconds):
self.loop()
elif (self.period <= 4 * 60):
hour_offset = int(self.period / 60)
if (current_time.hour % hour_offset == 0 and current_time.minute == 0 and current_time.second == self.offset_seconds):
self.loop()
elif (self.period <= 1 * 60 * 24):
if (current_time.hour == 0
and current_time.minute == 0
and current_time.second == self.offset_seconds):
self.loop()
else:
if (current_time.weekday() == 0
and current_time.hour == 0
and current_time.minute == 0
and current_time.second == self.offset_seconds):
self.loop()
def loop(self):
"""Once we waited for the period to start, we can loop over the periods. At every period we
call compute with the latest data.
"""
while (1):
current_time = datetime.datetime.utcnow()
self.logger.log("ℹ️", f"Downloading latest data at {current_time}")
data = self.exchange.get_latest_data(self.ticker, self.period_text, self.periods_needed)
self.logger.price(data.iloc[-1]['close'])
self.compute(data)
time.sleep(self.offset_seconds + self.period * 60 - datetime.datetime.now().second)
def backtest(self, start_date, end_date):
self.exchange.init_fake_balance()
self.data.reset()
price = []
date = []
data = self.exchange.get_data(start_date, end_date, self.ticker, self.period_text)
if (data.shape[0] == 0):
self.logger.log("❌", "No data for the given time frame")
for i in range(self.periods_needed, data.shape[0]):
batch = data.iloc[i - self.periods_needed:i]
self.exchange.fake_current_price = batch.iloc[-1]['close']
self.exchange.fake_current_date = batch.iloc[-1]['date']
price.append(batch.iloc[-1]['close'])
date.append(batch.iloc[-1]['date'])
self.compute(batch.copy())
hist = pd.DataFrame()
hist['date'] = date
hist['price'] = price
for order in self.exchange.fake_orders:
hist.loc[hist['date'] == order['date'], 'action'] = order['action']
return (self.exchange.fake_balance, self.exchange.fake_pnl, hist)
def setup(self):
"""To implement. Set the bot variable, instantiate classes... This will be done once before the bot
starts.
"""
pass
def compute(self, data):
"""To implement. Called every period, you have the latest data available. You can here take decisions.
"""
pass
def main():
parser = argparse.ArgumentParser(
description="Simulation of drivers' behavior")
parser.add_argument(
'-f',
'--fleet',
help=
'Fleet sizes to simulate, formatted as comma-separated list (i.e. "-f 250,275,300")'
)
parser.add_argument(
'-m',
'--multiplier',
help=
'Surge multiplier, formatted as comma-separated list (i.e. "-m 1,1.5,2")'
)
parser.add_argument('-b', '--bonus', type=int, help='Bonus')
parser.add_argument('-d', '--demand', help='Percent false demand ')
parser.add_argument(
'-k',
'--know',
help=
'Percent knowing fare, formatted as comma-separated list (i.e. "-m 1,1.5,2") '
)
parser.add_argument(
'-p',
'--pro',
help=
'Percent pro drivers, formatted as comma-separated list (i.e. "-m 1,1.5,2") '
)
parser.add_argument('-r',
'--replications',
help='number of times to run the simulation')
parser.add_argument('-bb', '--beta', help='BETA')
parser.add_argument('-b_policy', '--bonus_policy', help='bonus per zone ')
parser.add_argument('-budget', '--budget', help='budget ')
args = parser.parse_args()
# TODO: argpars should get the bonus policy as input
data_instance = Data()
if args.fleet:
fleet_sizes = [int(args.fleet)]
else:
fleet_sizes = data_instance.FLEET_SIZE
if args.multiplier:
# surge = args.multiplier
surges = [float(x) for x in args.multiplier.split(',')]
else:
surges = [data_instance.SURGE_MULTIPLIER]
if args.know:
perc_know = [float(args.know)]
else:
perc_know = [data_instance.PERCE_KNOW]
if args.bonus:
bonus = args.bonus
else:
bonus = data_instance.BONUS
if args.beta:
beta = float(args.beta)
else:
beta = configs_dict["BETA"]
if args.pro:
pro_share = [float(x) for x in args.pro.split(',')]
else:
pro_share = [data_instance.PRO_SHARE]
if args.demand:
percent_false_demand = float(args.demand)
else:
percent_false_demand = data_instance.PERCENT_FALSE_DEMAND
if args.replications:
n_rep = int(args.replications)
else:
n_rep = 1
if args.bonus_policy:
bonus_policy = args.bonus_policy
else:
bonus_policy = data_instance.BONUS_POLICY
if args.budget:
budget = args.budget
else:
budget = data_instance.BUDGET
# output_path = "./Outputs/avg_fare_info/" + str(beta) + "/"
for fleet_size in fleet_sizes:
for surge in surges:
for perc_k in perc_know:
for pro_s in pro_share:
for repl in range(n_rep):
output_path = "./Outputs/avg_fare_info/" + str(budget) + "_" + str(bonus_policy) + "_" + \
str(datetime.datetime.now()).split('.')[0] + "/"
if not os.path.exists(output_path):
os.makedirs(output_path)
print("iteration number ", repl)
print('Fleet size is {f}'.format(f=fleet_size))
print('Surge is {}'.format(surge))
print('Percentage knowing fares is {}'.format(perc_k))
print('Percentage of professional drivers {}'.format(
pro_s))
data_instance.FLEET_SIZE = fleet_size
data_instance.PRO_SHARE = pro_s
data_instance.SURGE_MULTIPLIER = surge
data_instance.BONUS = bonus
data_instance.PERCENT_FALSE_DEMAND = percent_false_demand
data_instance.PERCE_KNOW = perc_k
m = Model(data_instance, configs_dict, beta)
# start time
stime = time.time()
# # dispatch the system for T_TOTAL seconds, at the interval of INT_ASSIGN
for T in range(data_instance.WARMUP_TIME_SECONDS,
data_instance.T_TOTAL_SECONDS,
data_instance.INT_ASSIGN):
m.dispatch_at_time(T)
print('Total drivers: ', len(m.vehicles))
print(
'# of Pro drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
]))
print(
'# of naive drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.NAIVE
]))
print(
'# of inexperienced drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.INEXPERIENCED
]))
# end time
etime = time.time()
# run time of this simulation
runtime = etime - stime
print("The run time was {runtime} minutes ".format(
runtime=runtime / 60))
report = m.get_service_rate_per_zone()
# So that it doesn't save a file with 1.5.py, rather 15.py
ss = str(surge).split('.')
ss = ''.join(ss)
report.to_csv(output_path + "report for fleet size " +
str(fleet_size) + " surge " + str(ss) +
"fdemand= " + str(percent_false_demand) +
"perc_k " + str(perc_k) + "pro_s " +
str(pro_s) + " repl" + str(repl) +
".csv")
def userSessions(_userId):
_d = Data(_userId)
_sessions = _d.getSessions(False)
u = UserDispatcher()
_user = u.get(_userId,False)
return render_template("userFeed.html",user = _user,sessions = _sessions)
def getData(_userId,_sessionId,_type,_json=True): _d = Data(_userId) _sid = _sessionId return _d.getDataFromSession(_sid,_type,_json)
def main():
print("Start of main()")
# TODO: all these should be cleaned up like this:
# https://github.com/sisl/MADRL/blob/master/madrl_environments/walker/train_multi_walker.py
parser = argparse.ArgumentParser(description="Simulation of drivers' behavior")
parser.add_argument('-f', '--fleet',
help='Fleet sizes to simulate, formatted as comma-separated list (i.e. "-f 250,275,300")')
parser.add_argument('-m', '--multiplier',
help='Surge multiplier, formatted as comma-separated list (i.e. "-m 1,1.5,2")')
parser.add_argument('-b', '--bonus', type=int,
help='Bonus')
parser.add_argument('-d', '--demand',
help='Percent false demand ')
parser.add_argument('-k', '--know',
help='Percent knowing fare, formatted as comma-separated list (i.e. "-m 1,1.5,2") ')
parser.add_argument('-p', '--pro',
help='Percent pro drivers, formatted as comma-separated list (i.e. "-m 1,1.5,2") ')
parser.add_argument('-r', '--replications',
help='number of times to run the simulation')
parser.add_argument('-bb', '--beta',
help='BETA')
parser.add_argument('-b_policy', '--bonus_policy',
help='bonus per zone ')
parser.add_argument('-budget', '--budget',
help='budget ')
args = parser.parse_args()
# TODO: argpars should get the bonus policy as input
print("instantiate Data object")
data_instance = Data()
if args.fleet:
fleet_sizes = [int(args.fleet)]
else:
fleet_sizes = data_instance.FLEET_SIZE
if args.multiplier:
# surge = args.multiplier
surges = [float(x) for x in args.multiplier.split(',')]
else:
surges = [data_instance.SURGE_MULTIPLIER]
if args.know:
perc_know = [float(args.know)]
else:
perc_know = [data_instance.PERCE_KNOW]
if args.bonus:
bonus = args.bonus
else:
bonus = data_instance.BONUS
if args.beta:
beta = float(args.beta)
else:
beta = config_dict["BETA"]
if args.pro:
pro_share = [float(x) for x in args.pro.split(',')]
else:
pro_share = [data_instance.PRO_SHARE]
if args.demand:
percent_false_demand = float(args.demand)
else:
percent_false_demand = data_instance.PERCENT_FALSE_DEMAND
if args.replications:
n_rep = int(args.replications)
else:
n_rep = 5
if args.bonus_policy:
bonus_policy = args.bonus_policy
else:
bonus_policy = data_instance.BONUS_POLICY
if args.budget:
budget = args.budget
else:
budget = data_instance.BUDGET
# output_path = "./Outputs/avg_fare_info/" + str(beta) + "/"
for fleet_size in fleet_sizes:
for surge in surges:
for perc_k in perc_know:
for pro_s in pro_share:
for repl in range(n_rep):
# output_path = "./Outputs/avg_fare_info/" + str(budget) + "_" + str(bonus_policy) + "_" + \
# str(datetime.datetime.now()).split('.')[0] + "/"
# if not os.path.exists(output_path):
# os.makedirs(output_path)
print("iteration number ", repl)
print('Fleet size is {f}'.format(f=fleet_size))
print('Surge is {}'.format(surge))
print('Percentage knowing fares is {}'.format(perc_k))
print('Percentage of professional drivers {}'.format(pro_s))
data_instance.FLEET_SIZE = fleet_size
data_instance.PRO_SHARE = pro_s
data_instance.SURGE_MULTIPLIER = surge
data_instance.BONUS = bonus
data_instance.PERCENT_FALSE_DEMAND = percent_false_demand
data_instance.PERCE_KNOW = perc_k
print("Instantiated the model")
m = Model(data_instance, beta)
# start time
stime = time.time()
# # dispatch the system for T_TOTAL seconds, at the interval of INT_ASSIGN
for T in range(data_instance.WARMUP_TIME_SECONDS,
data_instance.T_TOTAL_SECONDS,
data_instance.INT_ASSIGN):
m.dispatch_at_time(T)
# end time
etime = time.time()
# run time of this simulation
runtime = etime - stime
print("The run time was {runtime} minutes ".format(runtime=runtime / 60))
m.save_zonal_stats("../performance_stats/")
class Exchange:
"""This class is used to interact with the market. It's a wrapper for a ccxt exchange object.
"""
def __init__(self,
logger,
data,
capital_allowed,
live_mode,
default_ticker=None,
default_period=None):
"""
- logger: a Logger instance linked to a bot
- data: a Data instance linked to a bot
- capital_allowed: float, percentage (0-100) of the capital this instance is able to trade with
- live_mode: bool, should we make real orders on the markets or simulate them
- default_ticker: string, the ticker formatted like that: ASSET1/ASSET2, optional
- default_period: string, the period resolution you want to get data in (1m, 5m, 1h...), optional
"""
self.logger = logger
self.data = data
self.capital_allowed = capital_allowed
self.default_ticker = default_ticker
self.default_period = default_period
self.live_mode = live_mode
self.fake_balance = []
self.fake_pnl = []
self.fake_current_price = None
self.fake_current_date = None
self.fake_orders = []
if (self.default_period and self.default_ticker):
self.cache = Data(
f"{self.default_period}-{''.join(self.default_ticker.split('/'))}-cache"
)
else:
self.cache = None
self.client = ccxt.binance(config.get_creds()['binance'])
def candles_to_df(self, candles):
data = pd.DataFrame()
data['timestamp'] = candles[:, 0]
data['date'] = pd.to_datetime(data['timestamp'] * 1000000)
data['open'] = candles[:, 1]
data['high'] = candles[:, 2]
data['low'] = candles[:, 3]
data['close'] = candles[:, 4]
data['volume'] = candles[:, 5]
return data
def get_latest_data(self, ticker=None, period=None, length=0):
"""Fetch the latest data for the given ticker. It will return at least lenth candles.
- ticker: string, the ticker formatted like that: ASSET1/ASSET2, optional if default is set
- period: string, the period resolution you want (1m, 5m, 1h...), optional is default is set
- length: int, the minimum number of past candles you want, optional
Returns a pandas dataframe.
"""
if (ticker is None):
ticker = self.default_ticker
if (period is None):
period = self.default_period
candles = None
try:
candles = np.array(self.client.fetch_ohlcv(ticker, period))
except:
self.logger.log(
"❗️", "Unable to fetch live data, retrying in 10 seconds")
time.sleep(10)
return self.get_latest_data(ticker, length)
return self.candles_to_df(candles)
def get_data(self, start_date, end_date, ticker=None, period=None):
"""Get historical data between the given dates.
- start_date: string, a date formatted in ISO 8601 from when to download data
- end_date: string, a date formatted in ISO 8601
"""
if (ticker is None):
ticker = self.default_ticker
if (period is None):
period = self.default_period
start = self.client.parse8601(start_date)
end = self.client.parse8601(end_date)
candles = None
last_date = start
while (last_date < end):
print(
f"Downloading {datetime.utcfromtimestamp(last_date / 1000).isoformat()}"
)
if (not self.cache is None
and not self.cache.get(last_date) is None):
new_candles = self.cache.get(last_date)
print("Found in cache")
else:
new_candles = np.array(
self.client.fetch_ohlcv(ticker, period, last_date))
if (not self.cache is None):
self.cache.set(last_date, new_candles)
time.sleep(1)
if (candles is None):
candles = new_candles
else:
candles = np.vstack([candles, new_candles])
last_date = int(candles[-1][0])
df = self.candles_to_df(candles)
return df
def buy(self, ticker=None, max_try=3):
"""Buy the given ticker.
- ticker: string, the ticker formatted like that: ASSET1/ASSET2, optional if default is set
Returns a trade object.
"""
if (max_try <= 0):
self.logger.log("❌", "Failed 3 times to buy, giving up")
return None
if (ticker is None):
ticker = self.default_ticker
if (not self.live_mode):
return self.fake_buy(ticker)
asset1 = ticker.split("/")[0]
asset2 = ticker.split("/")[1]
balance = self.get_balance(asset2)
price = self.client.fetch_ticker(ticker)['last']
proportion = (self.capital_allowed / 100)
qty = (balance * proportion) / price
qty = self.client.amount_to_precision(ticker, qty)
try:
self.logger.log("ℹ️", f"Buying {qty}{asset1}")
trade = self.client.create_market_buy_order(ticker, qty)
self.logger.log(
"💵", f"Bought {qty}{asset1} for {trade['price']:.2f}{asset2}")
self.logger.order('buy', trade['price'], trade['cost'])
self.data.set("buy_cost", trade['cost'])
return trade
except:
traceback.print_exc()
self.logger.log("❌", "Cannot buy, retrying in 3 seconds")
time.sleep(3)
return self.buy(ticker, max_try - 1)
def sell(self, ticker=None, max_try=3):
"""Sell the given ticker.
- ticker: string, the ticker formatted like that: ASSET1/ASSET2, optional if default is set
Returns a trade object.
"""
if (max_try <= 0):
self.logger.log("❌", "Failed 3 times to sell, giving up")
return None
if (ticker is None):
ticker = self.default_ticker
if (not self.live_mode):
return self.fake_sell(ticker)
asset1 = ticker.split("/")[0]
asset2 = ticker.split("/")[1]
balance = self.get_balance(asset1)
try:
self.logger.log("ℹ️", f"Selling {balance}{asset1}")
trade = self.client.create_market_sell_order(ticker, balance)
self.logger.log(
"💵", f"Sold {balance}{asset1} for {trade['price']}{asset2}")
self.logger.order('sell', trade['price'], trade['cost'])
balance_diff = trade['cost'] - self.data.get("buy_cost")
self.logger.pnl(balance_diff / self.data.get("buy_cost") * 100,
balance_diff)
self.logger.balance(self.get_balance())
self.data.remove("buy_cost")
return trade
except:
traceback.print_exc()
self.logger.log("❌", "Cannot sell, retrying in 3 seconds")
time.sleep(3)
return self.sell(ticker, max_try - 1)
def fake_buy(self, ticker):
"""This functions creates a fake buy order.
The buy function of a bot in backtracking mode is redirected here.
It's called automatically by the backtracking algorithm, you shouldn't
have to use it.
"""
self.data.set("buy_price", self.fake_current_price)
self.fake_orders.append({
'action': 'buy',
'date': self.fake_current_date
})
def fake_sell(self, ticker):
"""This functions creates a fake sell order.
The sell function of a bot in backtracking mode is redirected here.
It's called automatically by the backtracking algorithm, you shouldn't
have to use it.
It will calculate estimated PNL for the trade.
"""
diff_cost = self.fake_current_price - self.data.get("buy_price")
diff_per = (diff_cost / self.data.get("buy_price")) * 100
diff_per -= 0.2
profit = self.fake_balance[-1] * (diff_per / 100)
self.fake_balance.append(self.fake_balance[-1] + profit)
self.fake_pnl.append(diff_per)
self.data.remove("buy_price")
self.fake_orders.append({
'action': 'sell',
'date': self.fake_current_date
})
def get_balance(self, asset=None):
"""Get the balance of the account for the given asset.
- asset: string, the asset to check, optional if default is set
Returns the current asset balance as float.
"""
if (not self.live_mode):
return None
if (asset is None):
asset = self.default_ticker.split("/")[1]
try:
return self.client.fetch_balance()[asset]['free']
except:
self.logger.log("❌", f"Cannot fetch balance for {asset}")
return None
def init_fake_balance(self):
"""This functions initializes the backtesting fake balance array
"""
self.fake_balance = [100]
self.fake_pnl = []
self.fake_orders = []
def setUp(self):
self.data = Data('testdb.sqlite')
self.data.set('runTests', 'true')
class TestDataMethods(unittest.TestCase):
def setUp(self):
self.data = Data('testdb.sqlite')
self.data.set('runTests', 'true')
def test_initial_state(self):
''' self.data.keys will be empty at first '''
self.assertEqual(self.data.keys[0], 'runTests')
def test_get(self):
''' self.data.get test '''
self.assertEqual(self.data.get('runTests'), 'true')
def test_getAll(self):
''' set a couple more keys '''
self.data.set('anotherKey', 'false')
self.data.set('maybeOneMore', 'test')
self.assertEqual(self.data.getAll(), {
'anotherKey': 'false',
'runTests': 'true',
'maybeOneMore': 'test'
})
def test_delete(self):
self.data.delete('runTests')
self.assertEqual(self.data.get('runTests'), None)
# def test_get(self):
# ''' set a key/value pair '''
# self.data.set('runTests','true')
def tearDown(self):
os.remove('./data/testdb.sqlite')
def main():
parser = argparse.ArgumentParser(
description="Simulation of drivers' behavior")
# from lib.Constants import PERCE_KNOW
parser.add_argument(
'-f',
'--fleet',
help=
'Fleet sizes to simulate, formatted as comma-separated list (i.e. "-f 250,275,300")'
)
parser.add_argument(
'-m',
'--multiplier',
help=
'Surge multiplier, formatted as comma-separated list (i.e. "-m 1,1.5,2")'
)
parser.add_argument('-b', '--bonus', type=int, help='Bonus')
parser.add_argument('-d', '--demand', help='Percent false demand ')
parser.add_argument('-AV',
'--AV_fleet_size',
help="Number of Naive drivers ")
parser.add_argument('-NAIVE',
'--NAIVE_fleet_size',
help="Number of Naive drivers ")
parser.add_argument('-PRO',
'--PRO_fleet_size',
help="Number of Professional drivers ")
parser.add_argument(
'-BH',
'--behavioral_opt',
help="Perform behavioral optimization, pass 'yes' or 'no' ")
parser.add_argument('-SURGE',
'--surge_pricing',
help="should do surge pricing, pass 'yes' or 'no' ")
parser.add_argument(
'-k',
'--know',
help=
'Percent knowing fare, formatted as comma-separated list (i.e. "-m 1,1.5,2") '
)
parser.add_argument(
'-p',
'--pro',
help=
'Percent pro drivers, formatted as comma-separated list (i.e. "-m 1,1.5,2") '
)
parser.add_argument('-r',
'--replications',
help='number of times to run the simulation')
parser.add_argument('-bb', '--beta', help='BETA')
parser.add_argument('-b_policy', '--bonus_policy', help='bonus per zone ')
parser.add_argument('-budget', '--budget', help='budget ')
args = parser.parse_args()
# TODO: argpars should get the bonus policy as input
data_instance = Data()
if args.fleet:
fleet_sizes = [int(args.fleet)]
else:
fleet_sizes = data_instance.FLEET_SIZE
if args.behavioral_opt:
if args.behavioral_opt.lower() in ('yes', 'true'):
do_behavioral_opt = True
else:
do_behavioral_opt = False
else:
do_behavioral_opt = data_instance.do_behavioral_opt
if args.surge_pricing:
if args.surge_pricing.lower() in ('yes', 'true'):
do_surge_pricing = True
else:
do_surge_pricing = False
else:
do_surge_pricing = data_instance.do_surge_pricing
if args.PRO_fleet_size:
set_of_NUM_OF_PRO_DRIVERS = [int(args.PRO_fleet_size)]
else:
set_of_NUM_OF_PRO_DRIVERS = [data_instance.PRO_FLEET_SIZE]
if args.NAIVE_fleet_size:
set_of_NUM_OF_NAIVE_DRIVERS = [int(args.NAIVE_fleet_size)]
else:
set_of_NUM_OF_NAIVE_DRIVERS = [data_instance.NAIVE_FLEET_SIZE]
if args.AV_fleet_size:
set_of_NUM_OF_AV_DRIVERS = [int(args.AV_fleet_size)]
else:
set_of_NUM_OF_AV_DRIVERS = [data_instance.AV_FLEET_SIZE]
if args.multiplier:
# surge = args.multiplier
surges = [float(x) for x in args.multiplier.split(',')]
else:
surges = [data_instance.SURGE_MULTIPLIER]
if args.know:
perc_know = [float(args.know)]
else:
perc_know = [data_instance.PERCE_KNOW]
if args.bonus:
bonus = args.bonus
else:
bonus = data_instance.BONUS
if args.beta:
beta = float(args.beta)
else:
beta = configs_dict["BETA"]
if args.pro:
pro_share = [float(x) for x in args.pro.split(',')]
else:
pro_share = [data_instance.PRO_SHARE]
if args.demand:
percent_false_demand = float(args.demand)
else:
percent_false_demand = data_instance.PERCENT_FALSE_DEMAND
if args.replications:
n_rep = int(args.replications)
else:
n_rep = 1
if args.bonus_policy:
bonus_policy = args.bonus_policy
else:
bonus_policy = data_instance.BONUS_POLICY
if args.budget:
budget = args.budget
else:
budget = data_instance.BUDGET
# output_path = "./Outputs/avg_fare_info/" + str(beta) + "/"
from lib.rebalancing_optimizer import RebalancingOpt
for num_pros in set_of_NUM_OF_PRO_DRIVERS:
for num_naives in set_of_NUM_OF_NAIVE_DRIVERS:
for num_avs in set_of_NUM_OF_AV_DRIVERS:
for surge in surges:
for repl in range(n_rep):
TOTAL_FLEET_SIZE = 2500
num_naives = TOTAL_FLEET_SIZE - num_pros
data_instance.AV_FLEET_SIZE = num_avs
data_instance.NAIVE_FLEET_SIZE = num_naives
data_instance.PRO_FLEET_SIZE = num_pros
data_instance.do_behavioral_opt = do_behavioral_opt
data_instance.do_surge_pricing = do_surge_pricing
if do_behavioral_opt:
st = '/with_behavioral_opt/'
else:
st = '/no_behavioral_opt/'
output_path = "./Outputs/" + datetime.datetime.now().strftime("%Y-%m-%d-%H-%M") + \
st + str('Pro_') + str(num_pros) \
+ str('NAIVE_') + str(num_naives) \
+ str('AV_') + str(num_avs) \
+ str('budget_') + str(budget) + "_" \
+ str("bonus_policy") + "_" + str(bonus_policy) + "_" \
+ str('do_surge') + "_" + str(data_instance.do_surge_pricing) + "_" \
+ str('do_opt') + "_" + str(data_instance.do_behavioral_opt) + "_" \
+ str(datetime.datetime.now()).split('.')[0] + "/"
if not os.path.exists(output_path):
os.makedirs(output_path)
print("iteration number ", repl)
print('Surge is {}'.format(surge))
data_instance.SURGE_MULTIPLIER = surge
data_instance.BONUS = bonus
data_instance.output_path = output_path
# data_instance.do_behavioral_opt = False
m = Model(data_instance, configs_dict, beta,
output_path)
# start time
stime = time.time()
# dispatch the system for T_TOTAL seconds, at the interval of INT_ASSIGN
# TODO: every run should in include the policy from the start
# TODO: process Feb's month as well.
months = [1, 2]
days = [30, 15]
stop_month = months[-1]
for ix, month in enumerate(months):
for d_idx in range(1, days[ix]):
stop_day = days[ix]
if month == 1 and d_idx >= 15:
# NOTE: THIS WILL NOT HAVE THE DESIRED EFFECT, BC OPERATOR has attribute set in the beginning
data_instance.do_behavioral_opt = True
m.operator.do_behavioral_opt = True
# data_instance.do_surge_pricing = True
# m.operator.do_surge_pricing = True
for T in range(
data_instance.WARMUP_TIME_SECONDS,
data_instance.T_TOTAL_SECONDS,
data_instance.INT_ASSIGN):
m.dispatch_at_time(T, day_idx=d_idx)
m.get_service_rate_per_zone(d_idx, month)
m.get_drivers_earnings_for_one_day(
d_idx, month)
m.get_operators_earnings_for_one_day(
d_idx, month)
print(
f"Starting a new day, finished day number {d_idx + 1} of month {month}"
)
print(f"it took {(time.time() - stime) / 60}")
m.reset_after_one_day_of_operation(
stop_month, stop_day)
if num_pros > 0:
all_dfs = pd.concat([
v.report_learning_rates() for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
],
ignore_index=True)
all_dfs.to_csv(output_path +
"fmean for all drivers.csv")
all_fare_reliability_dfs = pd.concat(
[
v.report_fare_reliability_evolution()
for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
],
ignore_index=True)
all_fare_reliability_dfs.to_csv(
output_path +
"fare reliability for all drivers.csv")
all_m_reliability_dfs = pd.concat(
[
v.report_matching_reliability_evolution()
for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
],
ignore_index=True)
all_m_reliability_dfs.to_csv(
output_path +
"matching reliability for all drivers.csv")
all_fare_reliability_dfs = pd.concat(
[
v.report_surge_bonus_behavior()
for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
],
ignore_index=True)
all_fare_reliability_dfs.to_csv(
output_path +
"surge behavior for all drivers.csv")
all_earning_dfs = pd.concat(
[v.report_final_earnings() for v in m.vehicles],
ignore_index=True)
all_earning_dfs.to_csv(output_path +
"earnings for all drivers.csv")
operators_revenue = m.operator.report_final_revenue()
operators_revenue.to_csv(output_path +
"operators_revenue.csv")
print('Total drivers: ', len(m.vehicles))
print(
'# of Pro drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.PROFESSIONAL
]))
print(
'# of naive drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.NAIVE
]))
print(
'# of inexperienced drivers: ',
len([
v for v in m.vehicles
if v.driver_type == DriverType.INEXPERIENCED
]))
# end time
etime = time.time()
# run time of this simulation
runtime = etime - stime
print("The run time was {runtime} minutes ".format(
runtime=runtime / 60))
report = m.report_final_performance()
# So that it doesn't save a file with 1.5.py, rather 15.py
ss = str(surge).split('.')
ss = ''.join(ss)
fleet_size = num_avs + num_pros + num_naives
report.to_csv(output_path + "report for fleet size " +
str(fleet_size) + " surge " + str(ss) +
"pro_ " + str(num_pros) + "naive_ " +
str(num_naives) + "AV_" + str(num_avs) +
" repl" + str(repl) + ".csv")
import numpy as np
from lib.Data import Data
from lib.configs import config_dict
from lib.utils import Model
from lib.Constants import POLICY_UPDATE_INTERVAL, WARMUP_TIME_SECONDS, T_TOTAL_SECONDS, INT_ASSIGN, \
ANALYSIS_TIME_SECONDS, DEMAND_UPDATE_INTERVAL
import time
data_instance = Data.init_from_config_dic(config_dict)
m = Model(data_instance)
print('Fleet size is {f}'.format(f=data_instance.FLEET_SIZE))
stime = time.time()
# # dispatch the system for T_TOTAL seconds, at the interval of INT_ASSIGN
for T in range(data_instance.WARMUP_TIME_SECONDS,
data_instance.T_TOTAL_SECONDS,
data_instance.INT_ASSIGN):
m.dispatch_at_time(T)
# end time
etime = time.time()
# run time of this simulation
runtime = etime - stime
print("The run time was {runtime} minutes ".format(runtime=runtime / 60))
m.save_zonal_stats("../performance_stats/")
