def fetch_ohlcv(self, timeframe, since, instrument):
# adding one time interval because Bitmex api is returning us close times instead of open times
closeDate = since + timedelta(timeframe)
try:
result = self._client.Trade.Trade_getBucketed(
symbol=instrument,
reverse=False,
count=self._limit,
binSize=timeframe,
startTime=closeDate).result()[0]
except bravado.exception.HTTPTooManyRequests as err:
print("To many requests, try again later.")
result = []
candles = [candle for candle in result
if candle["open"]] # Filter bad data
# compute open and close times of each candle
for candle in candles:
candle["open_datetime_utc"] = candle['timestamp'] - timedelta(
timeframe)
candle["close_datetime_utc"] = candle['timestamp'] - timedelta(
'1s')
candle["trade_count"] = candle["trades"]
return candle_list_to_dataframe(candles)
def AddFromForm(self, form):
success = True
if self.id is None:
isNew = True
db.session.add(self)
db.session.commit()
else:
isNew = False
self.title = form.title.data
self.begin_time = form.begin_time.data - timedelta(hours = 8)
self.end_time = form.end_time.data - timedelta(hours = 8)
self.announcement = form.announcement.data
#check
if self.begin_time <= datetime.datetime.utcnow():
success = False
form.begin_time.errors.append("Begin Time should not early than current time")
if self.end_time <= self.begin_time:
success = False
form.end_time.errors.append("End Time should not early than Begin Time")
if not success and isNew:
db.session.delete(self)
db.session.commit()
return success
for cp in self.contest_problems.all():
db.session.delete(cp)
problem_id_list = map(int, form.problem_list.data.split(','))
for index, id in enumerate(problem_id_list):
prob = Problem.query.get(id)
if prob is not None:
cp = ContestProblem(contest_id = self.id, problem_id = id, problem_relative_id = index)
db.session.add(cp)
else:
success = False
form.problems.errors.append("Some problem maybe not exist")
try:
db.session.commit()
except IntegrityError:
db.session.rollback()
success = False
if not success and isNew:
db.session.delete(self)
db.session.commit()
if success:
db.session.add(self)
db.session.commit()
return success
def filter_events(events, indx_date, deliverables_path):
'''
Consider an observation window (2000 days) and prediction window (30 days). Remove
the events that occur outside the observation window.
Suggested steps:
1. Join indx_date with events on patient_id
2. Filter events occuring in the observation window(IndexDate-2000 to IndexDate)
Inputs: events is return from read_csv function, indx_date are returns from calculate_index_date function
Return filtered_events
IMPORTANT:
Save filtered_events to a csv file in the deliverables folder named as etl_filtered_events.csv.
Use the global variable deliverables_path while specifying the filepath.
Each row is of the form patient_id, event_id, value.
The csv file should have a header
'''
#Join indx_date with events on patient_id
joined_events = pd.merge(events, indx_date, on=['patient_id'], how = 'left')
#Filter events occuring in the observation window(IndexDate-2000 to IndexDate)
mask = np.logical_and(joined_events.timestamp <= joined_events.indx_date,
joined_events.timestamp >= joined_events.indx_date - utils.timedelta(2000))
filtered_events = joined_events.loc[mask, :]
#Save filtered_events to a csv file in the deliverables folder named as etl_filtered_events.csv
filtered_events.to_csv(deliverables_path + 'etl_filtered_events.csv', columns=['patient_id', 'event_id', 'value'], index=False)
return filtered_events
def reschedule_next_adventure(self):
# This method reschedules the next adventure if config time changes.
# You are expected to call this on all players if you want to update them.
minimum = config.delay_min
maximum = config.delay_max
minutes = utils.randfloat(minimum, maximum)
self.scheduled_adventure = self.last_adventure + utils.timedelta(
minutes=minutes)
def _get_nearest(self, dateStr, dateObj, direction):
# you must call this method only if you sure that requested date pass `self.dateWasLoaded`
if not self._loaded:
raise RuntimeError('Data not loaded yet')
if dateStr in self.__cache:
return dateStr, self.__cache[dateStr]
# this date have no data, so we swith it to next\prev with results
d=timedelta(days=direction)
while True:
dateObj+=d
dateStr=dateObj.strftime('%Y-%m-%d')
if dateStr in self.__cache:
return dateStr, self.__cache[dateStr]
def fetch_ohlcv(self, timeframe, since, instrument):
# Binance include the current (and unfinished) bar in the fetched data, we need to compute endTime to remove it
endTime = compute_end_timestamp(self.get_utc_time(), timeframe)
td = timedelta(timeframe)
td_1s = timedelta('1s')
result = self._client.get_product_historic_rates(
instrument,
granularity=int(td.total_seconds()),
start=since.timestamp())
result = reversed(
result
) # Coinbase pro is sending candles from newest to oldest, we need to reverse that
candles = [{
"open_datetime_utc":
datetime.fromtimestamp(int(data[0]), timezone.utc),
"close_datetime_utc":
datetime.fromtimestamp(int(data[0]), timezone.utc) + td - td_1s,
"open":
data[3],
"high":
data[2],
"low":
data[1],
"close":
data[4],
"volume":
data[5]
} for data in result]
# We need to manually filter candles because Coinsebase Pro API might give us candles before our startDate and after our endDate
candles = [
c for c in candles if since <= c['open_datetime_utc'] < endTime
]
return candle_list_to_dataframe(candles)
async def check_players_for_adventure():
# Embarks any player that needs an adventure.
if world.players is None or len(world.players) == 0:
return
for player_id, player_instance in world.players.items():
if is_player_online(player_id):
player_instance.last_online = utils.now()
if player_instance.paused is True:
continue
if player_instance.last_online + utils.timedelta(
minutes=config.keep_online) < utils.now():
continue
if player_instance.scheduled_adventure is None or player_instance.scheduled_adventure < utils.now(
):
message = player_instance.embark()
await broadcast(message)
def _get_nearest_date(self, direction, date):
dateStep = direction * self._loader.direction if direction else self._loader.direction
if date is not None and direction:
date = (to_date(date) +
timedelta(days=dateStep)).strftime('%Y-%m-%d')
if date in self._data:
dialog = 0 if dateStep < 0 else len(self._data[date]) - 1
msg = None if direction > 0 or not self.isOpened(
date,
dialog) else self._data[date][dialog].messageCount - 1
return date, dialog, msg
date, data = self._loader.get(date, dateStep)
if data is False:
return None, 0, None
elif date not in self._data:
print(f'DIALOG_CACHE_UPDATE {date}')
self._data[date] = tuple(
WidgetPlaceholderEx(DialogHeader(None, o)) for o in data)
dialog = 0 if dateStep < 0 else len(self._data[date]) - 1
msg = None if direction > 0 or not self.isOpened(
date, dialog) else self._data[date][dialog].messageCount - 1
return date, dialog, msg
def calculate_index_date(events, mortality, deliverables_path):
'''
Index date: The day on which mortality is to be predicted. Index date is evaluated as follows:
For deceased patients: Index date is 30 days prior to the death date (timestamp fi
eld)
in data/train/mortality events.csv.
For alive patients: Index date is the last event date in data/train/events.csv for
each alive patient.
Inputs are returns from read_csv function
Return indx_date
IMPORTANT:
Save indx_date to a csv file in the deliverables folder named as etl_index_dates.csv.
Use the global variable deliverables_path while specifying the filepath.
Each row is of the form patient_id, indx_date.
The csv file should have a header
'''
#Last event date
alive_date = pd.DataFrame({'timestamp' : events.groupby('patient_id').timestamp.max()}).reset_index()
#30 days prior to the death date for deceased patients
deceased_date = mortality[["patient_id","timestamp"]]
deceased_date.loc[:,"timestamp"] = deceased_date.loc[:,"timestamp"] - utils.timedelta(30)
#Last event date for alive patients
still_alive_date = alive_date[~alive_date.patient_id.isin(deceased_date.patient_id)]
#Conbine both alive and deceased patients into one dataframe
indx_date = pd.concat([still_alive_date, deceased_date], axis = 'rows')
indx_date.columns = ['patient_id','indx_date']
#Save indx_date to a csv file in the deliverables folder named as etl_index_dates.csv
indx_date.to_csv(deliverables_path + 'etl_index_dates.csv', columns=['patient_id', 'indx_date'], index=False)
return indx_date
def train(self):
self.model.train()
num_epochs = self.opts.num_epochs
batch_size = self.opts.batch_size
num_batch = math.ceil(self.opts.epoch_size / batch_size)
total_steps = num_epochs * num_batch
start_step = self.train_states["train_steps"]
remain_steps = total_steps - start_step
print("total_steps : {}".format(total_steps))
# data
train_iter = self.data.train_iterator(start_step % num_batch)
if start_step == 0:
print("{}\tStart training......".format(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())))
else:
print("{}\tContinue training......".format(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())))
# timing
step_stime = time.time()
for _ in range(remain_steps):
self.train_states["train_steps"] += 1
var_dict = self._forward(train_iter, mode="train")
cur_ce = var_dict["cur_ce"]
cur_kld = var_dict["cur_kld"]
kld_coeff = var_dict["kld_coeff"]
cur_bow_loss = var_dict["cur_bow_loss"]
cur_loss = cur_ce * self.opts.ce_coeff + cur_kld * kld_coeff + cur_bow_loss * self.opts.bow_coeff
if torch.isnan(cur_loss):
print("ERROR: NAN detected.")
exit(0)
self.optim.zero_grad()
cur_loss.backward()
nn.utils.clip_grad_norm_(self.model.parameters(),
self.opts.max_gradient_norm)
self.optim.step()
self.learning_rate_decay()
cur_loss = cur_ce * self.opts.ce_coeff + cur_kld + cur_bow_loss * self.opts.bow_coeff
var_dict["cur_loss"] = cur_loss
# update train_states
for k, v in var_dict.items():
if k.startswith("cur_"):
if (self.train_states["train_steps"] - 1) % self.opts.logging_rate + self.opts.logging_window \
>= self.opts.logging_rate:
self.train_states[k]["step"] += v.item()
if (self.train_states["train_steps"] - 1) % self.opts.validation_rate + self.opts.validation_window \
>= self.opts.validation_rate:
self.train_states[k]["mean"] += v.item()
for k, v in var_dict.items():
if k.startswith("cur_"):
self.metrics["trnes_" + k[4:]].append(v.item())
# logging
if self.train_states["train_steps"] % self.opts.logging_rate == 0:
duration = time.time() - step_stime
tc = self.metrics["train_time"] + self.metrics[
"valid_time"] + duration
rte = (total_steps - self.train_states["train_steps"]
) * tc / self.train_states["train_steps"]
print(
"Step {} progress rate: {:.3f} loss: {:.8f} ppl: {:.6f} "
"speed: {:.0f} samples per second remaining (approximately): {}"
.format(
self.train_states["train_steps"],
self.train_states["train_steps"] / total_steps,
self.train_states["cur_loss"]["step"] /
self.opts.logging_window,
self.train_states["cur_ppl"]["step"] /
self.opts.logging_window,
self.opts.logging_rate * batch_size / duration,
timedelta(rte)))
# update metrics
self.metrics["train_steps"].append(
self.train_states["train_steps"])
self.metrics["train_time"] += duration
self.metrics["kl_coefficient"].append(kld_coeff)
for k in var_dict.keys():
if k.startswith("cur_"):
self.metrics["train_" + k[4:]].append(
self.train_states[k]["step"] /
self.opts.logging_window)
# update train_states
for k in var_dict.keys():
if k.startswith("cur_"):
self.train_states[k]["step"] = 0.0
# reset timing
step_stime = time.time()
del var_dict
# valid
if self.train_states["train_steps"] % self.opts.validation_rate == 0 or \
self.train_states["train_steps"] in self.opts.specific_validations:
step_stime += self.validate()
# after each epoch
if self.train_states["train_steps"] % num_batch == 0:
completed_epochs = self.train_states["train_steps"] // num_batch
print(
"-------------------Epoch Completed %d/%d-------------------"
% (completed_epochs, num_epochs))
# save states
temp_stime = time.time()
self.maybe_save_checkpoints()
step_stime += time.time() - temp_stime
if -1 in self.opts.specific_validations:
self.validate()
if -1 in self.opts.specific_checkpoints:
if self.do_testing:
self.test()
self.uninterruptible_save(False)
else:
self.uninterruptible_save(True)
print("{}\tFinished!".format(
time.strftime('%Y-%m-%d %H:%M:%S', time.localtime())))
def main():
# pp = pprint.PrettyPrinter(indent=4)
args = parse_cli_args()
exchanges = {
"bitmex": make_bitmex_exchange,
"binance": make_binance_exchange,
"coinbasepro": make_coinbasepro_exchange
}
if args.action == "list-exchanges":
print(to_comma_separated_string(exchanges.keys()))
exit(0)
if args.exchange in exchanges.keys():
exchange = exchanges[args.exchange]()
else:
print("Unsupported exchange {}".format(args.exchange))
exit(-1)
if args.action == "time":
print(exchange.get_utc_time())
exit(0)
if args.action == "timestamp":
print(exchange.get_utc_timestamp())
exit(0)
if args.action == "list-instruments":
print(to_comma_separated_string(exchange.get_instruments()))
exit(0)
if args.action == "list-timeframes":
print(to_comma_separated_string(exchange.get_timeframes()))
exit(0)
assert (args.action == "fetch-ohlcv")
ensure_mkdir(args.folder)
timeframes = args.timeframes if args.timeframes else exchange.get_timeframes(
)
instruments = args.instruments if args.instruments else exchange.get_instruments(
)
exchange_timeframes = exchange.get_timeframes()
exchange_instruments = exchange.get_instruments()
exchange_time = exchange.get_utc_time()
print("Exchange {} at time {}.".format(args.exchange, exchange_time))
for instrument in instruments:
if not instrument in exchange_instruments:
print("[ERROR] Unsupported instrument {} for exchange {}.".format(
instrument, args.exchange))
continue
print("-- Fetching data for instrument {}.".format(instrument))
for tf in timeframes:
if not tf in exchange_timeframes:
print(
"[ERROR] Unsupported timeframe {} for exchange {}.".format(
tf, args.exchange))
continue
print("\t-- Fetching data for timeframe {}.".format(tf))
path_to_csv_file = os.path.join(
args.folder,
args.exchange + "-" + instrument + "-" + tf + ".csv")
since = origin_of_time
if (os.path.exists(path_to_csv_file)):
print(
"\t\t-- Loading existing history from file {} to get next timestamp."
.format(path_to_csv_file))
df = pd.read_csv(path_to_csv_file,
index_col='open_timestamp_utc')
since = datetime.fromtimestamp(
df.close_timestamp_utc.values[-1], timezone.utc)
next_open_date = compute_end_timestamp(since, tf) + timedelta('1s')
if exchange_time < next_open_date:
print(
"\t\t-- Exchange time is {} and next candle time is {}, no request needed."
.format(exchange_time, since + td))
continue
while True:
print("\t\t-- Fetching candles since {}".format(since))
df = exchange.fetch_ohlcv(timeframe=tf,
since=since,
instrument=instrument)
if df.empty:
print(
"\t\t-- No candles received for timeframe {}, work is done."
.format(tf))
break
else:
print("\t\t-- {} candles received.".format(len(df)))
df.to_csv(path_to_csv_file,
index_label='open_timestamp_utc',
mode='a',
header=not os.path.exists(path_to_csv_file))
since = datetime.fromtimestamp(
df.close_timestamp_utc.values[-1], timezone.utc)
time.sleep(
args.delay /
1000.0) # ensure we don't flood exchange API with requests
async def on_chat_message(self, message):
try:
content_type, chat_type, chat_id, _, msg_id = \
telepot.glance(message, long=True)
# we only work with "text" messages
if content_type != 'text':
if chat_type in self.PRIVATE_CHATS:
# only do this if it's a privmsg
await self.send_message(message,
self.INVALID_CMD,
quote_msg_id=msg_id)
return
# we only work in private, groups and supergroups
if chat_type not in self.PRIVATE_CHATS + self.PUBLIC_CHATS:
return
# get the name of the user and log it
name = self.format_name(message)
if not name:
utils.logger.warning('Message without a "from" field received')
return
longname = '{chat_id} ({name})'.format(chat_id=chat_id, name=name)
utils.logger.info('Message from %s: "%s"', longname,
message['text'])
# commands need special handling
if message['text'].startswith('/'):
command, rest = self._get_command(message['text'])
akari_commands = ('akari', 'akari@' + self.username)
if command not in akari_commands:
if chat_type in self.PRIVATE_CHATS:
if command in ('help', 'start'):
msg = self.help_msg_priv
else:
msg = self.INVALID_CMD
await self.send_message(message,
msg,
no_preview=True,
quote_msg_id=msg_id)
return
elif chat_type in self.PUBLIC_CHATS:
return # unknown cmd, this was meant for another bot
else:
rest = message['text'].strip()
# if the resulting message is empty...
if not rest:
if chat_type in self.PUBLIC_CHATS: # only show this in groups
await self.send_message(message,
self.help_msg_group,
quote_msg_id=msg_id)
return
# check rate limit if this chat id is not exempt from them
if chat_id not in self.rate_limit_exemptions:
rate_limit = utils.ratelimit_hit('telegram', chat_id)
if not rate_limit['allowed']:
msg = 'Message from %s: throttled (resets in %d seconds)'
utils.logger.warning(msg, longname, rate_limit['reset'])
msg = ('Not so fast! Try again in %s.' %
utils.timedelta(rate_limit['reset']))
await self.send_message(message, msg, quote_msg_id=msg_id)
return
msg = random.choice(self.COMPOSING_MSGS)
await self.send_message(message, msg, quote_msg_id=msg_id)
# first, search...
try:
akari = Akari(rest, type='animation', shuffle_results=True)
except ImageSearchNoResultsError:
await self.send_message(message,
'No results.',
quote_msg_id=msg_id)
return
# then, if successful, send the pic
utils.logger.info('Sending %s to %s', akari.filename, longname)
type = 'file' if akari.filename.endswith('.gif') else 'image'
await self.send_message(message,
type=type,
filename=akari.filename,
quote_msg_id=msg_id)
except Exception:
utils.logger.exception('Error handling %s (%s)', longname,
message['chat']['type'])
await self.send_message(message,
'Sorry, try again.',
quote_msg_id=msg_id)
