def get_start_date(created):
# 1 day before post date
us_holidays = holidays.UnitedStates(years=2021) + holidays.UnitedStates(
years=2020)
ticker_date = datetime.fromtimestamp(int(created) - 86400).date()
if ticker_date in us_holidays or ticker_date == datetime(2021, 4,
2).date():
return False
if ticker_date.weekday() in [5, 6]:
return False
return ticker_date
def clean_dataset(df: pd.DataFrame) -> pd.DataFrame:
us_holidays = holidays.UnitedStates()
df['datetime'] = pd.to_datetime(df['dteday']) + df['hr'].apply(
pd.Timedelta, unit='h')
df = df.set_index('datetime')
df = df.reindex(
pd.date_range(start=df.index[0], end=df.index[-1], freq='1H'))
df.drop(columns=['dteday', 'instant'], inplace=True)
null_index = df[df.isna().any(axis=1)].index
for index in null_index:
df.loc[index,
['hr', 'yr', 'mnth']] = (index.hour, int(index.year == 2012),
index.month)
df.loc[index , ['holiday' , 'weekday' , 'workingday','season','weathersit']] = \
(int(index in us_holidays) , index.dayofweek , int(index.dayofweek in [0,1,2,3,4]) ,_get_season(index),randint(1,4))
df['temp'].fillna(df['temp'].rolling(60, min_periods=1).mean(),
inplace=True)
df['atemp'].fillna(df['atemp'].rolling(60, min_periods=1).mean(),
inplace=True)
df['hum'].fillna(df['hum'].rolling(60, min_periods=1).mean(), inplace=True)
df['windspeed'].fillna(df['windspeed'].rolling(60, min_periods=1).mean(),
inplace=True)
df['casual'].fillna(df['casual'].rolling(60, min_periods=1).mean(),
inplace=True)
df['registered'].fillna(df['registered'].rolling(60, min_periods=1).mean(),
inplace=True)
df['cnt'] = df['registered'] + df['casual']
assert ((365 * 24 * 2) + 24) == df.shape[0]
return df
def is_date_holiday(this_date: Union[date, str]) -> bool:
this_date = validate_date(this_date)
us_holidays = holidays.UnitedStates(years=this_date.year)
# generate list without columbus day and veterans day since markets are open on those days
trading_holidays = [
"Columbus Day", "Columbus Day (Observed)", "Veterans Day",
"Veterans Day (Observed)"
]
# markets are open
# see here: https://www.barrons.com/articles/stock-market-open-close-new-years-eve-monday-hours-51640891577
if datetime.datetime(year=this_date.year + 1, month=1,
day=1).weekday() in [5, 6]:
trading_holidays += ["New Year's Day (Observed)"]
custom_holidays = [
that_date for that_date in list(us_holidays)
if us_holidays[that_date] not in trading_holidays
]
# add good friday to list since markets are closed on good friday
custom_holidays.append(
easter.easter(year=this_date.year) - datetime.timedelta(days=2))
return (this_date in custom_holidays)
def featureExternal(self, df, train_mode=True):
df = df.copy()
self.us_holidays = list(
holidays.UnitedStates(years=[2016, 2017, 2018]).keys())
df['external_us_holiday'] = (df['visitId'].dt.date.isin(
self.us_holidays)).astype(int)
df['external_us_holiday_delta_1'] = \
df['external_us_holiday'] + self.judge_us_holiday(df, 1)
df['external_us_holiday_delta_2'] = \
df['external_us_holiday_delta_1'] + self.judge_us_holiday(df, 2)
df['external_us_holiday_delta_3'] = \
df['external_us_holiday_delta_2'] + self.judge_us_holiday(df, 3)
df['external_us_holiday_delta_4'] = \
df['external_us_holiday_delta_3'] + self.judge_us_holiday(df, 4)
df['external_us_holiday_delta_5'] = \
df['external_us_holiday_delta_4'] + self.judge_us_holiday(df, 5)
df['external_visit_is_local_holiday'] = self.judge_local_holiday(df)
df['date'] = df['date'].astype(int)
component = self.UsdxIndexInput()
df = df.merge(component, on='date', how='left')
component = self.EconomicIndexInput()
df = df.merge(component, on='date', how='left')
df[['external_usdx_index','external_employment',
'external_rate','external_unemployment']] = \
df[['external_usdx_index','external_employment','external_rate',
'external_unemployment']].fillna(method='ffill')
return df
def blackGexFull(filename):
raw = pd.read_table(filename)
spotF = float(raw.columns[0].split(',')[-2])
ticker = raw.columns[0].split(',')[0][1:4]
rf = .02
pltDate = raw.loc[0][0].split(',')[0][:11]
pltTime = raw.loc[0][0].split(',')[0][-8:]
dtDate = datetime.datetime.strptime(pltDate, '%b %d %Y').date()
raw = pd.read_table(filename, sep=',', header=2)
c = raw.loc[:, :'Strike'].copy(deep=True)
c.columns = c.columns.str.replace('Calls', 'ID')
p = (raw.loc[:, 'Strike':].join(raw.loc[:, 'Expiration Date']))\
.copy(deep=True)
p.columns = p.columns.str.replace('Puts', 'ID')
p.columns = p.columns.str.replace('.1', '')
p = p[c.columns]
c['Flag'] = 'c'
p['Flag'] = 'p'
c['Expiry'] = pd.to_datetime(c['Expiration Date'],
infer_datetime_format=True)
p['Expiry'] = pd.to_datetime(p['Expiration Date'],
infer_datetime_format=True)
df = c.append(p, ignore_index=True)
df = df[(df['ID'].str[-3] != '-') &\
(df['ID'].str[-4] != '-')
].copy(deep=True)
for item in [
'Bid', 'Ask', 'Last Sale', 'IV', 'Delta', 'Gamma', 'Open Int',
'Strike'
]:
df[item] = pd.to_numeric(df[item], errors='coerce')
us_holidays = holidays.UnitedStates(years=list(range(2000, 2030)))
us_hlist = list(us_holidays.keys())
A = [d.date() for d in df['Expiry']]
df['BDTE'] = np.busday_count(dtDate,
A,
weekmask='1111100',
holidays=us_hlist)
df = df.loc[(df['BDTE'] >= 1) #&\
].copy(deep=True)
df['Mid'] = np.mean(df[['Bid', 'Ask']], axis=1)
df['IV'] = df.apply(lambda x: blackIV(x, F=spotF, rf=rf), axis=1)
df['Delta'] = df.apply(lambda x: blackDelta(x, F=spotF, rf=rf), axis=1)
df['Gamma'] = df.apply(lambda x: blackGamma(x, F=spotF, rf=rf), axis=1)
return df
def getWorkDates(length):
us_holidays = holidays.UnitedStates()
a = np.array(datetime.now())
for x in range(1, length):
a = np.append(a, datetime.now())
pass
x = 0
for y in range(45, 0, -1):
Date = datetime.now() + timedelta(days=-y)
if Date.weekday() <= 4 and not (Date in us_holidays):
a[x] = Date
x += 1
pass
if (x == length):
break
pass
print(a)
return a
def is_holiday(date_str):
date_str = date_str.replace(",", "")
date_params = date_str.split(" ")
month = months[date_params[0]]
us_holidays = holidays.UnitedStates()
return datetime.date(int(date_params[2]), int(month),
int(date_params[1])) in us_holidays
def is_market_day(today):
not_holiday = today not in holidays.UnitedStates()
isntweekend = today.weekday()
if not_holiday and (isntweekend != 6) and (isntweekend != 5):
return True
else:
return False
def market_day():
not_holiday = today not in holidays.UnitedStates()
isntweekend = today.isoweekday()
if not_holiday and (isntweekend != 7) and (isntweekend != 1):
return True
else:
return False
def first_working_day(date):
"""Returns the nearest working day today or in past.
This algorithm is used when back-end generates a cycle task.
"""
while _is_weekend(date) or date in holidays.UnitedStates():
date += datetime.timedelta(days=-1)
return date
def test_eq_ne(self):
us1 = holidays.UnitedStates()
us2 = holidays.US()
us3 = holidays.UnitedStates(years=[2014])
us4 = holidays.US(years=[2014])
ca1 = holidays.Canada()
ca2 = holidays.CA()
ca3 = holidays.Canada(years=[2014])
ca4 = holidays.CA(years=[2014])
self.assertEqual(us1, us2)
self.assertEqual(us3, us4)
self.assertEqual(ca1, ca2)
self.assertEqual(ca3, ca4)
self.assertNotEqual(us1, us3)
self.assertNotEqual(us1, ca1)
self.assertNotEqual(us3, ca3)
def business_days(start_date: date,
end_date: date) -> Generator[date, None, None]:
us_holidays = holidays.UnitedStates()
for n in range((end_date - start_date).days + 1):
the_date = start_date + timedelta(n)
if (the_date.weekday() < 5) and (the_date not in us_holidays):
yield the_date
def first_working_day_after_today(date):
"""Returns the nearest working day today or in future.
"""
date = date + datetime.timedelta(days=1)
while _is_weekend(date) or date in holidays.UnitedStates():
date += datetime.timedelta(days=1)
return date
def getWorkDates(length): us_holidays = holidays.UnitedStates() a=np.array(datetime.now()) for x in range(1,length): a=np.append(a,datetime.now()) pass x=0 for y in range(45,0,-1): Date=datetime.now()+timedelta(days=-y) if Date.weekday()<=4 and not (Date in us_holidays) : a[x]=Date x+=1 pass if(x==length): break pass print(a) return a #(Prediction DO NOT TOUCH WIHTOUT NOTIFYING ME)C:\\cygwin\bin\Main.py
def is_US_holiday(est_date):
# if not a holiday then holiday will be assigned NoneType
# if it is a holiday then holiday will be assigned a US Holiday Name -> 'Independence Day'
us_holidays = holidays.UnitedStates()
holiday = us_holidays.get(est_date)
# custom_holidays = holidays.HolidayBase()
# usage append: custom_holidays.append({"2018-03-30": "Good Friday"})
# usage get: custom_holidays.get("2018-03-30")
#
# Good Friday needs to be added as a custom holiday.
custom_holidays = holidays.HolidayBase()
custom_holidays.append({'2017-04-14': 'Good Friday'})
custom_holidays.append({'2018-03-30': 'Good Friday'})
custom_holidays.append({'2019-04-19': 'Good Friday'})
custom_holidays.append({'2020-04-10': 'Good Friday'})
custom_holidays.append({'2021-04-02': 'Good Friday'})
custom_holidays.append({'2022-04-15': 'Good Friday'})
custom_holidays.append({'2023-04-07': 'Good Friday'})
custom_holidays.append({'2024-03-29': 'Good Friday'})
custom_holidays.append({'2025-04-18': 'Good Friday'})
custom_holidays.append({'2026-04-03': 'Good Friday'})
if holiday == None:
holiday = custom_holidays.get(est_date)
if holiday == None:
holiday_result = ('not holiday')
else:
holiday_result = ('is holiday', holiday)
return holiday_result
def init_open_days():
store_holidays = holidays.UnitedStates()
# 2018 Federal Holidays when store was open
open_2018 = ['2018-01-15', '2018-02-19', '2018-10-08', '2018-11-12']
for day in open_2018:
store_holidays.pop(day)
# Closed first week of Jan 2017 for renovations
store_holidays.append(
['2018-01-09', '2018-01-10', '2018-01-11', '2018-01-12'])
# Snow day
store_holidays.append('2018-03-13')
# Closed Labor Day weekend
store_holidays.append('2018-09-01')
def is_open(day):
if day.day_of_week == 0:
if day > _THANKSGIVING and day < _NEWYEARS:
return True
else:
return False
elif day in store_holidays:
return False
else:
return True
return is_open
def create_holiday_frame():
'''
Create a dataframe with all holiday run ups and hangovers
A run up is two days prior to a holiday and a hangover is 2 days after
a given holiday
'''
def no_holiday(obs):
'''
This US holidays package returns a null if there is no holiday, and we
want to flip that null into a string so we can remove it later when we
encode the columns
'''
if obs == None:
return_value = 'No_Holiday'
else:
return_value = obs
return return_value
# Load the united states holidays
us_holidays = holidays.UnitedStates()
holiday_frame = pd.DataFrame()
# Iterate over every day starting on January 1st 2016 and check
start_date = datetime.datetime(year=2016, day=1, month=1)
for i in range(0, (365 * 10)):
holiday_frame.loc[i, 'event_date'] = start_date + datetime.timedelta(i)
holiday_frame.loc[i, 'Holiday'] = us_holidays.get(
holiday_frame.loc[i, 'event_date'])
# Get a list of unique holidays and the non nulls
unique_holidays = holiday_frame['Holiday'].unique().tolist()
unique_holidays = [holiday for holiday in unique_holidays if holiday !=\
None]
# But in the holiday hangovers for the two days after
for i in range(0, len(holiday_frame)):
if holiday_frame.loc[i, 'Holiday'] in unique_holidays:
for bar in range(1, 2):
holiday_frame.loc[i + bar, 'Holiday'] = 'Holiday_Hangover'
# Put in the holiday hangovers for the two days before
for i in range(0, len(holiday_frame)):
if holiday_frame.loc[i, 'Holiday'] in unique_holidays:
for bar in range(1, 2):
if i - bar < 0:
pass
else:
holiday_frame.loc[i - bar, 'Holiday'] = 'Holiday_RunUp'
# New years can create some negative indexes don't need them that far back
holiday_frame['event_date'] = holiday_frame['event_date'].\
apply(lambda x: pd.to_datetime(x).strftime('%Y-%m-%d'))
holiday_frame.index = holiday_frame['event_date']
holiday_frame['Holiday'] = holiday_frame['Holiday'].apply(lambda x: \
no_holiday(x))
return holiday_frame
def get_holidays_between(start_date: Union[date, str],
end_date: Union[date, str]) -> int:
if isinstance(start_date, date):
start_date = to_string(start_date)
if isinstance(end_date, date):
end_date = to_string(end_date)
us_holidays = holidays.UnitedStates()
return len(us_holidays[start_date:end_date])
def init_is_holiday(self, date_hour):
us_holidays = holidays.UnitedStates()
if date_hour.weekday() == 5 or 6:
return True
elif datetime in us_holidays:
return True
else:
return False
def feature_xform(inDictRaw):
def cur_morning_tolling(time):
hour = time.hour
minute = time.minute
if (hour == 9 and minute >= 30) or (hour == 13 and minute <= 30) or (
hour >= 10 and hour <= 12):
return 1
return 0
def cur_evening_tolling(time):
hour = time.hour
if (hour >= 19 and hour <= 23):
return 1
return 0
# df = pd.DataFrame(columns=db_to_df_name_map.keys())
df = pd.DataFrame(columns=db_to_df_name_map)
print(df)
for a in inDictRaw:
df = df.append(a, ignore_index=True)
df = df.rename(columns=db_to_df_name_map)
tz = pytz.timezone("GMT")
time = datetime.now().astimezone(tz)
#Add ML useable columns to represent the date and time
#df["timestamp"] = time
df["hour"] = time.hour
df["minute"] = time.minute
df["weekday"] = calendar.day_name[time.weekday()]
df["month"] = calendar.month_name[time.month]
df["year"] = time.year - 2017
#Add a feature that is a 1 if the date is a federal holiday, 0 otherwise
us_holidays = holidays.UnitedStates()
df['Holiday'] = int(str(time) in us_holidays)
#Add two columns to specify whether morning tolling or evening tolling is active
df['Morning Tolling'] = cur_morning_tolling(time)
df['Evening Tolling'] = cur_evening_tolling(time)
df = encoder.transform(df)
all_columns = [
'EndZoneID', 'StartZoneID', 'Direction_1', 'Direction_2',
'Temperature', 'Humidity', 'Wind Speed', 'Weather_1', 'Weather_2',
'Weather_3', 'Weather_4', 'hour', 'minute', 'weekday_1', 'weekday_2',
'weekday_3', 'weekday_4', 'weekday_5', 'weekday_6', 'weekday_7',
'month_1', 'month_2', 'month_3', 'month_4', 'month_5', 'month_6',
'month_7', 'month_8', 'month_9', 'month_10', 'month_11', 'month_12',
'year', 'Holiday', 'Morning Tolling', 'Evening Tolling', 'Toll_Prev_1',
'Toll_Prev_2'
]
dfNew = df[all_columns]
return dfNew
def business_days(start_date, end_date):
us_holidays = holidays.UnitedStates()
for num in range((end_date - start_date).days + 1):
the_date = start_date + timedelta(days=num)
if (the_date.weekday() < 5) and (the_date not in us_holidays):
print("yielding")
yield the_date
def select_holiday(num):
_holidays = []
for date in holidays.UnitedStates(years=2021).items():
_holidays.append(str(date[0]))
res = []
for i in range(num):
rnd = random.randint(0, len(_holidays) - 1)
res.append(_holidays[rnd])
return res
def market_day():
today = ((datetime.datetime.today()))
today2 = (datetime.datetime.today() - datetime.timedelta(days=1))
not_holiday = today2 not in holidays.UnitedStates()
isntweekend = today.isoweekday()
if not_holiday and (isntweekend != 7) and (isntweekend != 1):
return True
else:
return False
def save(self, *args, **kwargs):
today = datetime.datetime.now()
month = today.month
year = today.year
# first calculate the calculated date from the day of month given
self.calculated_date = datetime.date(day=self.day_of_month_of_occ,
year=year,
month=month)
while self.calculated_date < self.start_date:
next_day = self.day_of_month_of_occ
next_year = year
next_month = (self.calculated_date.month + 1) % 12
if (self.calculated_date.month + 1) > 12:
next_year += 1
created = False
while not created:
# handle days that don't exist in the month
try:
self.calculated_date = datetime.date(
day=next_day,
year=next_year,
month=next_month if next_month > 0 else 12)
created = True
except ValueError:
next_day -= 1
self.delivery_date = self.calculated_date
us_holidays = holidays.UnitedStates()
while self.delivery_date.weekday() > 4 or \
self.delivery_date in us_holidays:
self.delivery_date = self.delivery_date - datetime.timedelta(
days=1)
# make sure we have not gone back past the start date - if so
# go forward to the next month
if self.delivery_date < self.start_date:
next_day = self.day_of_month_of_occ
next_year = self.delivery_date.year
next_month = (self.delivery_date.month + 1) % 12
if (self.delivery_date.month + 1) > 12:
next_year += 1
created = False
while not created:
try:
self.day_of_month_of_occ = \
datetime.date(day=next_day,
year=next_year,
month=next_month)
except ValueError:
next_day -= 1
super(Event, self).save(*args, **kwargs)
def test_get_named(self):
us = holidays.UnitedStates(years=[2020])
# check for "New Year's Day" presence in get_named("new")
self.assertIn(date(2020, 1, 1), us.get_named("new"))
# check for searching holiday in US when the observed holiday is on
# a different year than input one
us = holidays.US(years=[2022])
us.get_named("Thanksgiving")
self.assertEqual([2022], list(us.years))
def business_days(start_date: date,
end_date: date) -> Generator[date, None, None]:
""" generate business days for a date range inclusive """
us_holidays = holidays.UnitedStates()
for num in range((end_date - start_date).days + 1):
the_date = start_date + timedelta(days=num)
if (the_date.weekday() < 5) and (the_date not in us_holidays):
yield the_date
def isLastDayOfTheBWeek(
date
): #checking if the date considered to be the last day of the week to know if we should start doing clculations
isHoliday = False
isFriday = date.weekday() == 4
nextDay = date + BDay(1)
isNextDayFRandHoliday = nextDay.weekday(
) == 4 and nextDay in hl.UnitedStates()
isHoliday = isFriday or isNextDayFRandHoliday
return isHoliday
def __init__(self, iso: str, start_date: str, end_date: str):
self.start = start_date
self.end = end_date
self.iso_name = iso
self.lat = GEO_COORDS[iso]["lat"]
self.lon = GEO_COORDS[iso]["lon"]
self.iso = self._set_iso(iso)
self.holidays = holidays.UnitedStates()
self.load = self.get_historical_load()
self.model_input = None
def init_holidays():
us_holidays = holidays.UnitedStates()
def is_holiday(day):
if day in us_holidays:
return True
else:
return False
return is_holiday
def get_animal_is_free_day(row):
week_day = calendar.day_name[row.datetime.weekday()]
us_holidays = holidays.UnitedStates()
holiday = row.datetime in us_holidays
if (week_day == "Sunday" or week_day == "Saturday" or holiday == True):
free_day = 1
else:
free_day = 0
return free_day
