def __init__(self, sample_buildings=500, modeling_hours=1000): # Inputs # random.seed(123) self.building_classes = 25 # building class self.total_buildings = 815032 # building parcels self.sample_buildings = sample_buildings self.doe_archetypes = 19 # building archetypes self.total_hours = 8784 # hours in a year self.modeling_hours = modeling_hours # hours to model # Load Files and store as np arrays self.m = pd.read_csv('/Users/jonathanroth/PycharmProjects/UBEM/M.csv').values[:, 1:].astype(float) # [self.building_classes x K] self.a = pd.read_csv('/Users/jonathanroth/PycharmProjects/UBEM/A.csv').values[:, 2:].astype(float) # [self.sample_buildings x self.building_classes] self.pluto_export = pd.read_csv('/Users/jonathanroth/PycharmProjects/UBEM/Pluto_export.csv') self.building_energy = self.pluto_export['Energy_kbtu'].values.astype(float) self.nyc_8784_electricity= pd.read_csv('/Users/jonathanroth/PycharmProjects/UBEM/NYC_8784_ElectricityUse.csv') self.city_electricity = self.nyc_8784_electricity['Load'].values.astype('float') self.city_electricity_scaled = self.city_electricity / np.mean(self.city_electricity[:self.total_hours]) self.doe_ref_buildings = pd.read_csv('/Users/jonathanroth/PycharmProjects/UBEM/DOE_RefBuildings.csv') self.temperature = self.doe_ref_buildings['Temperature'].values.astype(float).reshape((self.total_hours, 1)) self.cdd = self.doe_ref_buildings['Cooling_Degree_Hour'].values.astype(float).reshape((self.total_hours, 1)) self.date_time = self.doe_ref_buildings['Date_Time'].values.astype(str).reshape((self.total_hours, 1)) self.bday = np.array([isbday(self.date_time[i, 0][0:10], holidays=holidays.US()) for i in range(len(self.date_time))]).astype(int) # TODO # MANUAL SCALING and X matrix self.sf = 0.5 self.x = np.zeros([self.doe_archetypes, self.doe_archetypes, self.total_hours]).astype(float) for k in range(self.doe_archetypes): self.x[k, k, :] = self.doe_ref_buildings.values[:, k + 2] self.x[k, k, :] = self.sf * (self.x[k, k, :]) / np.mean(self.x[k, k, :self.total_hours]) + (1 - self.sf) print('Shape of X: ', np.shape(self.x))
def ReplaceWith(x,y): """ x is position of data, and y is corresponding date""" #first day replace with t-1 #first week replace with t-96 # all other t-n*96,n is earlier day of same type (business or holiday) if x==1: return 1 elif x>1 and x<96: return x-1 elif x>=96 and x<=96*7: return x-96 else: sim_day=1 nday=datetime.timedelta(days=1) while isbday(y, holidays=holidays.US())!=isbday(y-sim_day*nday, holidays=holidays.US()): sim_day=sim_day+1 return x-96*sim_day
def get_earliest_available_apt_datetime(now_date_time): """ This function takes a datetime object and returns the earliest possible business datetime starting from the given datetime object. :param now_date_time: :return: """ if not isbday(now_date_time): earliest_available_date_time = now_date_time + BDay(1) earliest_available_date_time = earliest_available_date_time.replace( hour=15, minute=0, second=0, microsecond=0) else: current_time = datetime.time(hour=now_date_time.hour, minute=now_date_time.minute, second=now_date_time.second, microsecond=now_date_time.microsecond) if current_time > END_OF_BUSINESS_TIMESTAMP: earliest_available_date_time = now_date_time + BDay(1) earliest_available_date_time = earliest_available_date_time.replace( hour=15, minute=0, second=0, microsecond=0) elif current_time < START_OF_BUSINESS_TIMESTAMP: earliest_available_date_time = now_date_time.replace(hour=15, minute=0, second=0, microsecond=0) else: earliest_available_date_time = now_date_time return earliest_available_date_time
def get_start_date(self, past_months): """ Returns the date user wants to start collecting information. :param past_months: number of months back user wants to research """ current_date = list(self.stock_dates.keys())[ 0] # Gets most current day of the stock market year = int(current_date[:4]) month = int(current_date[5:7]) day = int(current_date[8:10]) begin_date = date(year, month, day) - relativedelta(months=past_months) year = begin_date.year month = begin_date.month day = begin_date.day # Changes single digit to double. if len(str(month)) is 1: month = "0" + str(month) if len(str(day)) is 1: day = "0" + str(day) # Date changes format. start_date = str(year) + "-" + str(month) + "-" + str(day) # Checks to make sure start date is a business day, if not moves to next business day. if not isbday(start_date, holidays=holidays.US()): start_date += relativedelta(bdays=+1, holidays=holidays.US()) return str(start_date)[:10]
def generate_date_based_cols(data, date_col): dates = list( map(lambda x: datetime.strptime(str(x), '%Y%m%d'), data[date_col])) data['DayOfMonth'] = list(map(lambda x: x.day, dates)) data['DayOfWeek'] = list(map(lambda x: x.isoweekday(), dates)) data['Month'] = list(map(lambda x: x.month, dates)) data['isBusinessDay'] = list(map(lambda x: 1 * isbday(x), dates)) return data
def updTime(self, inc): ''' Increases the time by inc amount Changes both self.time and self.dataStore.time ''' # Business day support prevDate = datetime.fromtimestamp(self.time).date() nextTimeHour = datetime.fromtimestamp(self.time + inc).time().hour nextDate = datetime.fromtimestamp(self.time + inc).date() secInDay = 86400 if (debug): print 'Time before:', self.time, self.dataStore.time print 'Hour after increment:', nextTimeHour print 'Time right now:', datetime.fromtimestamp(self.time).time() if (not (nextTimeHour >= 9 and nextTimeHour < 16) or (not isbday(nextDate))): if ((nextTimeHour >= 16) or (not isbday(nextDate))): self.time = dt2ut( datetime.fromtimestamp(self.time) + (1) * BDay()) self.time = dt2ut( datetime.fromtimestamp(self.time).replace(hour=3, minute=30, second=0, microsecond=0)) self.dataStore.time = self.time self.nextLogTime = self.time if (debug): print( datetime.fromtimestamp(self.time).date(), datetime.fromtimestamp(self.time).time()) else: self.time += inc self.dataStore.time += inc self.nextLogTime += inc if (debug): print( datetime.fromtimestamp(self.time).date(), datetime.fromtimestamp(self.time).time()) if (debug): print 'Time After:', self.time, self.dataStore.time
def callback3(): global last_day_events, list2, list3 date_entry.config(state='disabled') date_submit.config(state='disabled') case_num.config(state='disabled') new_lbl = tk.Label(root, text="Last Day Event(s): ") new_lbl.grid(row=17, column=1) list2 = tk.Listbox(root, height=0, width=75, selectmode='SINGLE') list2.bind( '<<ListboxSelect>>', on_select1, ) list2.grid(row=18, column=1) list3_lbl = tk.Label(root, text="Last Day Date") list3_lbl.grid(row=17, column=2) list3 = tk.Listbox(root, height=0, width=20, selectmode='SINGLE') list3.bind('<<ListboxSelect>>', on_select2) list3.grid(row=18, column=2) for item in last_day_events: list2.insert('end', item) date = date_entry_var.get() us_holidays = holidays.UnitedStates() for item in num_days: if (item < 7): d = datetime.datetime.strptime( date, '%m/%d/%Y') + relativedelta(bdays=+item) if (isbday(d, holidays=holidays.US()) == False): d = d + relativedelta(bdays=+1) list3.insert('end', d.strftime('%m/%d/%Y')) elif (isbday(d, holidays=holidays.US()) == True): list3.insert('end', d.strftime('%m/%d/%Y')) elif (item >= 7): d = datetime.datetime.strptime( date, '%m/%d/%Y') + datetime.timedelta(days=item) if (isbday(d, holidays=holidays.US()) == False): d = d + relativedelta(bdays=+1) list3.insert('end', d.strftime('%m/%d/%Y')) elif (isbday(d, holidays=holidays.US()) == True): list3.insert('end', d.strftime('%m/%d/%Y'))
def is_quote_delayed(self, last_bar_time): dtnow = self.date_now() if bdateutil.isbday( dtnow, holidays=holidays.US()) and dtnow.hour > 8 and dtnow.hour < 13: if int(abs((dtnow - last_bar_time).total_seconds() / 60.0)) > self.args.delay: return True return False
def dealWith_business_day(Date): """ Deal with the date not in business. If the day is weekend, add 2 days to the date for getting a business day. """ date = dt.datetime.strptime(Date, '%Y-%m-%d') if not isbday(date): date += dt.timedelta(2) else: pass return date
def findLastClose(df): # Find the last business day dBefore = 1 while not isbday( (datetime.now() - timedelta(dBefore)), holidays=holidays.US()): dBefore += 1 # create new df for that day and reindex datestr = (datetime.now() - timedelta(dBefore)).strftime("%Y-%m-%d") df = df.loc[df["stamp"] == datestr] df.index = pd.RangeIndex(len(df.index)) return df.at[0, "close"]
def run(self): """ Application main() :return: """ if not bdateutil.isbday(datetime.datetime.now(), holidays=holidays.US()): self.log.info("Run is skipped due to non business day") return # Populating account positions operations = [] update_date = 'N/A' for acc_pos_dict in self.mongo_db['accounts_positions'].find({}): # Shrinking time of the timestamp update_date = datetime.datetime.combine(acc_pos_dict['date_now'].date(), datetime.time(0,0,0)) # 'date_now' - main timestamp of collection acc_pos_dict['date_now'] = update_date del acc_pos_dict['_id'] # Add MongoDB bulk operation operations.append(ReplaceOne( {'date_now': update_date, 'name': acc_pos_dict['name']}, acc_pos_dict, upsert=True, )) self.log.info("Last collection update date: {0}".format(update_date)) # Execute bulk upsert to Mongo pp = pprint.PrettyPrinter(indent=4) try: bulk_result = self.mongo_db['accounts_positions_archive'].bulk_write(operations, ordered=False) self.log.info("Bulk write result succeed: \n{0}".format(pp.pformat(bulk_result.bulk_api_result))) self.signal_app.send(MsgStatus("OK", "Positions archive created", notify=True, ) ) except BulkWriteError as exc: self.log.error("Bulk write error occured: {0}".format(pp.pformat(exc.details))) self.signal_app.send(MsgStatus("ERROR", "Positions archive error while writing to MongoDB", notify=True, ) )
def Homework1(self, tickerList, weight, notional, startDate=None, endDate=None, historicalWindow=252, intervals=[0.995, 0.99, 0.98, 0.975, 0.95]): if (startDate == None and endDate == None): today = dt.date.today() today = today - dt.timedelta(days=1) # Get the latest business day while (isbday(today, holidays=holidays.US()) == False): today = today - dt.timedelta(days=1) endDate = today startDate = today + relativedelta(bdays=-historicalWindow) result = {} paraDict = {} hisDict = {} weight = np.array(weight) priceTabel = self.dataSource.getPriceTable(tickerList, startDate.isoformat(), endDate.isoformat()) while (len(priceTabel) < historicalWindow): startDate = startDate - dt.timedelta(historicalWindow - len(priceTabel)) priceTabel = self.dataSource.getPriceTable(tickerList, startDate.isoformat(), endDate.isoformat()) parametric = ValueAtRisk(0.95, priceTabel.as_matrix(), weight) historical = HistoricalVaR(0.95, priceTabel.as_matrix(), weight) for interval in intervals: parametric.setCI(interval) historical.setCI(interval) key = interval # the 1 here means daily var paraDict[key] = parametric.var(marketValue=notional, window=1) hisDict[key] = historical.var(marketValue=notional, window=historicalWindow) result['Parametric'] = paraDict result['Historical'] = hisDict result['Cov-Var Matrix'] = parametric.covMatrix() return result
def main(): if isbday(datetime.datetime.now()): initial = 17231.72 cash = 10.03 stocks = ['SMTPC.PA', 'DG.PA', 'NXI.PA', 'COX.PA', 'ABCA.PA', 'IPH.PA'] pru = [33.64054, 65.31130, 47.38463, 11.37695, 6.03034, 10.81] nbaction = [148, 61, 84, 46, 673, 64] my_port = {'Stock': stocks, 'PRU': pru, 'Nb Action': nbaction} my_port = updatePortfolio(initial, my_port) updateStocksCsv('export_stocks.csv', my_port) account = computePortfolioYield(my_port, cash, initial) updateAccountCsv('export_account.csv', account) try: plotPortfolioVsIndexes('export_account.csv') except: print("Erreur") exit(1) else: print("Jour non-ouvré") exit(0)
def featureCreation(feed, window, h, grouper, dataDir, apiDic, r_id=None, longestfeed=False): import pandas as pd from bdateutil import isbday import holidays from sklearn.decomposition import PCA feed = pd.DataFrame(feed) r_lat_long = apiDic.loc[(apiDic['id'] == int(r_id)), 'lat_long'][1] # Quarter of hour counter = 0 array = [] for i in pd.date_range('00:00', '23:45', freq=grouper): feed.loc[(feed.index.hour == i.hour) & (feed.index.minute == i.minute), grouper] = counter array.append(feed.loc[feed[grouper] == counter].values) counter += 1 # Hour of day feed['hourofday'] = feed.index.hour # Day of week feed['dayofweek'] = feed.index.dayofweek # Month feed['month'] = feed.index.month # Working day f = np.vectorize(lambda x: isbday( x, holidays=holidays.UK(years=[2013, 2014, 2015, 2016, 2017]))) feed['isworkingday'] = f(feed.index.date) # Weather data weather = pd.DataFrame.from_csv( os.path.join(dataDir, 'WEATHER_DATA', '%s.csv' % r_lat_long.replace(" ", ""))) # Converting text date into datetime weather['cleandate'] = weather['utcdate'].apply(lambda x: evalDate(x)) weather.index = weather['cleandate'] # Deleting irrelevant columns if 'date' in weather.columns: del weather['date'] if 'date.1' in weather.columns: del weather['date.1'] if 'utcdate' in weather.columns: del weather['utcdate'] if 'Unnamed: 0' in weather.columns: del weather['Unnamed: 0'] # Droping duplicates weather = weather.drop_duplicates(subset='cleandate') weather = weather.reindex( pd.date_range(weather['cleandate'].min(), weather['cleandate'].max(), freq=grouper)) #, method='backfill') weather = weather.loc[:, ('conds', 'dewptm', 'fog', 'hail', 'hum', 'precipm', 'pressurem', 'rain', 'snow', 'tempm', 'thunder', 'wdire', 'wgustm', 'windchillm', 'wspdm')] weather.loc[:, 'conds'] = weather.loc[:, 'conds'].fillna('Unknown') weather.loc[:, 'wdire'] = weather.loc[:, 'wdire'].fillna('Variable') le = le2 = preprocessing.LabelEncoder() le.fit(weather['conds']) weather['conds'] = le.transform(weather['conds']) le2.fit(weather['wdire']) weather['wdire'] = le2.transform(weather['wdire']) weather.replace([-9999.0, -999.0], [np.nan, np.nan], inplace=True) weather.loc[:, ('precipm', 'wgustm')] = weather.loc[:, ('precipm', 'wgustm')].fillna(0) weather.windchillm = weather.windchillm.fillna(weather.tempm) weather = weather.interpolate() if (weather.index.min() < feed.index.min()): if (weather.index.max() < feed.index.max()): weather = weather.ix[feed.index.min():, :] feed = feed.ix[:weather.index.max(), :] else: weather = weather.ix[feed.index.min():feed.index.max(), :] else: if (weather.index.max() < feed.index.max()): feed = feed.ix[weather.index.min():weather.index.max(), :] else: feed = feed.ix[weather.index.min():, :] weather = weather.ix[:feed.index.max(), :] features, response = mlf.ts_to_mimo(feed.ix[:, 0], window, h) n_factors = 1 pca = PCA(n_components=n_factors) pca.fit(weather) while (pca.explained_variance_ratio_.sum() < 0.99): n_factors += 1 pca = PCA(n_components=n_factors) pca.fit(weather) reduced = pd.DataFrame(pca.transform(weather)) c = np.zeros((features.shape[0], (h * len(reduced.columns)))) for column in range(len(reduced.columns)): c[:, (column * h):((1 + column) * h)] = mlf.weather_to_mimo( reduced.ix[:, column], window, h) features = np.concatenate((feed.ix[(window + h - 1):, ('isworkingday', grouper, 'hourofday', 'dayofweek', 'month')], c, features), axis=1) print('Features created') return (features, response)
def verify_price(symbol_id, y_data, q_data): """ Verifies the OHLCAV data from Quandl and Yahoo for a given company, accounting for missing dates of data, missing entries of data (e.g. No closing price), and differences in precision. If one of the vendors is missing a date, we use the other vendors data. If both both vendors have different values for the same data point, we assign it zero. All zero entries will be reevaluated in the spike script. :param symbol_id: Used to identify the company :param y_data: OHLCAV Dataframe from Yahoo Finance :param q_data: OHLCAV Dataframe from Quandl :return: Dataframe of the cross referenced prices from Yahoo and Quandl """ number_of_data_points = 0 number_of_days = 0 invalid_data_points = 0 verified_prices = pd.DataFrame(columns=['price_date', 'created_date', 'last_updated_date', 'open_price', 'high_price', 'low_price', 'close_price', 'adj_close_price', 'volume']) # Gets the earliest and latest date. Used to handle the edge case # where we might have one vendors range of data differ from the others start_date, last_date = get_date_ranges(y_data, q_data) # Iterate through the dates, ignoring non-business days for date in pd.bdate_range(start_date, last_date): # Make sure to exclude American holidays if isbday(date, holidays=holidays.US()): zero_counter = 0 number_of_data_points += 4 number_of_days += 1 print "Processing prices for: " + date.strftime("%Y-%m-%d") new_data_row = [symbol_id, date, timestamp, timestamp] y_daily_price = None q_daily_price = None # Convert Pandas Timestamp to a Date (YYYY-mm-DD) date = date.to_datetime().date() # Only pull the OHLCAV data if it is exists in the Dataframe if isinstance(y_data, pd.DataFrame): if not (y_data.loc[y_data['price_date'] == date]).empty: y_daily_price = y_data.loc[y_data['price_date'] == date] if isinstance(q_data, pd.DataFrame): if not (q_data.loc[q_data['price_date'] == date]).empty: q_daily_price = q_data.loc[q_data['price_date'] == date] # Determine if either vendor is missing data temp_prices = check_date(date, y_daily_price, q_daily_price) # Case 1: Only one vendor had data for the specific date, so we will use that vendors data if isinstance(temp_prices, pd.DataFrame): ohlcav_data_row = format_data(temp_prices) new_data_row.extend(ohlcav_data_row) zero_counter = 4 # Case 2: Neither vendor had data, so we assign zeros to all values for this date elif not temp_prices: print "Neither Vendor had data for " + date.strftime( "%Y-%m-%d") + " so we will fill the days price data with zeros" new_data_row.extend([0, 0, 0, 0, 0, 0]) # Excluding Volume and Adjusted Close zero_counter = 4 # Case 3: Compare the OHLCAV data else: yOHLCAV = format_data(y_daily_price) qOHLCAV = format_data(q_daily_price) data_row, zero_counter = compare_price_data(yOHLCAV, qOHLCAV) new_data_row.extend(data_row) # Create dataframe row from list of values price_df = pd.DataFrame(data=[new_data_row], columns=['symbol_id', 'price_date', 'created_date', 'last_updated_date', 'open_price', 'high_price', 'low_price', 'close_price', 'adj_close_price', 'volume']) # Add the new price data for the specific date to our dataframe verified_prices = verified_prices.append(price_df, ignore_index=True) invalid_data_points += zero_counter else: print str(date.strftime("%Y-%m-%d")) + " is a US holiday!" stats_and_data = [verified_prices, number_of_data_points, invalid_data_points, number_of_days] return stats_and_data
total_tickers = 0 total_days = 0 """Parameters to use to gather price data over a period of time """ # Format: 'YYYY-MM-DD' start = '1998-01-01' end = '2016-10-13' """Parameters to use to gather the most recent days price data """ #start = datetime.date.today().strftime("%Y-%m-%d") #end = datetime.date.today().strftime("%Y-%m-%d") # When just updating the most recent day, end the script if it is not a business day if start == end: if not isbday(start, holidays=holidays.US()): print "Not a business day, ending program." sys.exit() tickers = retrieve_db_tickers(con) # Collect data for each company for t in tickers: total_tickers += 1 print "Cleaning price data for ticker: " + t[1] # Gather initial datasets from both vendors yahoo_data = retrieve_price_data(con, t[1], 1, start, end) quandl_data = retrieve_price_data(con, t[1], 2, start, end) if isinstance(yahoo_data, pd.DataFrame) or isinstance(quandl_data, pd.DataFrame): # Clean data and add to the database
def test_isbday(self): self.assertFalse(isbday(date(2014, 1, 4))) self.assertFalse(isbday("2014-01-04")) self.assertTrue(isbday(date(2014, 1, 1))) self.assertTrue(isbday("2014-01-01")) self.assertFalse(isbday(date(2014, 1, 1), holidays=holidays.US())) self.assertTrue(isbday(datetime(2014, 1, 1, 16, 30))) self.assertTrue(isbday(datetime(2014, 1, 1, 17, 30))) self.assertFalse(isbday(datetime(2014, 1, 1, 16, 30), holidays=holidays.US())) self.assertFalse(isbday(datetime(2014, 1, 1, 17, 30), holidays=holidays.US())) bdateutil.HOLIDAYS = holidays.Canada() self.assertFalse(isbday(date(2014, 7, 1))) self.assertTrue(isbday(date(2014, 7, 4))) self.assertFalse(isbday(date(2014, 1, 1))) self.assertTrue(isbday(date(2014, 7, 1), holidays=holidays.US())) self.assertFalse(isbday(date(2014, 7, 4), holidays=holidays.US())) bdateutil.HOLIDAYS = []
def getPriceTable(self, tickerList, startDate=None, endDate=None, localCheck=None, dateAscending=True, update=False): # Get the price series for multiple tickers # ----Input----- # tickerList: ticker name for multiple stocks # startDate: the start date of price series, the format is 'YYYY-MM-DD' # endDate: the end date of price series, the format is 'YYYY-MM-DD' # dateAscending: whether rank the price series by date ascending, the default value is true # localCheck: loading local csv file, check the existing data see whether we need to retrieve data from Yahoo. The local file should contain date as index. # update: whether to update local file # ----output---- # price series for single stock in pandas DataFrame format and use date as index if (endDate == None): endDate = dt.date.today() + relativedelta(bdays=-1, holidays=holidays.US()) else: endDate = dt.datetime.strptime(endDate, '%Y-%m-%d').date() if (startDate == None): startDate = endDate + relativedelta(bdays=-252, holidays=holidays.US()) else: startDate = dt.datetime.strptime(startDate, '%Y-%m-%d').date() if (startDate > endDate): raise Exception('startDate is later than endDate') if (isinstance(tickerList, str)): tickerList = [ tickerList, ] if (localCheck != None): try: localFile = pd.read_csv(localCheck, index_col='date', parse_dates=True) except: raise Exception('Read Local File Error') if (np.all([ticker in localFile.columns for ticker in tickerList]) == False): raise Exception( '''Local File Columns Doesn't Match Ticker List''') # Make sure it's business day if (isbday(startDate, holidays=holidays.US()) == False): startDateCehck = startDate + relativedelta(bdays=1) else: startDateCehck = startDate if (isbday(endDate, holidays=holidays.US()) == False): endDateCehck = endDate + relativedelta(bdays=-1) else: endDateCehck = endDate if (startDateCehck in localFile.index and endDateCehck in localFile.index): return localFile[startDate.isoformat():endDate.isoformat()] if (startDate < localFile.index[0].date()): readStart = startDate if (endDate <= localFile.index[-1].date()): readEnd = localFile.index[0].date() else: readEnd = endDate else: readStart = localFile.index[-1].date() readEnd = endDate tables = [] for ticker in tickerList: # print(ticker) tables.append( self.getPrice(ticker, readStart.isoformat(), readEnd.isoformat(), dateAscending=True)) missingComponents = pd.concat(tables, axis=1) localFile = pd.concat([localFile, missingComponents], axis=0).sort_index(ascending=True) localFile = localFile[~localFile.index.duplicated()] if (update): localFile.to_csv(localCheck, index=True) return localFile[startDate.isoformat():endDate.isoformat()] tables = [] for ticker in tickerList: tables.append( self.getPrice(ticker, startDate.isoformat(), endDate.isoformat(), dateAscending=True)) return pd.concat(tables, axis=1)
# -*- coding: utf-8 -*- # This script takes the JSON data handed to it from PHP # then parses it and sends it to IFTTT # # Pat O'Brien 4/25/2016 # import requests, sys, syslog, time, datetime, json import holidays from bdateutil import isbday today = (time.strftime("%m/%d/%Y")) businessDay = isbday(today, holidays=holidays.US()) now = datetime.datetime.now() iftttEventName = "YOUR EVENT NAME" iftttSecretKey = "YOUR KEY" data_raw = sys.argv[1] data = json.loads(data_raw) def logmsg(msg): print msg syslog.syslog(syslog.LOG_INFO, 'SmartThings Webhooks Notify: %s' % msg ) # Setup variables based on JSON keys if data.has_key("name"): name = data["name"] else: name = "" if data.has_key("display_name"): display_name = data["display_name"]
d = ast.literal_eval(x) df = df.append(d, ignore_index=True) #Creating a new column as a placeholder for now df['new'] = df['date'] + " " + df['time'] #converts dates to datetime df['date'] = pd.to_datetime(df['date']) df['new'] = pd.to_datetime(df['new']) #If after 4 pm add 1 to the date because it is now the next trading day for i in range(len(df['new'])): if df['new'][i].hour >= 16: df['new'][i] += timedelta(days=1) df['date'][i] += timedelta(days=1) #Check to see if date is on a business day #If it's not aka its on a saturday (no tweet news on sundays) add 2 for j in range(len(df['date'])): val = isbday( date(df['date'][j].year, df['date'][j].month, df['date'][j].day)) if val == False: df['date'][j] += timedelta(days=2) #Now dropping our place holder column df.drop(['new'], axis=1, inplace=True) df.head() #stock data imported from Yahoo start = datetime.datetime(2014, 1, 1) end = datetime.datetime(2019, 4, 1) tsla = web.DataReader('NFLX', 'yahoo', start, end) tsla['Returns'] = tsla['Close'].pct_change(1) #Getting rid of irrelevant data tsla.drop(['High', 'Low', 'Open', 'Adj Close', 'Volume'], axis=1, inplace=True) #New column called NYSE date which is = to the index because the data #Had the index as a date tsla['date'] = tsla.index
def excludeCompute(x,y): if(x['ClassName'] in y): print 'exclude ',x['ClassName'],' by setting nan' return np.nan else: return x['HeadCount'] #Data processing classAttendance['month'] = classAttendance['StartTimeLocal'] .apply(lambda x:x.strftime('%Y%m')) #year-month classAttendance['week'] = classAttendance['StartTimeLocal'] .apply(lambda x:x.strftime('%Y%V')) #GroupMon-Sun classAttendance['bday'] = classAttendance['StartTimeLocal'] .apply(lambda x:isbday( x, holidays=LOCALHOLIDAYS)) classAttendance['date'] = classAttendance['StartTimeLocal'] .apply(lambda x:x.strftime('%Y%m%d')) classAttendance['HourWeekGrp'] = classAttendance .apply(lambda x:getCONSTANT_CLASS_HOURWEEKGROUP( x['StartHour'], x['Weekday']), axis=1) classAttendance['WeekdayGrp'] = classAttendance['Weekday'] .apply(lambda x:getWeekdayGroup(x)) classAttendance['HourGrp'] = classAttendance['StartHour'] .apply(lambda x:getHourGroup(x)) #Remove invalid data classAttendance = classAttendance .loc[np.logical_not( (classAttendance['bday']==False) & (classAttendance['HeadCount']==0) )]
def test_isbday(self): self.assertFalse(isbday(date(2014, 1, 4))) self.assertFalse(isbday("2014-01-04")) self.assertTrue(isbday(date(2014, 1, 1))) self.assertTrue(isbday("2014-01-01")) self.assertFalse(isbday(date(2014, 1, 1), holidays=holidays.US())) self.assertTrue(isbday(datetime(2014, 1, 1, 16, 30))) self.assertTrue(isbday(datetime(2014, 1, 1, 17, 30))) self.assertFalse( isbday(datetime(2014, 1, 1, 16, 30), holidays=holidays.US())) self.assertFalse( isbday(datetime(2014, 1, 1, 17, 30), holidays=holidays.US())) isbday.holidays = holidays.US() self.assertFalse(isbday(date(2014, 1, 1))) self.assertFalse(isbday(date(2014, 7, 4))) self.assertTrue(isbday(date(2014, 7, 4), holidays=holidays.CA()))
def test_isbday(self): self.assertFalse(isbday(date(2014, 1, 4))) self.assertFalse(isbday("2014-01-04")) self.assertTrue(isbday(date(2014, 1, 1))) self.assertTrue(isbday("2014-01-01")) self.assertFalse(isbday(date(2014, 1, 1), holidays=holidays.US())) self.assertTrue(isbday(datetime(2014, 1, 1, 16, 30))) self.assertTrue(isbday(datetime(2014, 1, 1, 17, 30))) self.assertFalse(isbday(datetime(2014, 1, 1, 16, 30), holidays=holidays.US())) self.assertFalse(isbday(datetime(2014, 1, 1, 17, 30), holidays=holidays.US())) isbday.holidays = holidays.US() self.assertFalse(isbday(date(2014, 1, 1))) self.assertFalse(isbday(date(2014, 7, 4))) self.assertTrue(isbday(date(2014, 7, 4), holidays=holidays.CA()))
def isBDay(): today = datetime.now() return isbday(today, holidays=holidays.US())
def recentbday(date):#most recent business day that is not today date -=timedelta(1) while not isbday(date,holidays=holidays.US()): date -=timedelta(1) return date.strftime('%Y-%m-%d')