def test_handle_options_set(self):
"""Can set options"""
bc = BaseClient()
bc.handle_options(test="a", another=20)
self.assertEqual(bc.options["test"], "a")
self.assertEqual(bc.options["another"], 20)
self.assertFalse(bc.options["sliceable"])
def test_handle_options_set(self):
"""Can set options"""
bc = BaseClient()
bc.handle_options(test='a', another=20)
self.assertEqual(bc.options['test'], 'a')
self.assertEqual(bc.options['another'], 20)
self.assertFalse(bc.options['sliceable'])
def test_handle_options_latest(self):
"""Correct processing of time-related options for latest"""
bc = BaseClient()
bc.handle_options(latest=True)
self.assertTrue(bc.options['latest'])
self.assertFalse(bc.options['sliceable'])
self.assertFalse(bc.options['forecast'])
def test_handle_options_twice(self):
"""Overwrite options on second call"""
bc = BaseClient()
bc.handle_options(forecast=True)
self.assertTrue(bc.options['forecast'])
bc.handle_options(yesterday=True)
self.assertFalse(bc.options['forecast'])
def test_handle_options_future(self):
"""Correct processing of time-related options for future start and end times"""
bc = BaseClient()
bc.handle_options(start_at='2100-01-01', end_at='2100-02-01')
self.assertTrue(bc.options['sliceable'])
self.assertEqual(bc.options['start_at'], datetime(2100, 1, 1, 0, tzinfo=pytz.utc))
self.assertEqual(bc.options['end_at'], datetime(2100, 2, 1, 0, tzinfo=pytz.utc))
self.assertTrue(bc.options['forecast'])
def test_handle_options_past(self):
"""Correct processing of time-related options for historical start and end times"""
bc = BaseClient()
bc.handle_options(start_at="2014-01-01", end_at="2014-02-01")
self.assertTrue(bc.options["sliceable"])
self.assertEqual(bc.options["start_at"], datetime(2014, 1, 1, 0, tzinfo=pytz.utc))
self.assertEqual(bc.options["end_at"], datetime(2014, 2, 1, 0, tzinfo=pytz.utc))
self.assertFalse(bc.options["forecast"])
def test_handle_options_forecast(self):
"""Correct auto-setup of time-related options for forecast"""
bc = BaseClient()
bc.handle_options(forecast=True)
self.assertTrue(bc.options['sliceable'])
local_now = pytz.utc.localize(datetime.utcnow()).astimezone(pytz.utc).replace(microsecond=0)
self.assertEqual(bc.options['start_at'], local_now)
self.assertEqual(bc.options['end_at'], local_now + timedelta(days=2))
self.assertTrue(bc.options['forecast'])
def test_handle_options_forecast(self):
"""Correct auto-setup of time-related options for forecast"""
bc = BaseClient()
bc.handle_options(forecast=True)
self.assertTrue(bc.options['sliceable'])
local_now = pytz.utc.localize(datetime.utcnow()).astimezone(pytz.utc)
self.assertEqual(bc.options['start_at'], datetime(local_now.year, local_now.month, local_now.day, 0, tzinfo=pytz.utc))
self.assertEqual(bc.options['end_at'], datetime(local_now.year, local_now.month, local_now.day+2, 0, tzinfo=pytz.utc))
self.assertTrue(bc.options['forecast'])
def test_handle_options_yesterday(self):
"""Correct auto-setup of time-related options for yesterday"""
bc = BaseClient()
bc.handle_options(yesterday=True)
self.assertTrue(bc.options['sliceable'])
local_now = pytz.utc.localize(datetime.utcnow()).astimezone(pytz.utc)
midnight_today = datetime(local_now.year, local_now.month, local_now.day, 0, tzinfo=pytz.utc)
self.assertEqual(bc.options['start_at'], midnight_today - timedelta(days=1))
self.assertEqual(bc.options['end_at'], midnight_today)
self.assertFalse(bc.options['forecast'])
def test_handle_options_forecast(self):
"""Correct auto-setup of time-related options for forecast"""
bc = BaseClient()
bc.handle_options(forecast=True)
self.assertTrue(bc.options["sliceable"])
local_now = pytz.utc.localize(datetime.utcnow()).astimezone(pytz.utc)
midnight_today = datetime(local_now.year, local_now.month, local_now.day, 0, tzinfo=pytz.utc)
self.assertEqual(bc.options["start_at"], midnight_today)
self.assertEqual(bc.options["end_at"], midnight_today + timedelta(days=2))
self.assertTrue(bc.options["forecast"])
def setUp(self):
# set up expected values from base client
bc = BaseClient()
self.MARKET_CHOICES = bc.MARKET_CHOICES
self.FREQUENCY_CHOICES = bc.FREQUENCY_CHOICES
# set up other expected values
self.BA_CHOICES = BALANCING_AUTHORITIES.keys()
def test_freq_choices(self):
"""Frequency choices have expected values."""
bc = BaseClient()
self.assertEqual('1hr', bc.FREQUENCY_CHOICES.hourly)
self.assertEqual('5m', bc.FREQUENCY_CHOICES.fivemin)
self.assertEqual('10m', bc.FREQUENCY_CHOICES.tenmin)
self.assertEqual('n/a', bc.FREQUENCY_CHOICES.na)
def setUp(self):
# set up expected values from base client
bc = BaseClient()
self.MARKET_CHOICES = bc.MARKET_CHOICES
self.FREQUENCY_CHOICES = bc.FREQUENCY_CHOICES
# set up other expected values
self.BA_CHOICES = ['ISONE', 'MISO', 'SPP',
'BCH', 'BPA', 'CAISO', 'ERCOT',
'PJM', 'NYISO', 'NEVP', 'SPPC']
def test_has_logger(self):
"""BaseClient has logger attribute that acts like logger"""
bc = BaseClient()
# attribute exists
logger = getattr(bc, 'logger', None)
self.assertIsNotNone(logger)
# can accept handler
handler = logging.StreamHandler()
logger.addHandler(handler)
def test_handle_options_twice(self):
"""Overwrite options on second call"""
bc = BaseClient()
bc.handle_options(forecast=True)
self.assertTrue(bc.options['forecast'])
bc.handle_options(yesterday=True)
self.assertFalse(bc.options['forecast'])
def test_slice_empty(self):
bc = BaseClient()
indf = pd.DataFrame()
outdf = bc.slice_times(indf, {'latest': True})
self.assertEqual(len(outdf), 0)
def test_bad_zipfile(self):
bc = BaseClient()
badzip = 'I am not a zipfile'
result = bc.unzip(badzip)
self.assertIsNone(result)
def get_data(self, start_at, end_at, product='rt5m'):
if product == 'rt5m':
product = 'rt'
elif product == 'dahr':
product = 'dam'
bs = BaseClient()
start_year = start_at.year
start_month = start_at.month
end_year = end_at.year
end_month = end_at.month
year = start_year
month = start_month
df_new = []
while year < end_year or month <= end_month:
dl_str = self.base_url + product + 'asp/' + str(
year) + "{0:0>2}".format(
month) + '01' + product + 'asp_csv.zip'
url = urlopen(dl_str)
zipfile = ZipFile(StringIO(url.read()))
print 'Parsing month starting {0}'.format(zipfile.namelist()[0])
for f in zipfile.namelist():
df = pd.read_csv(zipfile.open(f))
if product == 'rt':
if 'NYCA Regulation Movement ($/MW)' not in df.columns and ' NYCA Regulation Movement ($/MW)':
#add a reg movement category in
df.loc[:, 'reg_move'] = 0.
elif 'NYCA Regulation Movement ($/MW)' in df.columns:
df.rename(columns={
'NYCA Regulation Movement ($/MW)': 'reg_move'
},
inplace=True)
elif ' NYCA Regulation Movement ($/MW)' in df.columns:
df.rename(columns={
' NYCA Regulation Movement ($/MW)':
'reg_move'
},
inplace=True)
#deal with the east/west thing
if 'Time Zone' in df.columns:
check_col = 2
else:
check_col = 1
if df.columns[check_col][0:4] == 'East' or df.columns[
check_col][0:4] == 'West' or df.columns[check_col][
0:4] == 'SENY':
#Stupid older one
if product == 'rt':
#Make two reg_moves
df.rename(columns={'reg_move': 'East_reg_move'},
inplace=True)
df.loc[:, 'West_reg_move'] = df['East_reg_move']
#if df.columns[check_col][0:4] == 'SENY':
# df.loc[:,'SENY_reg_move'] = df['East_reg_move']
columns = {}
east_columns = ['Time Stamp']
west_columns = ['Time Stamp']
#if df.columns[check_col][0:4] == 'SENY':
# seny_columns = ['Time Stamp']
if 'Time Zone' in df.columns:
east_columns.append('Time Zone')
west_columns.append('Time Zone')
# if df.columns[check_col][0:4] == 'SENY':
# seny_columns.append('Time Zone')
for c in df.columns:
if c[0:4] == 'East':
east_columns.append(c)
elif c[0:4] == 'West':
west_columns.append(c)
# elif c[0:4] == 'SENY':
# seny_columns.append(c)
east_df = df.loc[:, east_columns]
west_df = df.loc[:, west_columns]
#if df.columns[check_col][0:4] == 'SENY':
# seny_df = df.loc[:,seny_columns]
east_df.loc[:, 'name'] = 'East'
west_df.loc[:, 'name'] = 'West'
#if df.columns[check_col][0:4] == 'SENY':
# seny_df.loc[:,'name'] = 'SENY'
column_names = ['datatime']
if 'Time Zone' in df.columns:
column_names.append('tz')
column_names.append('spin_10')
column_names.append('spin_10_non_synch')
column_names.append('op_res_30')
column_names.append('reg_cap')
if product == 'rt':
column_names.append('reg_move')
column_names.append('name')
east_df.columns = column_names
west_df.columns = column_names
#if df.columns[check_col][0:4] == 'SENY':
# print seny_df
# seny_df.columns = column_names
df = east_df.append(west_df)
df.loc[:, 'iso_node_id'] = -99
else:
#nice and zonal
columns = [
'datatime', 'tz', 'name', 'iso_node_id', 'spin_10',
'spin_10_non_synch', 'op_res_30', 'reg_cap'
]
if product == 'rt':
columns.append('reg_move')
df.columns = columns
df['datatime'] = df['datatime'].apply(self._parse_datatime)
if 'tz' in df.columns:
df.loc[df['tz'] == 'EDT',
'datatime'] = df.loc[df['tz'] == 'EDT',
'datatime'] + timedelta(
hours=4)
df.loc[df['tz'] == 'EST',
'datatime'] = df.loc[df['tz'] == 'EST',
'datatime'] + timedelta(
hours=5)
df.drop('tz', inplace=True, axis=1)
else:
utc_cls = NYISOClient()
df.set_index('datatime', inplace=True)
if month == 11:
print 'Month 11, going each name for stupid dst. Day {0}'.format(
df.index[0].day)
nodes = pd.unique(df['name'])
df_new_dst = None
for i in xrange(0, len(nodes)):
node = nodes[i]
df_temp = df[df['name'] == node]
df_temp.index = utc_cls.utcify_index(df_temp.index)
if df_new_dst is not None:
df_new_dst = df_new_dst.append(df_temp)
else:
df_new_dst = df_temp
df = df_new_dst
else:
df.index = utc_cls.utcify_index(df.index)
df.reset_index(inplace=True)
df_new.append(df)
if month < 12:
month += 1
else:
year += 1
month = 1
return df_new
def test_handle_options_set_forecast(self):
bc = BaseClient()
start = datetime(2020, 5, 26, 0, 0, tzinfo=pytz.utc)
bc.handle_options(start_at=start, end_at=start+timedelta(days=2))
self.assertTrue(bc.options['forecast'])
def test_handle_options_inverted_start_end(self):
"""Raises error if end before start"""
bc = BaseClient()
self.assertRaises(AssertionError, bc.handle_options, start_at='2014-02-01', end_at='2014-01-01')
def test_init(self):
"""init creates empty options dict"""
bc = BaseClient()
self.assertIsNotNone(bc)
self.assertEqual(len(bc.options.keys()), 0)
def test_timeout(self):
bc = BaseClient(timeout_seconds=30)
self.assertEqual(bc.timeout_seconds, 30)
def test_slice_empty(self):
bc = BaseClient()
indf = pd.DataFrame()
outdf = bc.slice_times(indf, {'latest': True})
self.assertEqual(len(outdf), 0)
def test_bad_zipfile(self):
bc = BaseClient()
badzip = 'I am not a zipfile'
result = bc.unzip(badzip)
self.assertIsNone(result)
def test_handle_options_set_forecast(self):
bc = BaseClient()
start = datetime(2020, 5, 26, 0, 0, tzinfo=pytz.utc)
bc.handle_options(start_at=start, end_at=start + timedelta(days=2))
self.assertTrue(bc.options['forecast'])
