def spotprices(region, start_isodate, end_isodate):
start_date = pendulum.from_timestamp(int(start_isodate), tz=config.TZ)
end_date = pendulum.from_timestamp(int(end_isodate), tz=config.TZ)
print(start_date, end_date)
data = {}
if region == "ALL":
for region in ["NSW", "QLD", "TAS", "VIC", "SA"]:
search = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='AEMO_SPOT').order_by('date_time')
result = [p for p in search]
demand_search = Demand.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper()).order_by('date_time')
demand_result = [p for p in demand_search]
# print(region)
# print(demand_result)
data[region] = {
'spot':{
# 'dates':[ p.date_time for p in result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in result]
},
'demand':{
# 'dates':[ p.date_time for p in demand_result],
'demand':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.demand] for p in demand_result]
},
}
else:
search = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='AEMO_SPOT').order_by('date_time')
result = [p for p in search]
demand_search = Demand.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper()).order_by('date_time')
demand_result = [p for p in demand_search]
# print(region)
# print(demand_result)
data[region] = {
'spot':{
# 'dates':[ p.date_time for p in result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in result]
},
'demand':{
# 'dates':[ p.date_time for p in demand_result],
'demand':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.demand] for p in demand_result]
},
}
return jsonify(data)
p for p in participant_service.participant_metadata
if participant_service.participant_metadata[p]['state'] == state
]
for state in states:
for date in dates:
if not Price.objects(date_time=date, region=state, price_type='BASIC'):
print('Adding', state, date)
# Basic settlement.
rounded_time = pendulum.instance(date, tz=config.TZ)
if rounded_time.minute > 30:
rounded_time = rounded_time.start_of('hour').add(hours=1)
else:
rounded_time = rounded_time.start_of('hour').add(minutes=30)
print('rounded time', rounded_time)
demand = Demand.objects(date_time=rounded_time,
region=state).get().demand
print('demand', demand)
stack = BidStack.objects(trading_period=date).get()
all_bids = []
# Assemble a sortable bidstack
for name in stack.getParticipants():
if name in participants[state]:
bid = stack.getBid(name)
bid_list = [b for b in bid_to_list(bid) if b['volume'] > 0]
all_bids.extend(bid_list)
# Perform economic dispatch
sorted_bids = sorted(all_bids, key=lambda k: k['price'])
dates = BidStack.objects().distinct('trading_period')
dates = [date for date in dates if (pendulum.instance(date).minute==0 or pendulum.instance(date).minute==30)]
states = ['NSW', 'QLD', 'VIC', 'QLD', 'TAS']
participants = {}
for state in states:
participants[state] = [p for p in participant_service.participant_metadata if participant_service.participant_metadata[p]['state'] == state]
for state in states:
for date in dates:
if not Price.objects(date_time = date, region=state, price_type='TOTAL_INDUSTRIAL_COST_MIN_BID_ZERO'):
print("TOTAL_INDUSTRIAL_COST_MIN_BID_ZERO", date)
stack = BidStack.objects(trading_period=date).get()
demand = Demand.objects(date_time=date, region=state).get().demand
all_bids = []
lookup_duids = {}
lookup_participant_id = {}
# Assemble a sortable bidstack
for duid in stack.getParticipants():
if duid in participants[state]:
bid = stack.getBid(duid)
# bidder_ids[duid] = bid.bid_day_offer.PARTICIPANTID
participant_id = bid.bid_day_offer.PARTICIPANTID
if participant_id not in lookup_duids:
lookup_duids[participant_id] = []
lookup_duids[participant_id].append(duid)
lookup_participant_id[duid] = participant_id
bid_list = [b for b in bid_to_list(bid) if b['volume'] > 0]
def process_bidstacks(dt, timeseries={}):
""" Derives competition indicators from bids submitted by generators. """
# Get bidstacks for every time period.
bidstack = BidStack.objects.get(trading_period=dt)
# Filter based on hour so we don't process tonnes of them
timeseries[dt] = {} if not dt in timeseries else timeseries[dt]
# Grab demand data.
demand_req = Demand.objects(date_time=dt)
regional_demand = {d.region: d.demand for d in demand_req}
total_demand = int(
float(sum([regional_demand[region] for region in regional_demand])))
# Grab price data
price_req = Price.objects(date_time=dt, price_type='AEMO_SPOT')
regional_prices = {p.region: p.price for p in price_req}
# Get a dict of all the residual supply indices, augmented with max network flows.
network_residual_supply_indices = get_network_extended_residual_supply_indices(
bidstack, regional_demand)
network_extended_capacity_hhi = get_network_extended_capacity_hhi(
bidstack, regional_demand)
# Record price and demand.
timeseries[dt]['weighted_average_price'] = float(
sum([regional_prices[p] * regional_demand[p]
for p in regional_prices])) / float(total_demand)
timeseries[dt]['demand_ALL'] = total_demand
timeseries[dt]['price'] = regional_prices
for key in regional_demand:
timeseries[dt]['demand_' + key] = regional_demand[key]
for key in regional_prices:
timeseries[dt]['price_' + key] = regional_prices[key]
for state in config.STATES:
# Get a dict of all the bid-based market shares for this time period
generator_bid_shares = get_generator_bid_market_shares(bidstack, state)
# Get a dict of all the residual supply indices for this time period
residual_supply_indices = get_residual_supply_indices(
bidstack, total_demand, state)
# Get a dict of all the pivotal supplier indices for this time period.
pivotal_supplier_indices = get_pivotal_supplier_indices(
bidstack, total_demand, state)
# Loop through every state and analyse / record
if state != "ALL":
# Get a dict of all firm weighted offers
firm_weighted_offer_prices = get_firm_volume_weighted_offer_price(
bidstack, state)
for firm in firm_weighted_offer_prices:
timeseries[dt][firm.lower() + '_weighted_offer_price_' +
state] = firm_weighted_offer_prices[firm]
# Calculate average NERSI for all firms in the state.
for firm in network_residual_supply_indices[state]:
timeseries[dt][
firm.lower() + '_nersi_' +
state] = network_residual_supply_indices[state][firm]
timeseries[dt]['average_nersi_' + state] = float(
sum([
network_residual_supply_indices[state][firm]
for firm in network_residual_supply_indices[state]
])) / float(len(network_residual_supply_indices[state]))
timeseries[dt]['minimum_nersi_' + state] = min([
network_residual_supply_indices[state][firm]
for firm in network_residual_supply_indices[state]
])
# Record network-extended HHI
timeseries[dt]['nechhi_' +
state] = network_extended_capacity_hhi[state]
# Record results.
timeseries[dt]['hhi_bids_' + state] = get_hhi(generator_bid_shares)
timeseries[dt]['entropy_bids_' +
state] = get_entropy(generator_bid_shares)
timeseries[dt]['four_firm_concentration_ratio_bids_' +
state] = get_four_firm_concentration_ratio(
generator_bid_shares)
timeseries[dt]['average_rsi_' + state] = float(
sum([
residual_supply_indices[firm]
for firm in residual_supply_indices
])) / float(len([f for f in residual_supply_indices]))
timeseries[dt]['minimum_rsi_' + state] = min([
residual_supply_indices[firm] for firm in residual_supply_indices
])
timeseries[dt]['sum_psi_' + state] = sum([
pivotal_supplier_indices[firm] for firm in pivotal_supplier_indices
])
# Record RSI and PSI for each firm.
for firm in residual_supply_indices:
timeseries[dt][firm.lower() + '_rsi_' +
state] = residual_supply_indices[firm]
for firm in pivotal_supplier_indices:
timeseries[dt][firm.lower() + '_psi_' +
state] = pivotal_supplier_indices[firm]
return timeseries
def prices(region, start_isodate, end_isodate):
start_date = pendulum.from_timestamp(int(start_isodate), tz=config.TZ)
end_date = pendulum.from_timestamp(int(end_isodate), tz=config.TZ)
print(start_date, end_date)
search = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='AEMO_SPOT').order_by('date_time')
result = [p for p in search]
demand_search = Demand.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper()).order_by('date_time')
demand_result = [p for p in demand_search]
basic_dispatch_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='BASIC').order_by('date_time')
basic_dispatch_result = [p for p in basic_dispatch_price]
basic_min_bid_zero_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='BASIC_MIN_BID_ZERO').order_by('date_time')
basic_min_bid_zero_result = [p for p in basic_min_bid_zero_price]
lmp_dispatch_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='LMP').order_by('date_time')
lmp_dispatch_result = [p for p in lmp_dispatch_price]
basic_vcg_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='BASIC_VCG').order_by('date_time')
basic_vcg_result = [p for p in basic_vcg_price]
basic_vcg_min_bid_zero_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='BASIC_VCG_MIN_BID_ZERO').order_by('date_time')
basic_vcg_min_bid_zero_result = [p for p in basic_vcg_min_bid_zero_price]
lmp_vcg_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='LMP_VCG').order_by('date_time')
lmp_vcg_result = [p for p in lmp_vcg_price]
lmp_vcg_min_bid_zero_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='LMP_VCG_MIN_BID_ZERO').order_by('date_time')
lmp_vcg_min_bid_zero_result = [p for p in lmp_vcg_min_bid_zero_price]
total_industrial_cost_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='TOTAL_INDUSTRIAL_COST').order_by('date_time')
total_industrial_cost_result = [p for p in total_industrial_cost_price]
total_industrial_cost_min_bid_zero_price = Price.objects(date_time__gte=start_date, date_time__lte=end_date, region=region.upper(), price_type='TOTAL_INDUSTRIAL_COST_MIN_BID_ZERO').order_by('date_time')
total_industrial_cost_min_bid_zero_result = [p for p in total_industrial_cost_min_bid_zero_price]
return jsonify({
'spot':{
# 'dates':[ p.date_time for p in result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in result]
},
'demand':{
# 'dates':[ p.date_time for p in demand_result],
'demand':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.demand] for p in demand_result]
},
'basic_dispatch':{
# 'dates':[ p.date_time for p in basic_dispatch_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in basic_dispatch_result]
},
'basic_min_bid_zero':{
# 'dates':[ p.date_time for p in basic_min_bid_zero_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in basic_min_bid_zero_result]
},
'lmp_dispatch':{
# 'dates':[ p.date_time for p in lmp_dispatch_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, p.price] for p in lmp_dispatch_result]
},
'basic_vcg_dispatch':{
# 'dates':[ p.date_time for p in basic_vcg_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in basic_vcg_result]
},
'basic_vcg_min_bid_zero_dispatch':{
# 'dates':[ p.date_time for p in basic_vcg_min_bid_zero_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in basic_vcg_min_bid_zero_result]
},
'lmp_vcg_dispatch':{
# 'dates':[ p.date_time for p in lmp_vcg_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in lmp_vcg_result]
},
'lmp_vcg_min_bid_zero_dispatch':{
# 'dates':[ p.date_time for p in lmp_vcg_min_bid_zero_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in lmp_vcg_min_bid_zero_result]
},
'total_industrial_cost':{
# 'dates':[ p.date_time for p in lmp_vcg_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in total_industrial_cost_result]
},
'total_industrial_cost_min_bid_zero':{
# 'dates':[ p.date_time for p in lmp_vcg_result],
'prices':[ [pendulum.instance(p.date_time).timestamp() * 1000, round(p.price, 2)] for p in total_industrial_cost_min_bid_zero_result]
},
}
)
def process_bidstacks(start_date, end_date, timeseries={}):
""" Derives competition indicators from bids submitted by generators. """
print("Processing Bidstacks")
# Get bidstacks for every time period.
query = BidStack.objects(trading_period__gte=start_date,
trading_period__lte=end_date).fields(
trading_period=1, id=1)
print('Retrieved Bidstacks')
i = 0
for bidstack in query:
# Get the trading period label.
dt = pendulum.instance(bidstack.trading_period)
# if(dt.hour == 12 and dt.minute == 0 and dt.day %5 == 0):
# Filter based on hour so we don't process tonnes of them
if (dt.hour in RESEARCH_HOURS and dt.minute == 0):
timeseries[dt] = {} if not dt in timeseries else timeseries[dt]
print("Bid Analysis", dt)
# Grab the bids and order in economic dispatch order.
bidstack = BidStack.objects.get(id=bidstack.id)
# simple_bids, srmc_bids, lrmc_bids = settle(bidstack)
# print("Got bid stacks.")
# Grab demand data.
demand_req = Demand.objects(date_time=dt)
regional_demand = {d.region: d.demand for d in demand_req}
total_demand = int(
float(
sum([
regional_demand[region] for region in regional_demand
])))
# Grab price data
price_req = Price.objects(date_time=dt, price_type='AEMO_SPOT')
regional_prices = {p.region: p.price for p in price_req}
# print(regional_prices)
weighted_average_price = float(
sum([
regional_prices[p] * regional_demand[p]
for p in regional_prices
])) / float(total_demand)
# print(weighted_average_price)
# Get a dict of all the residual supply indices, augmented with max network flows.
network_residual_supply_indices = get_network_extended_residual_supply_indices(
bidstack, regional_demand)
timeseries[dt]['weighted_average_price'] = weighted_average_price
timeseries[dt]['demand_ALL'] = total_demand
timeseries[dt]['datetime'] = dt
timeseries[dt]['price'] = regional_prices
for key in regional_demand:
timeseries[dt]['demand_' + key] = regional_demand[key]
for key in regional_prices:
timeseries[dt]['price_' + key] = regional_prices[key]
for state in STATES:
# Get a dict of all the bid-based market shares for this time period
generator_bid_shares = get_generator_bid_market_shares(
bidstack, state)
# Get a dict of all the residual supply indices for this time period
residual_supply_indices = get_residual_supply_indices(
bidstack, total_demand, state)
# Get a dict of all the pivotal supplier indices for this time period.
pivotal_supplier_indices = get_pivotal_supplier_indices(
bidstack, total_demand, state)
if state != "ALL":
# Get a dict of all firm weighted offers
firm_weighted_offer_prices = get_firm_volume_weighted_offer_price(
bidstack, state)
for firm in firm_weighted_offer_prices:
timeseries[dt][
firm.lower() + '_weighted_offer_price_' +
state] = firm_weighted_offer_prices[firm]
# Calculate average NERSI for all firms in the state.
for firm in network_residual_supply_indices[state]:
timeseries[dt][
firm.lower() + '_nersi_' +
state] = network_residual_supply_indices[state][
firm]
timeseries[dt]['average_nersi_' + state] = float(
sum([
network_residual_supply_indices[state][firm]
for firm in network_residual_supply_indices[state]
])) / float(len(
network_residual_supply_indices[state]))
timeseries[dt]['minimum_nersi_' + state] = min([
network_residual_supply_indices[state][firm]
for firm in network_residual_supply_indices[state]
])
# Record results.
timeseries[dt]['hhi_bids_' +
state] = get_hhi(generator_bid_shares)
timeseries[dt]['entropy_bids_' +
state] = get_entropy(generator_bid_shares)
timeseries[dt]['four_firm_concentration_ratio_bids_' +
state] = get_four_firm_concentration_ratio(
generator_bid_shares)
timeseries[dt]['average_rsi_' + state] = float(
sum([
residual_supply_indices[firm]
for firm in residual_supply_indices
])) / float(len([f for f in residual_supply_indices]))
timeseries[dt]['minimum_rsi_' + state] = min([
residual_supply_indices[firm]
for firm in residual_supply_indices
])
timeseries[dt]['sum_psi_' + state] = sum([
pivotal_supplier_indices[firm]
for firm in pivotal_supplier_indices
])
for firm in residual_supply_indices:
timeseries[dt][firm.lower() + '_rsi_' +
state] = residual_supply_indices[firm]
for firm in pivotal_supplier_indices:
timeseries[dt][firm.lower() + '_psi_' +
state] = pivotal_supplier_indices[firm]
print("Finished Processing Bidstack")
return timeseries
def process_bidstacks(start_date, end_date):
print("Processing Bidstacks")
request = BidStack.objects(trading_period__gte=start_date,
trading_period__lte=end_date).fields(
trading_period=1, id=1)
print('Retrieved Bidstacks')
i = 0
timeseries = {}
for bidstack in request:
# Get the trading period label.
dt = pendulum.instance(bidstack.trading_period)
# if(dt.hour == 12 and dt.minute == 0 and dt.day %5 == 0):
if (dt.hour in [0, 6, 12, 18, 24] and dt.minute == 0):
print(dt)
# Grab the bids and order in economic dispatch order.
bidstack = BidStack.objects.get(id=bidstack.id)
simple_bids, srmc_bids, lrmc_bids = settle(bidstack)
# print("Got bid stacks.")
# Grab demand data.
demand_req = Demand.objects(date_time=dt)
regional_demand = {d.region: d.demand for d in demand_req}
total_demand = int(
float(
sum([
regional_demand[region] for region in regional_demand
])))
# Grab price data
price_req = Price.objects(date_time=dt, price_type='AEMO_SPOT')
regional_prices = {p.region: p.price for p in price_req}
weighted_average_price = float(
sum([
regional_prices[p] * regional_demand[p]
for p in regional_prices
])) / float(total_demand)
# Grab the representative strings.
bids_string = get_representative_string(simple_bids, total_demand)
srmc_string = get_representative_string(srmc_bids, total_demand)
lrmc_string = get_representative_string(lrmc_bids, total_demand)
# print("Got Representative Strings")
metrics = {
'srmc_string_comp':
compare_representative_strings(bids_string, srmc_string),
'lrmc_string_comp':
compare_representative_strings(bids_string, lrmc_string),
'srmc_fraction_MWh_different':
get_fraction_MWh_different(bids_string, srmc_string,
total_demand),
'lrmc_fraction_MWh_different':
get_fraction_MWh_different(bids_string, lrmc_string,
total_demand),
'datetime':
dt,
'price':
regional_prices,
'weighted_average_price':
weighted_average_price,
'total_demand':
total_demand,
'regional_demand':
regional_demand,
'strike': {
'simple': get_strike_price(simple_bids, total_demand),
'srmc': get_strike_price(srmc_bids, total_demand),
'lrmc': get_strike_price(lrmc_bids, total_demand),
}
}
timeseries[dt] = metrics
print("Finished Processing Bidstack")
return timeseries