def parse_config_item(self, item: Element):
"""
Parse a configuration item from xml
:param item: item to be parsed.
:return
"""
c_name = item.get("NAME")
c_value = item.text
if not c_name:
raise ValueError("The xml has a preference with no name")
if not c_value:
raise ValueError("The xml has a preference with no value")
self.name = c_name.lower()
self.value = c_value
c_type = item.get("TYPE")
if c_type:
self.type = ParseFormats.parse_type(c_type.lower())
else:
self.type = DataType.STRING
ParseFormats.parse_item(self.type, self.value)
def _load_control_data(self):
"""
Sets the control data defined in the configuration file
"""
self.logger.debug("Stating _load_control_data")
try:
self.use_ssl = ParseFormats.parse_bool(Config().get_config_param(
'Control', 'UseSSL'))
self.control_port = ParseFormats.parse_uint16(
Config().get_config_param('Control', 'ControlPort'))
self.log_on_connect = ParseFormats.parse_bool(
Config().get_config_param('Control', 'LogOnConnect'))
self.log_command = ParseFormats.parse_bool(
Config().get_config_param('Control', 'LogCommand'))
self.control_hosts = self.config['Control']['Access']['Host']
self.control_user, self.control_passwd = \
self.config['Control']['Access']['User'].split(':')
self.logger.debug("Ending _load_control_data")
except ValueError as verr:
self.logger.error(
"The value for control port{0} is not a valid number".format(
Config().get_config_param('Control', 'ControlPort')))
raise ValueError(
"The value for control port{0} is not a valid number".format(
Config().get_config_param('Control', 'ControlPort')))
def calculate_interval(self, option_name: str) -> Interval:
"""
Calculates the interval for a options record
:param option_name for which we want to create the interval.
:param last_stop datetime of the last stop for thebididng object.
:return: Interval created for the option
"""
duration = 0
interval = Interval()
fstart = self.get_option_value(option_name, "Start")
fstop = self.get_option_value(option_name, "Stop")
if fstart.value:
interval.start = ParseFormats.parse_time(fstart.value)
if not interval.start:
raise ValueError("Invalid start time {0}".format(fstart.value))
if fstop.value:
interval.stop = ParseFormats.parse_time(fstop.value)
if not interval.stop:
raise ValueError("Invalid stop time {0}".format(fstart.value))
# do we have a stop time defined that is in the past ?
if interval.stop and interval.stop <= datetime.now():
logger = log().get_logger()
logger.debug("Bidding object running time is already over")
if interval.start < datetime.now():
# start late tasks immediately
interval.start = datetime.now()
interval.duration = (interval.stop - interval.start).total_seconds()
return interval
def sort_bids_by_price(self,
bids: Dict[str, BiddingObject],
discriminatory_price: float = 0,
subsidy: float = 1) -> DefaultDict[float, list]:
"""
sort bids by price in descending order
:param bids: bids included for sorting
:param discriminatory_price price to applies subsidies
:param subsidy: value by which we multiply prices in order to give subsidies
:return:
"""
ordered_bids: DefaultDict[float, list] = defaultdict(list)
for bidding_object_key in bids:
bidding_object = bids[bidding_object_key]
elements = bidding_object.elements
for element_name in elements:
config_params = elements[element_name]
price = ParseFormats.parse_float(
config_params["unitprice"].value)
quantity = ParseFormats.parse_float(
config_params["quantity"].value)
alloc = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(), quantity,
price)
# applies the subsidy if it was given,
if discriminatory_price > 0:
if price < discriminatory_price:
price = price * subsidy
ordered_bids[price].append(alloc)
return ordered_bids
def _parse_global_options(self, item: Element) -> (dict, dict):
"""
Parse global options within the xml
:param item: Global Tag
:return: dictionary of configuration items.
list of actions.
"""
global_misc = {}
global_actions = {}
for sub_item in item.iterchildren():
if isinstance(sub_item.tag, str):
if sub_item.tag.lower() == "pref":
config_param = ConfigParam()
config_param.parse_config_item(sub_item)
global_misc[config_param.name] = config_param
if sub_item.tag.lower() == "action":
action = Action("", False, dict())
action.parse_action(sub_item)
default = sub_item.get("DEFAULT")
if not default:
raise ValueError(
"Auction Parser Error: missing name at line {0}".
format(str(sub_item.sourceline)))
else:
action.default_action = ParseFormats.parse_bool(
default)
global_actions[action.name] = action
return global_misc, global_actions
def increment_quantity_allocation(allocation: BiddingObject,
quantity: float):
"""
Increments the quantity assigned to an allocation
:param allocation: allocation to be incremented
:param quantity: quantity to increment
"""
elements = allocation.elements
# there is only one element
updated = False
for element_name in elements:
config_dict = elements[element_name]
# remove the field for updating quantities
field: ConfigParam = config_dict.pop('quantity')
# Insert again the field.
temp_qty = ParseFormats.parse_float(field.value)
temp_qty += quantity
fvalue = str(temp_qty)
field.value = fvalue
config_dict[field.name] = field
updated = True
break
if not updated:
raise ValueError(
"Field quantity was not included in the allocation")
def get_param_value(self, field_name: str, params: dict):
"""
Get the value of a parameter
:param field_name: name of the parameter
:param params: parameters
:return: value of the parameter.
"""
field_def = self.field_def_manager.get_field(field_name)
if field_def['type'] == DataType.DOUBLE:
return ParseFormats.parse_double(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.UINT32:
return ParseFormats.parse_ulong(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.STRING:
return params[field_name].get_exact_field_value()
elif field_def['type'] == DataType.UINT64:
return ParseFormats.parse_ulong(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.IPV4ADDR:
return ParseFormats.parse_ipaddress(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.IPV6ADDR:
return ParseFormats.parse_ipaddress(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.UINT8:
return ParseFormats.parse_uint8(
params[field_name].get_exact_field_value())
elif field_def['type'] == DataType.FLOAT:
return ParseFormats.parse_float(
params[field_name].get_exact_field_value())
else:
raise ValueError("Invalid type {0}".format(
field_def['type'].lower()))
def __init__(self, config_file_name: str):
self._pending_tasks_by_auction = {}
self.config = Config(config_file_name).get_config()
# Start Listening the web server application
loop = asyncio.get_event_loop()
self.app = Application(loop=loop)
# Gets the log file
log_file_name = self.config['DefaultLogFile']
self.logger = log(log_file_name).get_logger()
self.domain = ParseFormats.parse_int(Config().get_config_param(
'Main', 'Domain'))
self.immediate_start = ParseFormats.parse_bool(
Config().get_config_param('Main', 'ImmediateStart'))
self._load_main_data()
self._load_control_data()
def parse_allocation(self, object_key: IpapObjectKey, templates: list, data_records: list,
ipap_template_container: IpapTemplateContainer) -> Allocation:
"""
Parse an allocation object from the ipap_message
:param object_key: object key for the allocation that is going to be parsed.
:param templates: templates for the allocation
:param data_records: records for the allocation
:param ipap_template_container: template container where we have to include the templates.
:return: allocation object parsed.
"""
nbr_data_read = 0
nbr_option_read = 0
bidding_object_key = None
allocation_key = None
auction_key = None
status = None
data_misc = {}
data_template, opts_template = self.insert_auction_templates(object_key.get_object_type(),
templates, ipap_template_container)
opts = []
# Read data records
for data_record in data_records:
template_id = data_record.get_template_id()
# Read a data record for a data template
if template_id == data_template.get_template_id():
data_misc = self.read_record(data_template, data_record)
bidding_object_key = self.extract_param(data_misc, 'biddingobjectid').value
auction_key = self.extract_param(data_misc, 'auctionid').value
allocation_key = self.extract_param(data_misc, 'allocationid').value
status = self.extract_param(data_misc, 'status').value
nbr_data_read = nbr_data_read + 1
if template_id == opts_template.get_template_id():
opts_misc = self.read_record(opts_template, data_record)
opts.append(opts_misc)
nbr_option_read = nbr_option_read + 1
if nbr_data_read == 0:
raise ValueError("A data template was not given")
if nbr_option_read == 0:
raise ValueError("An option template was not given")
intervals = self.convert_options_interval(opts)
allocation = Allocation(auction_key, bidding_object_key, allocation_key, data_misc, intervals)
allocation.set_state(AuctioningObjectState(ParseFormats.parse_int(status.value)))
return allocation
def create_request(
self, bids: Dict[str,
BiddingObject]) -> (DefaultDict[float, list]):
"""
Creates allocation requests for low and high auctions. It promotes some high auction bid into the
low auction
:param self:
:param bids: bids competing in the auction
:return: allocations request in the auction.
"""
self.logger.debug("create request bids {0}".format(str(len(bids))))
# go through all high budget bids and pass some of their units as low auction requests.
auction_allocs: DefaultDict[float, list] = defaultdict(list)
for bidding_object_key in bids:
bidding_object = bids[bidding_object_key]
elements = bidding_object.elements
for element_name in elements:
config_params = elements[element_name]
price = ParseFormats.parse_float(
config_params["unitprice"].value)
quantity = ParseFormats.parse_float(
config_params["quantity"].value)
alloc = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(), quantity,
price)
auction_allocs[price].append(alloc)
self.logger.debug("create requests after budget bids {0}".format(
str(len(auction_allocs))))
return auction_allocs
def __init__(self):
self.domain = Config().get_config_param('Main','Domain')
use_ipv6 = Config().get_config_param('Main','UseIPv6')
self.use_ipv6 = ParseFormats.parse_bool(use_ipv6)
if self.use_ipv6:
self.ip_address6 = ParseFormats.parse_ipaddress(Config().get_config_param('Main','LocalAddr-V6'))
else:
self.ip_address4 = ParseFormats.parse_ipaddress(Config().get_config_param('Main','LocalAddr-V4'))
# Gets default ports (origin, destination)
self.local_port = ParseFormats.parse_uint16(Config().get_config_param('Main','LocalPort'))
self.protocol = ParseFormats.parse_uint8( Config().get_config_param('Main','DefaultProtocol'))
self.life_time = ParseFormats.parse_uint8( Config().get_config_param('Main','LifeTime'))
self.inmediate_start = ParseFormats.parse_bool( Config().get_config_param('Main','ImmediateStart'))
def __init__(self):
self.domain = Config().get_config_param('Main', 'Domain')
use_ipv6 = Config().get_config_param('Main', 'UseIPv6')
self.use_ipv6 = ParseFormats.parse_bool(use_ipv6)
self.ip_address6 = ParseFormats.parse_ipaddress(
Config().get_config_param('Main', 'LocalAddr-V6'))
self.destination_address6 = ParseFormats.parse_ipaddress(
Config().get_config_param('Main', 'DefaultDestinationAddr-V6'))
self.ip_address4 = ParseFormats.parse_ipaddress(
Config().get_config_param('Main', 'LocalAddr-V4'))
self.destination_address4 = ParseFormats.parse_ipaddress(
Config().get_config_param('Main', 'DefaultDestinationAddr-V4'))
# Gets default ports (origin, destination)
self.source_port = ParseFormats.parse_uint16(Config().get_config_param(
'Main', 'DefaultSourcePort'))
self.destination_port = ParseFormats.parse_uint16(
Config().get_config_param('Main', 'DefaultDestinationPort'))
self.protocol = ParseFormats.parse_uint8(Config().get_config_param(
'Main', 'DefaultProtocol'))
self.life_time = ParseFormats.parse_uint8(Config().get_config_param(
'Main', 'LifeTime'))
def get_allocation_quantity(bidding_object: BiddingObject) -> float:
"""
:param bidding_object:
:return:
"""
temp_qty = 0
elements = bidding_object.elements
# there is only one element.
for element_name in elements:
config_dict = elements[element_name]
# remove the field for updating quantities
field: ConfigParam = config_dict['quantity']
temp_qty = ParseFormats.parse_float(field.value)
break
return temp_qty
def parse_bidding_object(
self, object_key: IpapObjectKey, templates: list,
data_records: list,
ipap_template_container: IpapTemplateContainer) -> BiddingObject:
"""
Parse a bidding object from the ipap_message
:param templates: templates for the auction
:param data_records: records for the auction
:param ipap_template_container: template container where we have to include the templates.
:return: bidding object parsed.
"""
nbr_data_read = 0
nbr_option_read = 0
bidding_object_key = None
auction_key = None
s_bidding_object_type = None
status = None
data_template, opts_template = self.insert_auction_templates(
object_key.get_object_type(), templates, ipap_template_container)
# Read data records
elements = {}
for data_record in data_records:
template_id = data_record.get_template_id()
# Read a data record for a data template
if template_id == data_template.get_template_id():
data_misc = self.read_record(data_template, data_record)
record_id = self.extract_param(data_misc, 'recordid').value
elements[record_id] = data_misc
# all records have the following same information.
bidding_object_key_tmp = self.extract_param(
data_misc, 'biddingobjectid').value
auction_key_tmp = self.extract_param(data_misc,
'auctionid').value
auction_key, bidding_object_key = self.check_same_bidding_object(
auction_key, bidding_object_key, auction_key_tmp,
bidding_object_key_tmp)
s_bidding_object_type = self.extract_param(
data_misc, 'biddingobjecttype').value
status = self.extract_param(data_misc, 'status').value
# a new record was read
nbr_data_read = nbr_data_read + 1
options = {}
if template_id == opts_template.get_template_id():
opts_misc = self.read_record(opts_template, data_record)
record_id = self.extract_param(opts_misc, 'recordid').value
auction_key_tmp = self.extract_param(opts_misc,
'auctionid').value
bidding_object_key_tmp = self.extract_param(
opts_misc, 'biddingobjectid').value
auction_key, bidding_object_key = self.check_same_bidding_object(
auction_key, bidding_object_key, auction_key_tmp,
bidding_object_key_tmp)
options[record_id] = opts_misc
# a new record was read
nbr_option_read = nbr_option_read + 1
if nbr_data_read == 0:
raise ValueError("A data template was not given")
if nbr_option_read == 0:
raise ValueError("An option template was not given")
bidding_object_type = self.get_auctioning_object_type(
self.parse_type(s_bidding_object_type))
bidding_object = BiddingObject(auction_key, bidding_object_key,
bidding_object_type, elements, options)
status_val = ParseFormats.parse_int(status)
bidding_object.set_state(AuctioningObjectState(status_val))
return bidding_object
def parse_auction(self, object_key: IpapObjectKey, templates: list, data_records: list,
ipap_template_container: IpapTemplateContainer) -> Auction:
"""
Parse an auction from the ipap_message
:param templates: templates for the auction
:param data_records: records for the auction
:param ipap_template_container: template container where we have to include the templates.
:return: auction parsed.
"""
nbr_data_read = 0
nbr_option_read = 0
auction_key = None
auction_status = None
resource_key = None
template_list = None
action_name = None
data_misc = {}
opts_misc = {}
data_template, opts_template = self.insert_auction_templates(object_key.get_object_type(),
templates, ipap_template_container)
# Read data records
for data_record in data_records:
template_id = data_record.get_template_id()
# Read a data record for a data template
if template_id == data_template.get_template_id():
data_misc = self.read_record(data_template, data_record)
auction_key = self.extract_param(data_misc, 'auctionid')
auction_status = self.extract_param(data_misc, 'status')
resource_key = self.extract_param(data_misc, 'resourceid')
template_list = self.extract_param(data_misc, 'templatelist')
nbr_data_read = nbr_data_read + 1
if template_id == opts_template.get_template_id():
opts_misc = self.read_record(opts_template, data_record)
action_name = self.extract_param(opts_misc, 'algoritmname')
nbr_option_read = nbr_option_read + 1
if nbr_data_read > 1:
raise ValueError("The message included more than one data template")
if nbr_data_read == 0:
raise ValueError("The message not included a data template")
if nbr_option_read > 1:
raise ValueError("The message included more than one options template")
if nbr_option_read == 0:
raise ValueError("The message not included a options template")
action = Action(action_name.value, True, opts_misc)
auction = Auction(auction_key.value, resource_key.value, action, data_misc)
auction.set_state(AuctioningObjectState(ParseFormats.parse_int(auction_status.value)))
template_ids = template_list.value.split(',')
for template_sid in template_ids:
template = ipap_template_container.get_template(ParseFormats.parse_int(template_sid))
auction.set_bidding_object_template(template.get_object_type(template.get_type()),
template.get_type(), template.get_template_id())
return auction
def execute(self, request_params: Dict[str, FieldValue], auction_key: str,
start: datetime, stop: datetime, bids: dict) -> list:
"""
Executes the auction procedure for an specific auction.
:param request_params: request params included
:param auction_key: auction key identifying the auction
:param start: start datetime
:param stop: stop datetime
:param bids: bidding objects included
:return:
"""
self.logger.debug("bas module: start execute num bids:{0}".format(
str(len(bids))))
for bid_key in bids:
self.logger.info("Bid key in auction process:{0}".format(bid_key))
tot_demand = self.proc_module.calculate_requested_quantities(bids)
bandwidth_to_sell = self.proc_module.get_param_value(
'bandwidth', request_params)
reserve_price = self.proc_module.get_param_value(
'reserveprice', request_params)
# Order bids classifying them by whether they compete on the low and high auction.
bids_low_rct, bids_high_rct = self.proc_module.separate_bids(bids, 0.5)
# Calculate the number of requested quantities on both auctions.
nl = self.proc_module.calculate_requested_quantities(bids_low_rct)
nh = self.proc_module.calculate_requested_quantities(bids_high_rct)
ordered_bids = defaultdict(list)
for bidding_object_key in bids:
bidding_object = bids[bidding_object_key]
elements = bidding_object.elements
for element_name in elements:
config_params = elements[element_name]
price = ParseFormats.parse_float(
config_params["unitprice"].value)
quantity = ParseFormats.parse_float(
config_params["quantity"].value)
alloc = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(), quantity,
price)
ordered_bids[price].append(alloc)
qty_available = bandwidth_to_sell
sell_price = 0
self.logger.debug(
"bas module - qty available:{0}".format(qty_available))
sorted_prices = sorted(ordered_bids.keys(), reverse=True)
for price in sorted_prices:
alloc_temp = ordered_bids[price]
if price < reserve_price:
for i in range(0, len(alloc_temp)):
alloc_temp[i].quantity = 0
else:
for i in range(0, len(alloc_temp)):
if qty_available < alloc_temp[i].quantity:
alloc_temp[i].quantity = qty_available
if qty_available > 0:
sell_price = price
qty_available = 0
else:
qty_available = qty_available - alloc_temp[i].quantity
sell_price = price
# There are more units available than requested
if qty_available > 0:
sell_price = reserve_price
self.logger.debug("bas module: after executing the auction")
allocations = {}
# Creates allocations
sorted_prices = sorted(ordered_bids.keys(), reverse=True)
for price in sorted_prices:
alloc_temp = ordered_bids[price]
for i in range(0, len(alloc_temp)):
key = self.proc_module.make_key(
alloc_temp[i].auction_key,
alloc_temp[i].bidding_object_key)
if key in allocations:
self.proc_module.increment_quantity_allocation(
allocations[alloc_temp[i].bidding_object_key],
alloc_temp[i].quantity)
else:
allocation = self.proc_module.create_allocation(
self.domain, alloc_temp[i].session_id,
alloc_temp[i].bidding_object_key, start, stop,
alloc_temp[i].quantity, sell_price)
allocations[key] = allocation
# Convert from the map to the final allocationDB result
allocation_res = []
for allocation_key in allocations:
allocation = allocations[allocation_key]
allocation_res.append(allocation)
self.logger.info("allocation key: {0} - bidding object {1}".format(
allocation_key, allocation.get_parent_key()))
# Write a log with data of the auction
self.logger.debug("starttime: {0} endtime:{1}".format(
str(start), str(stop)))
self.logger.debug(
"demand: {0} - demand low: {1} demand high {2}".format(
str(tot_demand), str(nl), str(nh)))
self.logger.debug("qty_sell: {0}".format(
str(bandwidth_to_sell - qty_available)))
self.logger.debug("reservedPrice:{0} sell price:{1}".format(
str(reserve_price), str(sell_price)))
self.logger.info(
"bas module: end execute - nbr allocations: {0}".format(
len(allocation_res)))
return allocation_res
def create_request(
self, bids_low: Dict[str, BiddingObject],
bids_high: Dict[str, BiddingObject], q_star: float
) -> (DefaultDict[int, list], DefaultDict[float, list], int, int):
"""
Creates allocation requests for low and high auctions. It promotes some high auction bid into the
low auction
:param self:
:param bids_low: bids competing in the low auction
:param bids_high: bids competing in the high auction
:param q_star: probability of being promoted.
:return: allocations request in the low and high auctions.
"""
self.logger.debug(
"create requests Nbr low bids {0} - Nbr high bids {1}".format(
str(len(bids_low)), str(len(bids_high))))
nhtmp = 0
nltmp = 0
# creates the request for the L auction.
index = 0
low_auction_allocs: DefaultDict[int, list] = defaultdict(list)
for bidding_object_key in bids_low:
bidding_object = bids_low[bidding_object_key]
elements = bidding_object.elements
inserted = False
for element_name in elements:
config_params = elements[element_name]
price = ParseFormats.parse_float(
config_params["unitprice"].value)
quantity = ParseFormats.parse_float(
config_params["quantity"].value)
if quantity > 0:
alloc = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(), quantity,
price)
low_auction_allocs[index].append(alloc)
nltmp = nltmp + quantity
inserted = True
# Only increase index if there was another node inserted.
if inserted:
index = index + 1
self.logger.debug("create requests after low budget bids {0}".format(
str(len(low_auction_allocs))))
# go through all high budget bids and pass some of their units as low auction requests.
high_auction_allocs: DefaultDict[float, list] = defaultdict(list)
for bidding_object_key in bids_high:
alloc_bids = []
bidding_object = bids_high[bidding_object_key]
elements = bidding_object.elements
inserted = False
for element_name in elements:
config_params = elements[element_name]
price = ParseFormats.parse_float(
config_params["unitprice"].value)
quantity = ParseFormats.parse_float(
config_params["quantity"].value)
units_to_pass = 0
for k in range(0, floor(quantity)):
if self.get_probability() <= q_star: # pass a unit.
units_to_pass = units_to_pass + 1
# quantities in the H auction.
alloc1 = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(), quantity,
price)
high_auction_allocs[price].append(alloc1)
nhtmp = nhtmp + quantity - units_to_pass
if units_to_pass > 0:
# quantities in the L auction.
alloc2 = AllocProc(bidding_object.get_parent_key(),
bidding_object.get_key(), element_name,
bidding_object.get_session(),
units_to_pass, price)
alloc_bids.append(alloc2)
inserted = True
self.logger.debug("bid set: {0} units pass: {1}".format(
bidding_object.get_key(), str(units_to_pass)))
nltmp = nltmp + units_to_pass
# Only increase index if there was another node inserted.
if inserted:
low_auction_allocs[index].extend(alloc_bids)
index = index + 1
nl = nltmp
nh = nhtmp
self.logger.debug(
"ending create requests low_auction request {0} nl: {1}-high auction request {2} nh: {3}"
.format(str(len(low_auction_allocs)), str(nl),
str(len(high_auction_allocs)), str(nh)))
return low_auction_allocs, high_auction_allocs, nl, nh