def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = Math.floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = Math.floor(m / amount.month) + r
return output
elif Math.is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = Math.floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = Math.floor(m / amount.month) + r
return output
elif Math.is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def _make_range_domain(self, domain, column_name):
width = (domain.max - domain.min) / domain.interval
digits = Math.floor(Math.log10(width - 1))
if digits == 0:
value = "a.value"
else:
value = SQL("+").join("1" + ("0" * j) + "*" + text_type(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1))
if domain.interval == 1:
if domain.min == 0:
domain = (
SQL_SELECT + value + column_name +
SQL_FROM + "__digits__ a"
)
else:
domain = (
SQL_SELECT + sql_iso(value) + " + " + quote_value(domain.min) + column_name +
SQL_FROM + "__digits__ a"
)
else:
if domain.min == 0:
domain = (
SQL_SELECT + value + " * " + quote_value(domain.interval) +column_name +
SQL_FROM + "__digits__ a"
)
else:
domain = (
SQL_SELECT + sql_iso(value + " * " + quote_value(domain.interval)) + " + " + quote_value(domain.min) + column_name +
SQL_FROM + "__digits__ a"
)
for j in range(digits):
domain += SQL_INNER_JOIN + "__digits__" + text_type(chr(ord(b'a') + j + 1)) + " ON " +SQL_TRUE
domain += SQL_WHERE + value + " < " + quote_value(width)
return domain
def _make_range_domain(self, domain, column_name):
width = (domain.max - domain.min) / domain.interval
digits = Math.floor(Math.log10(width - 1))
if digits == 0:
value = "a.value"
else:
value = "+".join("1" + ("0" * j) + "*" +
text_type(chr(ord(b'a') + j)) + ".value"
for j in range(digits + 1))
if domain.interval == 1:
if domain.min == 0:
domain = "SELECT " + value + " " + column_name + \
"\nFROM __digits__ a"
else:
domain = "SELECT (" + value + ") + " + quote_value(domain.min) + " " + column_name + \
"\nFROM __digits__ a"
else:
if domain.min == 0:
domain = "SELECT " + value + " * " + quote_value(domain.interval) + " " + column_name + \
"\nFROM __digits__ a"
else:
domain = "SELECT (" + value + " * " + quote_value(domain.interval) + ") + " + quote_value(
domain.min) + " " + column_name + \
"\nFROM __digits__ a"
for j in range(digits):
domain += "\nJOIN __digits__ " + text_type(
chr(ord(b'a') + j + 1)) + " ON 1=1"
domain += "\nWHERE " + value + " < " + quote_value(width)
return domain
def icompressed2ibytes(source):
"""
:param source: GENERATOR OF COMPRESSED BYTES
:return: GENERATOR OF BYTES
"""
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines
bytes_count = 0
for bytes_ in source:
data = decompressor.decompress(bytes_)
bytes_count += len(data)
if Math.floor(last_bytes_count, 1000000) != Math.floor(
bytes_count, 1000000):
last_bytes_count = bytes_count
if DEBUG:
Log.note("bytes={{bytes}}", bytes=bytes_count)
yield data
def floor(self, interval=None):
if not isinstance(interval, Duration):
from mo_logs import Log
Log.error("Expecting an interval as a Duration object")
output = Duration(0)
if interval.month:
if self.month:
output.month = int(Math.floor(self.month / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
return output
# A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH
output.month = int(Math.floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
else:
output.milli = Math.floor(self.milli / (interval.milli)) * (interval.milli)
return output
def floor(self, interval=None):
if not isinstance(interval, Duration):
from mo_logs import Log
Log.error("Expecting an interval as a Duration object")
output = Duration(0)
if interval.month:
if self.month:
output.month = int(Math.floor(self.month / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
return output
# A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH
output.month = int(Math.floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
else:
output.milli = Math.floor(self.milli / (interval.milli)) * (interval.milli)
return output
def icompressed2ibytes(source):
"""
:param source: GENERATOR OF COMPRESSED BYTES
:return: GENERATOR OF BYTES
"""
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines
bytes_count = 0
for bytes_ in source:
try:
data = decompressor.decompress(bytes_)
except Exception as e:
Log.error("problem", cause=e)
bytes_count += len(data)
if Math.floor(last_bytes_count, 1000000) != Math.floor(bytes_count, 1000000):
last_bytes_count = bytes_count
DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count)
yield data
def pop(self, wait=SECOND, till=None):
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
m = self.queue.read(wait_time_seconds=Math.floor(wait.seconds))
if not m:
return None
self.pending.append(m)
output = mo_json.json2value(m.get_body())
return output
def pop(self, wait=SECOND, till=None):
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
m = self.queue.read(wait_time_seconds=Math.floor(wait.seconds))
if not m:
return None
self.pending.append(m)
output = mo_json.json2value(m.get_body())
return output
def intervals(_min, _max=None, size=1):
"""
RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE
THE LAST PAIR MAY BE SMALLER
Yes! It's just like range(), only cooler!
"""
if _max == None:
_max = _min
_min = 0
_max = int(Math.ceiling(_max))
_min = int(Math.floor(_min))
output = ((x, min(x + size, _max)) for x in _range(_min, _max, size))
return output
def pop_message(self, wait=SECOND, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
message = self.queue.read(wait_time_seconds=Math.floor(wait.seconds))
if not message:
return None
message.delete = lambda: self.queue.delete_message(message)
payload = mo_json.json2value(message.get_body())
return message, payload
def pop_message(self, wait=SECOND, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
message = self.queue.read(wait_time_seconds=Math.floor(wait.seconds))
if not message:
return None
message.delete = lambda: self.queue.delete_message(message)
payload = mo_json.json2value(message.get_body())
return message, payload
def intervals(_min, _max=None, size=1):
"""
RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE
THE LAST PAIR MAY BE SMALLER
Yes! It's just like range(), only cooler!
"""
if _max == None:
_max = _min
_min = 0
_max = int(Math.ceiling(_max))
_min = int(Math.floor(_min))
output = ((x, min(x + size, _max)) for x in __builtin__.range(_min, _max, size))
return output
def setup(
self,
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
utility # THE utility OBJECT FOUND IN CONFIG
):
with self.locker:
self.instance = instance
gigabytes = Math.floor(utility.memory)
Log.note("setup {{instance}}", instance=instance.id)
with hide('output'):
self._config_fabric(instance)
self._install_indexer()
self._install_es(gigabytes)
self._install_supervisor()
self._start_supervisor()
def setup(
self,
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
utility # THE utility OBJECT FOUND IN CONFIG
):
with self.locker:
self.instance = instance
gigabytes = Math.floor(utility.memory)
Log.note("setup {{instance}}", instance=instance.id)
with hide('output'):
self._config_fabric(instance)
self._install_indexer()
self._install_es(gigabytes)
self._install_supervisor()
self._start_supervisor()
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)
) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = Math.abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text_type(rem) + "milli" + output
rest = Math.floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "second" + output
rest = Math.floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "minute" + output
rest = Math.floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text_type(rem) + "hour" + output
rest = Math.floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = Math.floor(rest / 7)
if rem != 0:
output = "+" + text_type(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text_type(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = Math.abs(self.month)
if month <= 18 and month != 12:
output = sign + text_type(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text_type(m) + "month" + output
y = Math.floor(month / 12)
output = sign + text_type(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not Math.is_number(output[1]):
output = output[1::]
return output
def add_instances(self, net_new_utility, remaining_budget):
prices = self.pricing()
for p in prices:
if net_new_utility <= 0 or remaining_budget <= 0:
break
if p.current_price == None:
Log.note("{{type}} has no current price",
type=p.type.instance_type
)
continue
if self.settings.utility[p.type.instance_type].blacklist or \
p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones):
Log.note("{{type}} in {{zone}} skipped due to blacklist", type=p.type.instance_type, zone=p.availability_zone)
continue
# DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY
max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount
max_bid = Math.min(p.higher_price, max_acceptable_price, remaining_budget)
min_bid = p.price_80
if min_bid > max_acceptable_price:
Log.note(
"Price of ${{price}}/hour on {{type}}: Over remaining acceptable price of ${{remaining}}/hour",
type=p.type.instance_type,
price=min_bid,
remaining=max_acceptable_price
)
continue
elif min_bid > remaining_budget:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Over budget of ${{remaining_budget}}/hour",
type=p.type.instance_type,
bid=min_bid,
remaining_budget=remaining_budget
)
continue
elif min_bid > max_bid:
Log.error("not expected")
naive_number_needed = int(Math.round(float(net_new_utility) / float(p.type.utility), decimal=0))
limit_total = None
if self.settings.max_percent_per_type < 1:
current_count = sum(1 for a in self.active if a.launch_specification.instance_type == p.type.instance_type and a.launch_specification.placement == p.availability_zone)
all_count = sum(1 for a in self.active if a.launch_specification.placement == p.availability_zone)
all_count = max(all_count, naive_number_needed)
limit_total = int(Math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type)))
num = Math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type)
if num < 0:
Log.note(
"{{type}} is over {{limit|percent}} of instances, no more requested",
limit=self.settings.max_percent_per_type,
type=p.type.instance_type
)
continue
elif num == 1:
min_bid = Math.min(Math.max(p.current_price * 1.1, min_bid), max_acceptable_price)
price_interval = 0
else:
price_interval = Math.min(min_bid / 10, (max_bid - min_bid) / (num - 1))
for i in range(num):
bid_per_machine = min_bid + (i * price_interval)
if bid_per_machine < p.current_price:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Under current price of ${{current_price}}/hour",
type=p.type.instance_type,
bid=bid_per_machine - p.type.discount,
current_price=p.current_price
)
continue
if bid_per_machine - p.type.discount > remaining_budget:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Over remaining budget of ${{remaining}}/hour",
type=p.type.instance_type,
bid=bid_per_machine - p.type.discount,
remaining=remaining_budget
)
continue
try:
if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1:
Log.error("Spot Manager can only request machine one-at-a-time")
new_requests = self._request_spot_instances(
price=bid_per_machine,
availability_zone_group=p.availability_zone,
instance_type=p.type.instance_type,
kwargs=copy(self.settings.ec2.request)
)
Log.note(
"Request {{num}} instance {{type}} in {{zone}} with utility {{utility}} at ${{price}}/hour",
num=len(new_requests),
type=p.type.instance_type,
zone=p.availability_zone,
utility=p.type.utility,
price=bid_per_machine
)
net_new_utility -= p.type.utility * len(new_requests)
remaining_budget -= (bid_per_machine - p.type.discount) * len(new_requests)
with self.net_new_locker:
for ii in new_requests:
self.net_new_spot_requests.add(ii)
except Exception as e:
Log.warning(
"Request instance {{type}} failed because {{reason}}",
type=p.type.instance_type,
reason=e.message,
cause=e
)
if "Max spot instance count exceeded" in e.message:
Log.note("No further spot requests will be attempted.")
return net_new_utility, remaining_budget
return net_new_utility, remaining_budget
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = Math.abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text_type(rem) + "milli" + output
rest = Math.floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "second" + output
rest = Math.floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "minute" + output
rest = Math.floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text_type(rem) + "hour" + output
rest = Math.floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = Math.floor(rest / 7)
if rem != 0:
output = "+" + text_type(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text_type(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = Math.abs(self.month)
if month <= 18 and month != 12:
output = sign + text_type(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text_type(m) + "month" + output
y = Math.floor(month / 12)
output = sign + text_type(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not Math.is_number(output[1]):
output = output[1::]
return output
