def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = floor(m / amount.month) + r
return output
elif is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def __div__(self, amount):
if isinstance(amount, Duration) and amount.month:
m = self.month
r = self.milli
# DO NOT CONSIDER TIME OF DAY
tod = r % MILLI_VALUES.day
r = r - tod
if m == 0 and r > (MILLI_VALUES.year / 3):
m = floor(12 * self.milli / MILLI_VALUES.year)
r -= (m / 12) * MILLI_VALUES.year
else:
r = r - (self.month * MILLI_VALUES.month)
if r >= MILLI_VALUES.day * 31:
from mo_logs import Log
Log.error("Do not know how to handle")
r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)])
output = floor(m / amount.month) + r
return output
elif is_number(amount):
output = Duration(0)
output.milli = self.milli / amount
output.month = self.month / amount
return output
else:
return self.milli / amount.milli
def test_floor_mod_identity_w_ints(self):
for i in range(100):
x = randoms.float() * 200 - 100.0
m = floor(abs(random.gauss(0, 5)))
if m == 0:
self.assertEqual(mo_math.floor(x, m), None)
self.assertEqual(mo_math.mod(x, m), None)
else:
self.assertAlmostEqual(mo_math.floor(x, m) + mo_math.mod(x, m),
x,
places=7)
def _make_range_domain(self, domain, column_name):
width = (domain.max - domain.min) / domain.interval
digits = mo_math.floor(mo_math.log10(width - 1))
if digits == 0:
value = "a.value"
else:
value = SQL("+").join("1" + ("0" * j) + "*" +
text_type(chr(ord(b'a') + j)) + ".value"
for j in range(digits + 1))
if domain.interval == 1:
if domain.min == 0:
domain = (SQL_SELECT + value + column_name + SQL_FROM +
"__digits__ a")
else:
domain = (SQL_SELECT + sql_iso(value) + " + " +
quote_value(domain.min) + column_name + SQL_FROM +
"__digits__ a")
else:
if domain.min == 0:
domain = (SQL_SELECT + value + " * " +
quote_value(domain.interval) + column_name +
SQL_FROM + "__digits__ a")
else:
domain = (
SQL_SELECT +
sql_iso(value + " * " + quote_value(domain.interval)) +
" + " + quote_value(domain.min) + column_name + SQL_FROM +
"__digits__ a")
for j in range(digits):
domain += SQL_INNER_JOIN + "__digits__" + text_type(
chr(ord(b'a') + j + 1)) + " ON " + SQL_TRUE
domain += SQL_WHERE + value + " < " + quote_value(width)
return domain
def test_floor_mod_identity(self):
for i in range(100):
x = randoms.float() * 200 - 100.0
m = abs(random.gauss(0, 5))
self.assertAlmostEqual(mo_math.floor(x, m) + mo_math.mod(x, m),
x,
places=7)
def _next_run_time(self):
"""
:return: return signal for next
"""
interval = mo_math.floor((Date.now() - self.starting) / self.duration)
next_time = self.starting + (interval * self.duration)
return next_time
def icompressed2ibytes(source):
"""
:param source: GENERATOR OF COMPRESSED BYTES
:return: GENERATOR OF BYTES
"""
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines
bytes_count = 0
for bytes_ in source:
try:
data = decompressor.decompress(bytes_)
except Exception as e:
Log.error("problem", cause=e)
bytes_count += len(data)
if mo_math.floor(last_bytes_count, 1000000) != mo_math.floor(bytes_count, 1000000):
last_bytes_count = bytes_count
DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count)
yield data
def floor(self, interval=None):
if not isinstance(interval, Duration):
from mo_logs import Log
Log.error("Expecting an interval as a Duration object")
output = Duration(0)
if interval.month:
if self.month:
output.month = int(floor(self.month / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
return output
# A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH
output.month = int(floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
else:
output.milli = floor(self.milli / (interval.milli)) * (interval.milli)
return output
def icompressed2ibytes(source):
"""
:param source: GENERATOR OF COMPRESSED BYTES
:return: GENERATOR OF BYTES
"""
decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS)
last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines
bytes_count = 0
for bytes_ in source:
data = decompressor.decompress(bytes_)
bytes_count += len(data)
if mo_math.floor(last_bytes_count, 1000000) != mo_math.floor(
bytes_count, 1000000):
last_bytes_count = bytes_count
DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count)
yield data
def pop(self, wait=SECOND, till=None):
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
m = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds))
if not m:
return None
self.pending.append(m)
output = mo_json.json2value(m.get_body())
return output
def pop(self, wait=SECOND, till=None):
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
m = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds))
if not m:
return None
self.pending.append(m)
output = mo_json.json2value(m.get_body())
return output
def floor(self, interval=None):
if not isinstance(interval, Duration):
from mo_logs import Log
Log.error("Expecting an interval as a Duration object")
output = Duration(0)
if interval.month:
if self.month:
output.month = int(
floor(self.month / interval.month) * interval.month)
output.milli = output.month * MILLI_VALUES.month
return output
# A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH
output.month = int(
floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month)
* interval.month)
output.milli = output.month * MILLI_VALUES.month
else:
output.milli = floor(self.milli /
(interval.milli)) * (interval.milli)
return output
def intervals(_min, _max=None, size=1):
"""
RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE
THE LAST PAIR MAY BE SMALLER
Yes! It's just like range(), only cooler!
"""
if _max == None:
_max = _min
_min = 0
_max = int(mo_math.ceiling(_max))
_min = int(mo_math.floor(_min))
output = ((x, min(x + size, _max)) for x in _range(_min, _max, size))
return output
def intervals(_min, _max=None, size=1):
"""
RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE
THE LAST PAIR MAY BE SMALLER
Yes! It's just like range(), only cooler!
"""
if _max == None:
_max = _min
_min = 0
_max = int(mo_math.ceiling(_max))
_min = int(mo_math.floor(_min))
output = ((x, min(x + size, _max)) for x in _range(_min, _max, size))
return output
def pop_message(self, wait=SECOND, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
message = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds))
if not message:
return None
message.delete = lambda: self.queue.delete_message(message)
payload = mo_json.json2value(message.get_body())
return message, payload
def pop_message(self, wait=SECOND, till=None):
"""
RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE
"""
if till is not None and not isinstance(till, Signal):
Log.error("Expecting a signal")
message = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds))
if not message:
return None
message.delete = lambda: self.queue.delete_message(message)
payload = mo_json.json2value(message.get_body())
return message, payload
def setup(
self,
instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP
utility, # THE utility OBJECT FOUND IN CONFIG
please_stop
):
try:
with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn:
gigabytes = mo_math.floor(utility.memory)
Log.note("setup {{instance}}", instance=instance.id)
_install_python_indexer(instance=instance, conn=conn)
_install_es(gigabytes, instance=instance, conn=conn)
_install_supervisor(instance=instance, conn=conn)
_start_supervisor(conn=conn)
Log.alert("Done install of {{host}}", host=instance.ip_address)
except Exception as e:
Log.error("could not setup ES at {{ip}}", ip=instance.ip_address, cause=e)
def _make_range_domain(self, domain, column_name):
width = (domain.max - domain.min) / domain.interval
digits = mo_math.floor(mo_math.log10(width - 1))
if digits == 0:
value = quote_column("a", "value")
else:
value = SQL_PLUS.join("1" + ("0" * j) + SQL_STAR +
text(chr(ord(b'a') + j)) + ".value"
for j in range(digits + 1))
if domain.interval == 1:
if domain.min == 0:
domain = (SQL_SELECT + sql_alias(value, column_name) +
SQL_FROM +
sql_alias(quote_column(DIGITS_TABLE), "a"))
else:
domain = (SQL_SELECT + sql_alias(
sql_iso(value) + SQL_PLUS + quote_value(domain.min),
column_name) + SQL_FROM +
sql_alias(quote_column(DIGITS_TABLE), "a"))
else:
if domain.min == 0:
domain = ConcatSQL(
SQL_SELECT,
sql_alias(value + SQL_STAR + quote_value(domain.interval),
column_name), SQL_FROM,
sql_alias(quote_column(DIGITS_TABLE), "a"))
else:
domain = ConcatSQL(
SQL_SELECT,
sql_alias(
sql_iso(value, SQL_STAR, quote_value(domain.interval))
+ SQL_PLUS + quote_value(domain.min), column_name),
SQL_FROM, sql_alias(quote_column(DIGITS_TABLE), "a"))
for j in range(digits):
domain += SQL_INNER_JOIN + sql_alias(
quote_column(DIGITS_TABLE), text(
chr(ord(b'a') + j + 1))) + SQL_ON + SQL_TRUE
domain += SQL_WHERE + value + " < " + quote_value(width)
return domain
def test_floor(self):
self.assertEqual(mo_math.floor(0, 1), 0)
self.assertEqual(mo_math.floor(1, 1), 1)
self.assertEqual(mo_math.floor(-1, 1), -1)
self.assertEqual(mo_math.floor(0.1, 1), 0)
self.assertEqual(mo_math.floor(1.1, 1), 1)
self.assertEqual(mo_math.floor(-1.1, 1), -2)
self.assertEqual(mo_math.floor(0, 2), 0)
self.assertEqual(mo_math.floor(1, 2), 0)
self.assertEqual(mo_math.floor(-1, 2), -2)
self.assertEqual(mo_math.floor(0.1, 2), 0)
self.assertEqual(mo_math.floor(1.1, 2), 0)
self.assertEqual(mo_math.floor(-1.1, 2), -2)
self.assertEqual(mo_math.floor(-10, 2), -10)
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)
) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text(rem) + "milli" + output
rest = floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text(rem) + "second" + output
rest = floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text(rem) + "minute" + output
rest = floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text(rem) + "hour" + output
rest = floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = floor(rest / 7)
if rem != 0:
output = "+" + text(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = abs(self.month)
if month <= 18 and month != 12:
output = sign + text(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text(m) + "month" + output
y = floor(month / 12)
output = sign + text(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not is_number(output[1]):
output = output[1::]
return output
def __unicode__(self):
if not self.milli:
return "zero"
output = ""
rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS
isNegative = (rest < 0)
rest = abs(rest)
# MILLI
rem = rest % 1000
if rem != 0:
output = "+" + text_type(rem) + "milli" + output
rest = floor(rest / 1000)
# SECOND
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "second" + output
rest = floor(rest / 60)
# MINUTE
rem = rest % 60
if rem != 0:
output = "+" + text_type(rem) + "minute" + output
rest = floor(rest / 60)
# HOUR
rem = rest % 24
if rem != 0:
output = "+" + text_type(rem) + "hour" + output
rest = floor(rest / 24)
# DAY
if (rest < 11 and rest != 7) or rest % 10 == 0:
rem = rest
rest = 0
else:
rem = rest % 7
rest = floor(rest / 7)
if rem != 0:
output = "+" + text_type(rem) + "day" + output
# WEEK
if rest != 0:
output = "+" + text_type(rest) + "week" + output
if isNegative:
output = output.replace("+", "-")
# MONTH AND YEAR
if self.month:
sign = "-" if self.month < 0 else "+"
month = abs(self.month)
if month <= 18 and month != 12:
output = sign + text_type(month) + "month" + output
else:
m = month % 12
if m != 0:
output = sign + text_type(m) + "month" + output
y = floor(month / 12)
output = sign + text_type(y) + "year" + output
if output[0] == "+":
output = output[1::]
if output[0] == '1' and not is_number(output[1]):
output = output[1::]
return output
def add_instances(self, net_new_utility, remaining_budget):
prices = self.pricing()
for p in prices:
if net_new_utility <= 0 or remaining_budget <= 0:
break
if p.current_price == None:
Log.note("{{type}} has no current price",
type=p.type.instance_type)
continue
if self.settings.utility[p.type.instance_type].blacklist or \
p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones):
Log.note("{{type}} in {{zone}} skipped due to blacklist",
type=p.type.instance_type,
zone=p.availability_zone)
continue
# DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY
max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount
max_bid = mo_math.min(p.higher_price, max_acceptable_price,
remaining_budget)
min_bid = p.price_80
if min_bid > max_acceptable_price:
Log.note(
"Price of ${{price}}/hour on {{type}}: Over remaining acceptable price of ${{remaining}}/hour",
type=p.type.instance_type,
price=min_bid,
remaining=max_acceptable_price)
continue
elif min_bid > remaining_budget:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Over budget of ${{remaining_budget}}/hour",
type=p.type.instance_type,
bid=min_bid,
remaining_budget=remaining_budget)
continue
elif min_bid > max_bid:
Log.error("not expected")
naive_number_needed = int(
mo_math.round(float(net_new_utility) / float(p.type.utility),
decimal=0))
limit_total = None
if self.settings.max_percent_per_type < 1:
current_count = sum(
1 for a in self.active
if a.launch_specification.instance_type ==
p.type.instance_type and a.launch_specification.placement
== p.availability_zone)
all_count = sum(
1 for a in self.active
if a.launch_specification.placement == p.availability_zone)
all_count = max(all_count, naive_number_needed)
limit_total = int(
mo_math.floor(
(all_count * self.settings.max_percent_per_type -
current_count) /
(1 - self.settings.max_percent_per_type)))
num = mo_math.min(naive_number_needed, limit_total,
self.settings.max_requests_per_type)
if num < 0:
Log.note(
"{{type}} is over {{limit|percent}} of instances, no more requested",
limit=self.settings.max_percent_per_type,
type=p.type.instance_type)
continue
elif num == 1:
min_bid = mo_math.min(
mo_math.max(p.current_price * 1.1, min_bid),
max_acceptable_price)
price_interval = 0
else:
price_interval = mo_math.min(min_bid / 10,
(max_bid - min_bid) / (num - 1))
for i in range(num):
bid_per_machine = min_bid + (i * price_interval)
if bid_per_machine < p.current_price:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Under current price of ${{current_price}}/hour",
type=p.type.instance_type,
bid=bid_per_machine - p.type.discount,
current_price=p.current_price)
continue
if bid_per_machine - p.type.discount > remaining_budget:
Log.note(
"Did not bid ${{bid}}/hour on {{type}}: Over remaining budget of ${{remaining}}/hour",
type=p.type.instance_type,
bid=bid_per_machine - p.type.discount,
remaining=remaining_budget)
continue
last_no_capacity_message = self.no_capacity.get(
p.type.instance_type, Null)
if last_no_capacity_message > Date.now(
) - CAPACITY_NOT_AVAILABLE_RETRY:
Log.note(
"Did not bid on {{type}}: \"No capacity\" last seen at {{last_time|datetime}}",
type=p.type.instance_type,
last_time=last_no_capacity_message)
continue
try:
if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1:
Log.error(
"Spot Manager can only request machine one-at-a-time"
)
new_requests = self._request_spot_instances(
price=bid_per_machine,
availability_zone_group=p.availability_zone,
instance_type=p.type.instance_type,
kwargs=copy(self.settings.ec2.request))
Log.note(
"Request {{num}} instance {{type}} in {{zone}} with utility {{utility}} at ${{price}}/hour",
num=len(new_requests),
type=p.type.instance_type,
zone=p.availability_zone,
utility=p.type.utility,
price=bid_per_machine)
net_new_utility -= p.type.utility * len(new_requests)
remaining_budget -= (bid_per_machine -
p.type.discount) * len(new_requests)
with self.net_new_locker:
for ii in new_requests:
self.net_new_spot_requests.add(ii)
except Exception as e:
Log.warning(
"Request instance {{type}} failed because {{reason}}",
type=p.type.instance_type,
reason=e.message,
cause=e)
if "Max spot instance count exceeded" in e.message:
Log.note("No further spot requests will be attempted.")
return net_new_utility, remaining_budget
return net_new_utility, remaining_budget
