def __div__(self, amount): if isinstance(amount, Duration) and amount.month: m = self.month r = self.milli # DO NOT CONSIDER TIME OF DAY tod = r % MILLI_VALUES.day r = r - tod if m == 0 and r > (MILLI_VALUES.year / 3): m = floor(12 * self.milli / MILLI_VALUES.year) r -= (m / 12) * MILLI_VALUES.year else: r = r - (self.month * MILLI_VALUES.month) if r >= MILLI_VALUES.day * 31: from mo_logs import Log Log.error("Do not know how to handle") r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)]) output = floor(m / amount.month) + r return output elif is_number(amount): output = Duration(0) output.milli = self.milli / amount output.month = self.month / amount return output else: return self.milli / amount.milli
def test_floor_mod_identity_w_ints(self): for i in range(100): x = randoms.float() * 200 - 100.0 m = floor(abs(random.gauss(0, 5))) if m == 0: self.assertEqual(mo_math.floor(x, m), None) self.assertEqual(mo_math.mod(x, m), None) else: self.assertAlmostEqual(mo_math.floor(x, m) + mo_math.mod(x, m), x, places=7)
def _make_range_domain(self, domain, column_name): width = (domain.max - domain.min) / domain.interval digits = mo_math.floor(mo_math.log10(width - 1)) if digits == 0: value = "a.value" else: value = SQL("+").join("1" + ("0" * j) + "*" + text_type(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1)) if domain.interval == 1: if domain.min == 0: domain = (SQL_SELECT + value + column_name + SQL_FROM + "__digits__ a") else: domain = (SQL_SELECT + sql_iso(value) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a") else: if domain.min == 0: domain = (SQL_SELECT + value + " * " + quote_value(domain.interval) + column_name + SQL_FROM + "__digits__ a") else: domain = ( SQL_SELECT + sql_iso(value + " * " + quote_value(domain.interval)) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a") for j in range(digits): domain += SQL_INNER_JOIN + "__digits__" + text_type( chr(ord(b'a') + j + 1)) + " ON " + SQL_TRUE domain += SQL_WHERE + value + " < " + quote_value(width) return domain
def test_floor_mod_identity(self): for i in range(100): x = randoms.float() * 200 - 100.0 m = abs(random.gauss(0, 5)) self.assertAlmostEqual(mo_math.floor(x, m) + mo_math.mod(x, m), x, places=7)
def _next_run_time(self): """ :return: return signal for next """ interval = mo_math.floor((Date.now() - self.starting) / self.duration) next_time = self.starting + (interval * self.duration) return next_time
def icompressed2ibytes(source): """ :param source: GENERATOR OF COMPRESSED BYTES :return: GENERATOR OF BYTES """ decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines bytes_count = 0 for bytes_ in source: try: data = decompressor.decompress(bytes_) except Exception as e: Log.error("problem", cause=e) bytes_count += len(data) if mo_math.floor(last_bytes_count, 1000000) != mo_math.floor(bytes_count, 1000000): last_bytes_count = bytes_count DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count) yield data
def floor(self, interval=None): if not isinstance(interval, Duration): from mo_logs import Log Log.error("Expecting an interval as a Duration object") output = Duration(0) if interval.month: if self.month: output.month = int(floor(self.month / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month return output # A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH output.month = int(floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month else: output.milli = floor(self.milli / (interval.milli)) * (interval.milli) return output
def icompressed2ibytes(source): """ :param source: GENERATOR OF COMPRESSED BYTES :return: GENERATOR OF BYTES """ decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines bytes_count = 0 for bytes_ in source: data = decompressor.decompress(bytes_) bytes_count += len(data) if mo_math.floor(last_bytes_count, 1000000) != mo_math.floor( bytes_count, 1000000): last_bytes_count = bytes_count DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count) yield data
def pop(self, wait=SECOND, till=None): if till is not None and not isinstance(till, Signal): Log.error("Expecting a signal") m = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds)) if not m: return None self.pending.append(m) output = mo_json.json2value(m.get_body()) return output
def floor(self, interval=None): if not isinstance(interval, Duration): from mo_logs import Log Log.error("Expecting an interval as a Duration object") output = Duration(0) if interval.month: if self.month: output.month = int( floor(self.month / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month return output # A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH output.month = int( floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month else: output.milli = floor(self.milli / (interval.milli)) * (interval.milli) return output
def intervals(_min, _max=None, size=1): """ RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE THE LAST PAIR MAY BE SMALLER Yes! It's just like range(), only cooler! """ if _max == None: _max = _min _min = 0 _max = int(mo_math.ceiling(_max)) _min = int(mo_math.floor(_min)) output = ((x, min(x + size, _max)) for x in _range(_min, _max, size)) return output
def pop_message(self, wait=SECOND, till=None): """ RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE """ if till is not None and not isinstance(till, Signal): Log.error("Expecting a signal") message = self.queue.read(wait_time_seconds=mo_math.floor(wait.seconds)) if not message: return None message.delete = lambda: self.queue.delete_message(message) payload = mo_json.json2value(message.get_body()) return message, payload
def setup( self, instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP utility, # THE utility OBJECT FOUND IN CONFIG please_stop ): try: with Connection(host=instance.ip_address, kwargs=self.settings.connect) as conn: gigabytes = mo_math.floor(utility.memory) Log.note("setup {{instance}}", instance=instance.id) _install_python_indexer(instance=instance, conn=conn) _install_es(gigabytes, instance=instance, conn=conn) _install_supervisor(instance=instance, conn=conn) _start_supervisor(conn=conn) Log.alert("Done install of {{host}}", host=instance.ip_address) except Exception as e: Log.error("could not setup ES at {{ip}}", ip=instance.ip_address, cause=e)
def _make_range_domain(self, domain, column_name): width = (domain.max - domain.min) / domain.interval digits = mo_math.floor(mo_math.log10(width - 1)) if digits == 0: value = quote_column("a", "value") else: value = SQL_PLUS.join("1" + ("0" * j) + SQL_STAR + text(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1)) if domain.interval == 1: if domain.min == 0: domain = (SQL_SELECT + sql_alias(value, column_name) + SQL_FROM + sql_alias(quote_column(DIGITS_TABLE), "a")) else: domain = (SQL_SELECT + sql_alias( sql_iso(value) + SQL_PLUS + quote_value(domain.min), column_name) + SQL_FROM + sql_alias(quote_column(DIGITS_TABLE), "a")) else: if domain.min == 0: domain = ConcatSQL( SQL_SELECT, sql_alias(value + SQL_STAR + quote_value(domain.interval), column_name), SQL_FROM, sql_alias(quote_column(DIGITS_TABLE), "a")) else: domain = ConcatSQL( SQL_SELECT, sql_alias( sql_iso(value, SQL_STAR, quote_value(domain.interval)) + SQL_PLUS + quote_value(domain.min), column_name), SQL_FROM, sql_alias(quote_column(DIGITS_TABLE), "a")) for j in range(digits): domain += SQL_INNER_JOIN + sql_alias( quote_column(DIGITS_TABLE), text( chr(ord(b'a') + j + 1))) + SQL_ON + SQL_TRUE domain += SQL_WHERE + value + " < " + quote_value(width) return domain
def test_floor(self): self.assertEqual(mo_math.floor(0, 1), 0) self.assertEqual(mo_math.floor(1, 1), 1) self.assertEqual(mo_math.floor(-1, 1), -1) self.assertEqual(mo_math.floor(0.1, 1), 0) self.assertEqual(mo_math.floor(1.1, 1), 1) self.assertEqual(mo_math.floor(-1.1, 1), -2) self.assertEqual(mo_math.floor(0, 2), 0) self.assertEqual(mo_math.floor(1, 2), 0) self.assertEqual(mo_math.floor(-1, 2), -2) self.assertEqual(mo_math.floor(0.1, 2), 0) self.assertEqual(mo_math.floor(1.1, 2), 0) self.assertEqual(mo_math.floor(-1.1, 2), -2) self.assertEqual(mo_math.floor(-10, 2), -10)
def __unicode__(self): if not self.milli: return "zero" output = "" rest = (self.milli - (MILLI_VALUES.month * self.month) ) # DO NOT INCLUDE THE MONTH'S MILLIS isNegative = (rest < 0) rest = abs(rest) # MILLI rem = rest % 1000 if rem != 0: output = "+" + text(rem) + "milli" + output rest = floor(rest / 1000) # SECOND rem = rest % 60 if rem != 0: output = "+" + text(rem) + "second" + output rest = floor(rest / 60) # MINUTE rem = rest % 60 if rem != 0: output = "+" + text(rem) + "minute" + output rest = floor(rest / 60) # HOUR rem = rest % 24 if rem != 0: output = "+" + text(rem) + "hour" + output rest = floor(rest / 24) # DAY if (rest < 11 and rest != 7) or rest % 10 == 0: rem = rest rest = 0 else: rem = rest % 7 rest = floor(rest / 7) if rem != 0: output = "+" + text(rem) + "day" + output # WEEK if rest != 0: output = "+" + text(rest) + "week" + output if isNegative: output = output.replace("+", "-") # MONTH AND YEAR if self.month: sign = "-" if self.month < 0 else "+" month = abs(self.month) if month <= 18 and month != 12: output = sign + text(month) + "month" + output else: m = month % 12 if m != 0: output = sign + text(m) + "month" + output y = floor(month / 12) output = sign + text(y) + "year" + output if output[0] == "+": output = output[1::] if output[0] == '1' and not is_number(output[1]): output = output[1::] return output
def __unicode__(self): if not self.milli: return "zero" output = "" rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS isNegative = (rest < 0) rest = abs(rest) # MILLI rem = rest % 1000 if rem != 0: output = "+" + text_type(rem) + "milli" + output rest = floor(rest / 1000) # SECOND rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "second" + output rest = floor(rest / 60) # MINUTE rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "minute" + output rest = floor(rest / 60) # HOUR rem = rest % 24 if rem != 0: output = "+" + text_type(rem) + "hour" + output rest = floor(rest / 24) # DAY if (rest < 11 and rest != 7) or rest % 10 == 0: rem = rest rest = 0 else: rem = rest % 7 rest = floor(rest / 7) if rem != 0: output = "+" + text_type(rem) + "day" + output # WEEK if rest != 0: output = "+" + text_type(rest) + "week" + output if isNegative: output = output.replace("+", "-") # MONTH AND YEAR if self.month: sign = "-" if self.month < 0 else "+" month = abs(self.month) if month <= 18 and month != 12: output = sign + text_type(month) + "month" + output else: m = month % 12 if m != 0: output = sign + text_type(m) + "month" + output y = floor(month / 12) output = sign + text_type(y) + "year" + output if output[0] == "+": output = output[1::] if output[0] == '1' and not is_number(output[1]): output = output[1::] return output
def add_instances(self, net_new_utility, remaining_budget): prices = self.pricing() for p in prices: if net_new_utility <= 0 or remaining_budget <= 0: break if p.current_price == None: Log.note("{{type}} has no current price", type=p.type.instance_type) continue if self.settings.utility[p.type.instance_type].blacklist or \ p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones): Log.note("{{type}} in {{zone}} skipped due to blacklist", type=p.type.instance_type, zone=p.availability_zone) continue # DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount max_bid = mo_math.min(p.higher_price, max_acceptable_price, remaining_budget) min_bid = p.price_80 if min_bid > max_acceptable_price: Log.note( "Price of ${{price}}/hour on {{type}}: Over remaining acceptable price of ${{remaining}}/hour", type=p.type.instance_type, price=min_bid, remaining=max_acceptable_price) continue elif min_bid > remaining_budget: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Over budget of ${{remaining_budget}}/hour", type=p.type.instance_type, bid=min_bid, remaining_budget=remaining_budget) continue elif min_bid > max_bid: Log.error("not expected") naive_number_needed = int( mo_math.round(float(net_new_utility) / float(p.type.utility), decimal=0)) limit_total = None if self.settings.max_percent_per_type < 1: current_count = sum( 1 for a in self.active if a.launch_specification.instance_type == p.type.instance_type and a.launch_specification.placement == p.availability_zone) all_count = sum( 1 for a in self.active if a.launch_specification.placement == p.availability_zone) all_count = max(all_count, naive_number_needed) limit_total = int( mo_math.floor( (all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type))) num = mo_math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type) if num < 0: Log.note( "{{type}} is over {{limit|percent}} of instances, no more requested", limit=self.settings.max_percent_per_type, type=p.type.instance_type) continue elif num == 1: min_bid = mo_math.min( mo_math.max(p.current_price * 1.1, min_bid), max_acceptable_price) price_interval = 0 else: price_interval = mo_math.min(min_bid / 10, (max_bid - min_bid) / (num - 1)) for i in range(num): bid_per_machine = min_bid + (i * price_interval) if bid_per_machine < p.current_price: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Under current price of ${{current_price}}/hour", type=p.type.instance_type, bid=bid_per_machine - p.type.discount, current_price=p.current_price) continue if bid_per_machine - p.type.discount > remaining_budget: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Over remaining budget of ${{remaining}}/hour", type=p.type.instance_type, bid=bid_per_machine - p.type.discount, remaining=remaining_budget) continue last_no_capacity_message = self.no_capacity.get( p.type.instance_type, Null) if last_no_capacity_message > Date.now( ) - CAPACITY_NOT_AVAILABLE_RETRY: Log.note( "Did not bid on {{type}}: \"No capacity\" last seen at {{last_time|datetime}}", type=p.type.instance_type, last_time=last_no_capacity_message) continue try: if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1: Log.error( "Spot Manager can only request machine one-at-a-time" ) new_requests = self._request_spot_instances( price=bid_per_machine, availability_zone_group=p.availability_zone, instance_type=p.type.instance_type, kwargs=copy(self.settings.ec2.request)) Log.note( "Request {{num}} instance {{type}} in {{zone}} with utility {{utility}} at ${{price}}/hour", num=len(new_requests), type=p.type.instance_type, zone=p.availability_zone, utility=p.type.utility, price=bid_per_machine) net_new_utility -= p.type.utility * len(new_requests) remaining_budget -= (bid_per_machine - p.type.discount) * len(new_requests) with self.net_new_locker: for ii in new_requests: self.net_new_spot_requests.add(ii) except Exception as e: Log.warning( "Request instance {{type}} failed because {{reason}}", type=p.type.instance_type, reason=e.message, cause=e) if "Max spot instance count exceeded" in e.message: Log.note("No further spot requests will be attempted.") return net_new_utility, remaining_budget return net_new_utility, remaining_budget