def __div__(self, amount): if isinstance(amount, Duration) and amount.month: m = self.month r = self.milli # DO NOT CONSIDER TIME OF DAY tod = r % MILLI_VALUES.day r = r - tod if m == 0 and r > (MILLI_VALUES.year / 3): m = Math.floor(12 * self.milli / MILLI_VALUES.year) r -= (m / 12) * MILLI_VALUES.year else: r = r - (self.month * MILLI_VALUES.month) if r >= MILLI_VALUES.day * 31: from mo_logs import Log Log.error("Do not know how to handle") r = MIN([29 / 30, (r + tod) / (MILLI_VALUES.day * 30)]) output = Math.floor(m / amount.month) + r return output elif Math.is_number(amount): output = Duration(0) output.milli = self.milli / amount output.month = self.month / amount return output else: return self.milli / amount.milli
def _make_range_domain(self, domain, column_name): width = (domain.max - domain.min) / domain.interval digits = Math.floor(Math.log10(width - 1)) if digits == 0: value = "a.value" else: value = SQL("+").join("1" + ("0" * j) + "*" + text_type(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1)) if domain.interval == 1: if domain.min == 0: domain = ( SQL_SELECT + value + column_name + SQL_FROM + "__digits__ a" ) else: domain = ( SQL_SELECT + sql_iso(value) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a" ) else: if domain.min == 0: domain = ( SQL_SELECT + value + " * " + quote_value(domain.interval) +column_name + SQL_FROM + "__digits__ a" ) else: domain = ( SQL_SELECT + sql_iso(value + " * " + quote_value(domain.interval)) + " + " + quote_value(domain.min) + column_name + SQL_FROM + "__digits__ a" ) for j in range(digits): domain += SQL_INNER_JOIN + "__digits__" + text_type(chr(ord(b'a') + j + 1)) + " ON " +SQL_TRUE domain += SQL_WHERE + value + " < " + quote_value(width) return domain
def _make_range_domain(self, domain, column_name): width = (domain.max - domain.min) / domain.interval digits = Math.floor(Math.log10(width - 1)) if digits == 0: value = "a.value" else: value = "+".join("1" + ("0" * j) + "*" + text_type(chr(ord(b'a') + j)) + ".value" for j in range(digits + 1)) if domain.interval == 1: if domain.min == 0: domain = "SELECT " + value + " " + column_name + \ "\nFROM __digits__ a" else: domain = "SELECT (" + value + ") + " + quote_value(domain.min) + " " + column_name + \ "\nFROM __digits__ a" else: if domain.min == 0: domain = "SELECT " + value + " * " + quote_value(domain.interval) + " " + column_name + \ "\nFROM __digits__ a" else: domain = "SELECT (" + value + " * " + quote_value(domain.interval) + ") + " + quote_value( domain.min) + " " + column_name + \ "\nFROM __digits__ a" for j in range(digits): domain += "\nJOIN __digits__ " + text_type( chr(ord(b'a') + j + 1)) + " ON 1=1" domain += "\nWHERE " + value + " < " + quote_value(width) return domain
def icompressed2ibytes(source): """ :param source: GENERATOR OF COMPRESSED BYTES :return: GENERATOR OF BYTES """ decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines bytes_count = 0 for bytes_ in source: data = decompressor.decompress(bytes_) bytes_count += len(data) if Math.floor(last_bytes_count, 1000000) != Math.floor( bytes_count, 1000000): last_bytes_count = bytes_count if DEBUG: Log.note("bytes={{bytes}}", bytes=bytes_count) yield data
def floor(self, interval=None): if not isinstance(interval, Duration): from mo_logs import Log Log.error("Expecting an interval as a Duration object") output = Duration(0) if interval.month: if self.month: output.month = int(Math.floor(self.month / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month return output # A MONTH OF DURATION IS BIGGER THAN A CANONICAL MONTH output.month = int(Math.floor(self.milli * 12 / MILLI_VALUES["year"] / interval.month) * interval.month) output.milli = output.month * MILLI_VALUES.month else: output.milli = Math.floor(self.milli / (interval.milli)) * (interval.milli) return output
def icompressed2ibytes(source): """ :param source: GENERATOR OF COMPRESSED BYTES :return: GENERATOR OF BYTES """ decompressor = zlib.decompressobj(16 + zlib.MAX_WBITS) last_bytes_count = 0 # Track the last byte count, so we do not show too many debug lines bytes_count = 0 for bytes_ in source: try: data = decompressor.decompress(bytes_) except Exception as e: Log.error("problem", cause=e) bytes_count += len(data) if Math.floor(last_bytes_count, 1000000) != Math.floor(bytes_count, 1000000): last_bytes_count = bytes_count DEBUG and Log.note("bytes={{bytes}}", bytes=bytes_count) yield data
def pop(self, wait=SECOND, till=None): if till is not None and not isinstance(till, Signal): Log.error("Expecting a signal") m = self.queue.read(wait_time_seconds=Math.floor(wait.seconds)) if not m: return None self.pending.append(m) output = mo_json.json2value(m.get_body()) return output
def intervals(_min, _max=None, size=1): """ RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE THE LAST PAIR MAY BE SMALLER Yes! It's just like range(), only cooler! """ if _max == None: _max = _min _min = 0 _max = int(Math.ceiling(_max)) _min = int(Math.floor(_min)) output = ((x, min(x + size, _max)) for x in _range(_min, _max, size)) return output
def pop_message(self, wait=SECOND, till=None): """ RETURN TUPLE (message, payload) CALLER IS RESPONSIBLE FOR CALLING message.delete() WHEN DONE """ if till is not None and not isinstance(till, Signal): Log.error("Expecting a signal") message = self.queue.read(wait_time_seconds=Math.floor(wait.seconds)) if not message: return None message.delete = lambda: self.queue.delete_message(message) payload = mo_json.json2value(message.get_body()) return message, payload
def intervals(_min, _max=None, size=1): """ RETURN (min, max) PAIRS OF GIVEN SIZE, WHICH COVER THE _min, _max RANGE THE LAST PAIR MAY BE SMALLER Yes! It's just like range(), only cooler! """ if _max == None: _max = _min _min = 0 _max = int(Math.ceiling(_max)) _min = int(Math.floor(_min)) output = ((x, min(x + size, _max)) for x in __builtin__.range(_min, _max, size)) return output
def setup( self, instance, # THE boto INSTANCE OBJECT FOR THE MACHINE TO SETUP utility # THE utility OBJECT FOUND IN CONFIG ): with self.locker: self.instance = instance gigabytes = Math.floor(utility.memory) Log.note("setup {{instance}}", instance=instance.id) with hide('output'): self._config_fabric(instance) self._install_indexer() self._install_es(gigabytes) self._install_supervisor() self._start_supervisor()
def __unicode__(self): if not self.milli: return "zero" output = "" rest = (self.milli - (MILLI_VALUES.month * self.month) ) # DO NOT INCLUDE THE MONTH'S MILLIS isNegative = (rest < 0) rest = Math.abs(rest) # MILLI rem = rest % 1000 if rem != 0: output = "+" + text_type(rem) + "milli" + output rest = Math.floor(rest / 1000) # SECOND rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "second" + output rest = Math.floor(rest / 60) # MINUTE rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "minute" + output rest = Math.floor(rest / 60) # HOUR rem = rest % 24 if rem != 0: output = "+" + text_type(rem) + "hour" + output rest = Math.floor(rest / 24) # DAY if (rest < 11 and rest != 7) or rest % 10 == 0: rem = rest rest = 0 else: rem = rest % 7 rest = Math.floor(rest / 7) if rem != 0: output = "+" + text_type(rem) + "day" + output # WEEK if rest != 0: output = "+" + text_type(rest) + "week" + output if isNegative: output = output.replace("+", "-") # MONTH AND YEAR if self.month: sign = "-" if self.month < 0 else "+" month = Math.abs(self.month) if month <= 18 and month != 12: output = sign + text_type(month) + "month" + output else: m = month % 12 if m != 0: output = sign + text_type(m) + "month" + output y = Math.floor(month / 12) output = sign + text_type(y) + "year" + output if output[0] == "+": output = output[1::] if output[0] == '1' and not Math.is_number(output[1]): output = output[1::] return output
def add_instances(self, net_new_utility, remaining_budget): prices = self.pricing() for p in prices: if net_new_utility <= 0 or remaining_budget <= 0: break if p.current_price == None: Log.note("{{type}} has no current price", type=p.type.instance_type ) continue if self.settings.utility[p.type.instance_type].blacklist or \ p.availability_zone in listwrap(self.settings.utility[p.type.instance_type].blacklist_zones): Log.note("{{type}} in {{zone}} skipped due to blacklist", type=p.type.instance_type, zone=p.availability_zone) continue # DO NOT BID HIGHER THAN WHAT WE ARE WILLING TO PAY max_acceptable_price = p.type.utility * self.settings.max_utility_price + p.type.discount max_bid = Math.min(p.higher_price, max_acceptable_price, remaining_budget) min_bid = p.price_80 if min_bid > max_acceptable_price: Log.note( "Price of ${{price}}/hour on {{type}}: Over remaining acceptable price of ${{remaining}}/hour", type=p.type.instance_type, price=min_bid, remaining=max_acceptable_price ) continue elif min_bid > remaining_budget: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Over budget of ${{remaining_budget}}/hour", type=p.type.instance_type, bid=min_bid, remaining_budget=remaining_budget ) continue elif min_bid > max_bid: Log.error("not expected") naive_number_needed = int(Math.round(float(net_new_utility) / float(p.type.utility), decimal=0)) limit_total = None if self.settings.max_percent_per_type < 1: current_count = sum(1 for a in self.active if a.launch_specification.instance_type == p.type.instance_type and a.launch_specification.placement == p.availability_zone) all_count = sum(1 for a in self.active if a.launch_specification.placement == p.availability_zone) all_count = max(all_count, naive_number_needed) limit_total = int(Math.floor((all_count * self.settings.max_percent_per_type - current_count) / (1 - self.settings.max_percent_per_type))) num = Math.min(naive_number_needed, limit_total, self.settings.max_requests_per_type) if num < 0: Log.note( "{{type}} is over {{limit|percent}} of instances, no more requested", limit=self.settings.max_percent_per_type, type=p.type.instance_type ) continue elif num == 1: min_bid = Math.min(Math.max(p.current_price * 1.1, min_bid), max_acceptable_price) price_interval = 0 else: price_interval = Math.min(min_bid / 10, (max_bid - min_bid) / (num - 1)) for i in range(num): bid_per_machine = min_bid + (i * price_interval) if bid_per_machine < p.current_price: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Under current price of ${{current_price}}/hour", type=p.type.instance_type, bid=bid_per_machine - p.type.discount, current_price=p.current_price ) continue if bid_per_machine - p.type.discount > remaining_budget: Log.note( "Did not bid ${{bid}}/hour on {{type}}: Over remaining budget of ${{remaining}}/hour", type=p.type.instance_type, bid=bid_per_machine - p.type.discount, remaining=remaining_budget ) continue try: if self.settings.ec2.request.count == None or self.settings.ec2.request.count != 1: Log.error("Spot Manager can only request machine one-at-a-time") new_requests = self._request_spot_instances( price=bid_per_machine, availability_zone_group=p.availability_zone, instance_type=p.type.instance_type, kwargs=copy(self.settings.ec2.request) ) Log.note( "Request {{num}} instance {{type}} in {{zone}} with utility {{utility}} at ${{price}}/hour", num=len(new_requests), type=p.type.instance_type, zone=p.availability_zone, utility=p.type.utility, price=bid_per_machine ) net_new_utility -= p.type.utility * len(new_requests) remaining_budget -= (bid_per_machine - p.type.discount) * len(new_requests) with self.net_new_locker: for ii in new_requests: self.net_new_spot_requests.add(ii) except Exception as e: Log.warning( "Request instance {{type}} failed because {{reason}}", type=p.type.instance_type, reason=e.message, cause=e ) if "Max spot instance count exceeded" in e.message: Log.note("No further spot requests will be attempted.") return net_new_utility, remaining_budget return net_new_utility, remaining_budget
def __unicode__(self): if not self.milli: return "zero" output = "" rest = (self.milli - (MILLI_VALUES.month * self.month)) # DO NOT INCLUDE THE MONTH'S MILLIS isNegative = (rest < 0) rest = Math.abs(rest) # MILLI rem = rest % 1000 if rem != 0: output = "+" + text_type(rem) + "milli" + output rest = Math.floor(rest / 1000) # SECOND rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "second" + output rest = Math.floor(rest / 60) # MINUTE rem = rest % 60 if rem != 0: output = "+" + text_type(rem) + "minute" + output rest = Math.floor(rest / 60) # HOUR rem = rest % 24 if rem != 0: output = "+" + text_type(rem) + "hour" + output rest = Math.floor(rest / 24) # DAY if (rest < 11 and rest != 7) or rest % 10 == 0: rem = rest rest = 0 else: rem = rest % 7 rest = Math.floor(rest / 7) if rem != 0: output = "+" + text_type(rem) + "day" + output # WEEK if rest != 0: output = "+" + text_type(rest) + "week" + output if isNegative: output = output.replace("+", "-") # MONTH AND YEAR if self.month: sign = "-" if self.month < 0 else "+" month = Math.abs(self.month) if month <= 18 and month != 12: output = sign + text_type(month) + "month" + output else: m = month % 12 if m != 0: output = sign + text_type(m) + "month" + output y = Math.floor(month / 12) output = sign + text_type(y) + "year" + output if output[0] == "+": output = output[1::] if output[0] == '1' and not Math.is_number(output[1]): output = output[1::] return output