def _gen_queue_number(timestamp,
priority,
scale_factor_us=365 * 24 * 60 * 60 * 1000 * 1000):
"""Generates a 64 bit packed value used for TaskToRun.queue_number.
The lower value the higher importance.
Arguments:
- timestamp: datetime.datetime when the TaskRequest was filed in. This value
at 1µs is used for the FIFO ordering.
- priority: priority of the TaskRequest. It's a 8 bit integer. Lower is higher
priority.
- scale_factor_us: multiplicative factor of the priority versus time. Higher
values means that priority will become a strict stack of FIFO queues.
Lower values means a blend between priority and time, where lower
priority tasks (higher `priority` value) will slowly preempt higher
priority tasks (lower `priority` value) when they waited long enough.
Default is each priority increment is worth one year.
Returns:
queue_number is a 63 bit integer with timestamp at µs resolution plus
priority scaled with scale_factor_us as a delay factor.
"""
assert isinstance(timestamp, datetime.datetime), '%r' % timestamp
assert isinstance(priority, int), '%r' % priority
assert isinstance(scale_factor_us, (int, long)), '%r' % scale_factor_us
task_request.validate_priority(priority)
assert 0 <= priority <= 255, 'Just for clarity, validate_priority() checks it'
if not 0 <= scale_factor_us <= 2**54:
raise ValueError('Invalid scale_factor_us (%s)' % scale_factor_us)
value = int(
round((timestamp - utils.EPOCH).total_seconds() * 1000. * 1000.))
assert 0 <= value < 2**58, hex(value)
return value + priority * scale_factor_us
def _gen_queue_number(
timestamp, priority, scale_factor_us=365*24*60*60*1000*1000):
"""Generates a 64 bit packed value used for TaskToRun.queue_number.
The lower value the higher importance.
Arguments:
- timestamp: datetime.datetime when the TaskRequest was filed in. This value
at 1µs is used for the FIFO ordering.
- priority: priority of the TaskRequest. It's a 8 bit integer. Lower is higher
priority.
- scale_factor_us: multiplicative factor of the priority versus time. Higher
values means that priority will become a strict stack of FIFO queues.
Lower values means a blend between priority and time, where lower
priority tasks (higher `priority` value) will slowly preempt higher
priority tasks (lower `priority` value) when they waited long enough.
Default is each priority increment is worth one year.
Returns:
queue_number is a 63 bit integer with timestamp at µs resolution plus
priority scaled with scale_factor_us as a delay factor.
"""
assert isinstance(timestamp, datetime.datetime), '%r' % timestamp
assert isinstance(priority, int), '%r' % priority
assert isinstance(scale_factor_us, (int, long)), '%r' % scale_factor_us
task_request.validate_priority(priority)
assert 0 <= priority <= 255, 'Just for clarity, validate_priority() checks it'
if not 0 <= scale_factor_us <= 2**54:
raise ValueError('Invalid scale_factor_us (%s)' % scale_factor_us)
value = int(round((timestamp - utils.EPOCH).total_seconds() * 1000. * 1000.))
assert 0 <= value < 2**58, hex(value)
return value + priority * scale_factor_us
def _gen_queue_number(dimensions_hash, timestamp, priority):
"""Generates a 63 bit packed value used for TaskToRun.queue_number.
Arguments:
- dimensions_hash: 32 bit integer to classify in a queue.
- timestamp: datetime.datetime when the TaskRequest was filed in. This value
is used for FIFO or LIFO ordering (depending on configuration) with a
100ms granularity; the year is ignored.
- priority: priority of the TaskRequest. It's a 8 bit integer. Lower is higher
priority.
Returns:
queue_number is a 63 bit integer with dimension_hash, timestamp at 100ms
resolution plus priority.
"""
# dimensions_hash should be 32 bits but on AppEngine, which is using 32 bits
# python, it is silently upgraded to long.
assert isinstance(dimensions_hash, (int, long)), repr(dimensions_hash)
assert dimensions_hash > 0 and dimensions_hash <= 0xFFFFFFFF, hex(
dimensions_hash)
assert isinstance(timestamp, datetime.datetime), repr(timestamp)
task_request.validate_priority(priority)
# Ignore the year.
if config.settings().use_lifo:
next_year = datetime.datetime(timestamp.year + 1, 1, 1)
# It is guaranteed to fit 32 bits but upgrade to long right away to ensure
# assert works.
t = long(round((next_year - timestamp).total_seconds() * 10.))
else:
year_start = datetime.datetime(timestamp.year, 1, 1)
# It is guaranteed to fit 32 bits but upgrade to long right away to ensure
# assert works.
t = long(round((timestamp - year_start).total_seconds() * 10.))
assert t >= 0 and t <= 0x7FFFFFFF, (hex(t), dimensions_hash, timestamp,
priority)
# 31-22 == 9, leaving room for overflow with the addition.
# 0x3fc00000 is the priority mask.
# It is important that priority mixed with time is an addition, not a bitwise
# or.
low_part = (long(priority) << 22) + t
assert low_part >= 0 and low_part <= 0xFFFFFFFF, '0x%X is out of band' % (
low_part)
# int may be 32 bits, upgrade to long for consistency, albeit this is
# significantly slower.
high_part = long(dimensions_hash) << 31
return high_part | low_part
def test_validate_priority(self):
with self.assertRaises(TypeError):
task_request.validate_priority('1')
with self.assertRaises(datastore_errors.BadValueError):
task_request.validate_priority(-1)
with self.assertRaises(datastore_errors.BadValueError):
task_request.validate_priority(task_request.MAXIMUM_PRIORITY + 1)
task_request.validate_priority(0)
task_request.validate_priority(1)
task_request.validate_priority(task_request.MAXIMUM_PRIORITY)
def test_validate_priority(self):
with self.assertRaises(TypeError):
task_request.validate_priority('1')
with self.assertRaises(datastore_errors.BadValueError):
task_request.validate_priority(-1)
with self.assertRaises(datastore_errors.BadValueError):
task_request.validate_priority(task_request.MAXIMUM_PRIORITY+1)
task_request.validate_priority(0)
task_request.validate_priority(1)
task_request.validate_priority(task_request.MAXIMUM_PRIORITY)
