def iter_chunks(l, size):
"""
Returns a generator containing chunks of *size* of a list, integer or generator *l*. A *size*
smaller than 1 results in no chunking at all.
"""
if isinstance(l, six.integer_types):
l = six.range(l)
if isinstance(l, types.GeneratorType):
if size < 1:
yield list(l)
else:
chunk = []
for elem in l:
if len(chunk) < size:
chunk.append(elem)
else:
yield chunk
chunk = []
if chunk:
yield chunk
else:
if size < 1:
yield l
else:
for i in six.range(0, len(l), size):
yield l[i:i + size]
def test_simple_api(cluster):
client = cluster.get_routing_client()
with client.map() as map_client:
for x in six.range(10):
map_client.set('key:%d' % x, x)
map_client.zadd('zset:%d' % x, **dict(make_zset_data(x)))
for x in six.range(10):
assert client.get('key:%d' % x) == str(x)
assert client.zrange('zset:%d' % x, 0, -1,
withscores=True) == make_zset_data(x)
results = [] # (promise, expected result)
with client.map() as map_client:
for x in six.range(10):
results.append((
map_client.zrange('zset:%d' % x, 0, -1, withscores=True),
make_zset_data(x),
))
for promise, expectation in results:
assert promise.value == expectation
with client.map() as map_client:
for x in six.range(10):
map_client.delete('key:%d' % x)
for x in six.range(10):
assert client.get('key:%d' % x) is None
def test_promise_api(cluster):
results = []
with cluster.map() as client:
for x in six.range(10):
client.set('key-%d' % x, x)
for x in six.range(10):
client.get('key-%d' % x).then(lambda x: results.append(int(x)))
assert sorted(results) == list(six.range(10))
def test_simple_api(cluster, poll, monkeypatch):
monkeypatch.setattr(clients, 'poll', poll)
client = cluster.get_routing_client()
with client.map() as map_client:
for x in six.range(10):
map_client.set('key:%s' % x, x)
for x in six.range(10):
assert client.get('key:%d' % x) == str(x)
def test_basic_cluster_disabled_batch(cluster):
iterations = 10000
with cluster.map(auto_batch=False) as client:
for x in six.range(iterations):
client.set('key-%06d' % x, x)
responses = []
with cluster.map(auto_batch=False) as client:
for x in six.range(iterations):
responses.append(client.get('key-%06d' % x))
ref_sum = sum(int(x.value) for x in responses)
assert ref_sum == sum(six.range(iterations))
def test_reconnect(cluster):
with cluster.map() as client:
for x in six.range(10):
client.set(str(x), str(x))
with cluster.all() as client:
client.config_set('timeout', 1)
time.sleep(2)
with cluster.map() as client:
rv = Promise.all([client.get(str(x)) for x in six.range(10)])
assert rv.value == list(map(str, six.range(10)))
def create_block(self, val):
"""Create a solid monochrome box
Create a box of size `__box_size x __box_size`.
Args:
val:
Binary input. If `True` or non-0, creates a box made of `fg`
pixels, otherwise made of `bg` pixels
Returns:
Monochrome box
"""
if val: return [[self.fg]*self.get_box_size() for _ in six.range(self.get_box_size())]
else: return [[self.bg]*self.get_box_size() for _ in six.range(self.get_box_size())]
def _read_count_and_string_table(cls, bs):
""" Read a count then return the strings in a list """
result = []
entry_count = bs.read("uint:8")
for _ in six.range(0, entry_count):
result.append( cls._read_string(bs) )
return result
def train():
from keras.metrics import categorical_accuracy
TransformImage.register('flatten', lambda img: img.flatten())
transform = (TransformImage(0)
.by('rerange', 0, 255, 0, 1, 'float32')
.by('flatten'))
build_batch = (BuildBatch(BATCH_SIZE)
.by(0, 'vector', 'float32')
.by(1, 'one_hot', 'uint8', NUM_CLASSES))
plot = PlotLines((0, 1), layout=(2, 1), every_sec=1)
print('loading data...')
train_samples, test_samples = load_samples()
print('creating network ...')
network = create_network()
print('training...', NUM_EPOCHS)
for epoch in range(NUM_EPOCHS):
print('EPOCH:', epoch)
t_loss, t_acc = (train_samples >> PrintProgress(train_samples) >>
transform >> build_batch >>
network.train() >> plot >> Unzip())
print('train loss : {:.6f}'.format(t_loss >> Mean()))
print('train acc : {:.1f}'.format(100 * (t_acc >> Mean())))
e_acc = (test_samples >> transform >> build_batch >>
network.evaluate([categorical_accuracy]))
print('test acc : {:.1f}'.format(100 * e_acc))
network.save_best(e_acc, isloss=False)
def _parse_asrt(cls, box_bs):
""" Parse asrt / Segment Run Table Box """
asrt = SegmentRunTable()
asrt.header = cls._read_box_header(box_bs)
# read the entire box in case there's padding
asrt_bs_box = box_bs.read(asrt.header.box_size * 8)
asrt_bs_box.pos += 8
update_flag = asrt_bs_box.read("uint:24")
asrt.update = True if update_flag == 1 else False
asrt.quality_segment_url_modifiers = cls._read_count_and_string_table(
asrt_bs_box)
asrt.segment_run_table_entries = []
segment_count = asrt_bs_box.read("uint:32")
for _ in six.range(0, segment_count):
first_segment = asrt_bs_box.read("uint:32")
fragments_per_segment = asrt_bs_box.read("uint:32")
asrt.segment_run_table_entries.append(
SegmentRunTable.SegmentRunTableEntry(
first_segment=first_segment,
fragments_per_segment=fragments_per_segment))
return asrt
def assert_gapless_hosts(hosts):
if not hosts:
raise BadHostSetup('No hosts were configured.')
for x in six.range(len(hosts)):
if hosts.get(x) is None:
raise BadHostSetup('Expected host with ID "%d" but no such '
'host was found.' % x)
def fit_slopes_to_facet_lines(self,
polynomial_degree=4,
curvature_threshold=0.0004):
"""
Fits (linear) lines of best fit to extracted profiles, already stored as
class properties.
"""
avg_slopes_linear = []
avg_slopes_poly = []
curv_of_flattest_part_list = []
slope_min_curv = []
rsqd_list = []
big_slope_small_curv = []
elev_at_bssc = []
for i in six.range(len(self.profile_x_facet_pts)):
x = self.profile_x_facet_pts[i]
z = self.profile_z_facet_pts[i]
(grad, offset) = np.polyfit(x, z, 1)
coeffs, residuals = np.polyfit(x, z, polynomial_degree,
full=True)[:2]
rsqd = 1. - residuals / (z.size * z.var())
# differentiate the coeffs to get slope:
diff_multiplier = np.arange(polynomial_degree + 1)[::-1]
curv_multiplier = np.arange(polynomial_degree)[::-1]
z_equ = np.poly1d(coeffs)
S_equ = np.poly1d((coeffs * diff_multiplier)[:-1])
curv_equ = np.poly1d(
((coeffs * diff_multiplier)[:-1] * curv_multiplier)[:-1])
S_at_each_pt = S_equ(x)
curv_at_each_pt = curv_equ(x)
avg_slopes_linear.append(abs(grad))
avg_slopes_poly.append(np.amax(np.fabs(S_at_each_pt)))
loc_of_flattest_part = np.argmin(np.fabs(
curv_at_each_pt[2:-2])) + 2
curv_of_flattest_part = curv_at_each_pt[loc_of_flattest_part]
S_at_min_curve_untested = abs(S_at_each_pt[loc_of_flattest_part])
small_curves = np.less(np.fabs(curv_at_each_pt[2:-2]),
curvature_threshold)
try:
big_slope_small_curv.append(np.amax(
S_at_each_pt[small_curves]))
elev_at_bssc.append(z[np.argmax(S_at_each_pt[small_curves])])
except ValueError:
big_slope_small_curv.append(np.nan)
elev_at_bssc.append(np.nan)
slope_min_curv.append(S_at_min_curve_untested)
curv_of_flattest_part_list.append(curv_of_flattest_part)
rsqd_list.append(rsqd)
# figure(8)
# synthetic_z = grad*x + offset
synthetic_z = z_equ(x)
plot(x, z, "x")
plot(x, synthetic_z, "-")
self.avg_slopes_linear = np.array(avg_slopes_linear)
self.avg_slopes_poly = np.array(avg_slopes_poly)
self.curv_of_flattest_part = np.array(curv_of_flattest_part_list)
self.slope_min_curv = np.array(slope_min_curv)
self.big_slope_small_curv = np.array(big_slope_small_curv)
self.elev_at_bssc = np.array(elev_at_bssc)
self.rsqd = np.array(rsqd_list)
def __init__(self,
servers=4,
databases_each=8,
server_executable='redis-server'):
self._fd_dir = tempfile.mkdtemp()
self.databases_each = databases_each
self.server_executable = server_executable
self.servers = []
for server in six.range(servers):
self.spawn_server()
def main(request, response):
delay = float(request.GET.first(b"ms", 500)) / 1E3
count = int(request.GET.first(b"count", 50))
# Read request body
request.body
time.sleep(delay)
response.headers.set(b"Content-type", b"text/plain")
response.write_status_headers()
time.sleep(delay)
for i in range(count):
response.writer.write_content(b"TEST_TRICKLE\n")
time.sleep(delay)
def test(k):
data = {}
for i in six.range(1000):
tower = k.get_node('a%s' % i)
data.setdefault(tower, 0)
data[tower] += 1
return [
k.get_node('Apple'),
k.get_node('Hello'),
k.get_node('Data'),
k.get_node('Computer')
]
def _rebuild_circle(self):
"""Updates the hash ring."""
self._hashring = {}
self._sorted_keys = []
total_weight = 0
for node in self._nodes:
total_weight += self._weights.get(node, 1)
for node in self._nodes:
weight = self._weights.get(node, 1)
ks = math.floor((40 * len(self._nodes) * weight) / total_weight)
for i in six.range(0, int(ks)):
k = md5_bytes('%s-%s-salt' % (node, i))
for l in six.range(0, 4):
key = ((k[3 + l * 4] << 24) | (k[2 + l * 4] << 16) |
(k[1 + l * 4] << 8) | k[l * 4])
self._hashring[key] = node
self._sorted_keys.append(key)
self._sorted_keys.sort()
def make_cluster(self):
"""Creates a correctly configured cluster from the servers
spawned. This also automatically waits for the servers to be up.
"""
self.wait_for_servers()
hosts = []
host_id = 0
for server in self.servers:
for x in six.range(self.databases_each):
hosts.append({
'host_id': host_id,
'unix_socket_path': server.socket_path,
'db': x,
})
host_id += 1
return Cluster(hosts)
def _parse_afrt(cls, box_bs):
""" Parse afrt / Fragment Run Table Box """
afrt = FragmentRunTable()
afrt.header = cls._read_box_header(box_bs)
# read the entire box in case there's padding
afrt_bs_box = box_bs.read(afrt.header.box_size * 8)
afrt_bs_box.pos += 8
update_flag = afrt_bs_box.read("uint:24")
afrt.update = True if update_flag == 1 else False
afrt.time_scale = afrt_bs_box.read("uint:32")
afrt.quality_fragment_url_modifiers = cls._read_count_and_string_table(
afrt_bs_box)
fragment_count = afrt_bs_box.read("uint:32")
afrt.fragments = []
for _ in six.range(0, fragment_count):
first_fragment = afrt_bs_box.read("uint:32")
first_fragment_timestamp_raw = afrt_bs_box.read("uint:64")
try:
first_fragment_timestamp = datetime.utcfromtimestamp(
first_fragment_timestamp_raw / float(afrt.time_scale))
except ValueError:
# Elemental sometimes create odd timestamps
first_fragment_timestamp = None
fragment_duration = afrt_bs_box.read("uint:32")
if fragment_duration == 0:
discontinuity_indicator = afrt_bs_box.read("uint:8")
else:
discontinuity_indicator = None
frte = FragmentRunTable.FragmentRunTableEntry(
first_fragment=first_fragment,
first_fragment_timestamp=first_fragment_timestamp,
fragment_duration=fragment_duration,
discontinuity_indicator=discontinuity_indicator)
afrt.fragments.append(frte)
return afrt
def train():
from keras.metrics import categorical_accuracy
rerange = TransformImage(0).by('rerange', 0, 255, 0, 1, 'float32')
build_batch = (BuildBatch(BATCH_SIZE).by(0, 'image', 'float32').by(
1, 'one_hot', 'uint8', NUM_CLASSES))
p = 0.1
augment = (AugmentImage(0).by('identical', 1.0).by(
'brightness', p, [0.7, 1.3]).by('color', p, [0.7, 1.3]).by(
'shear', p, [0, 0.1]).by('fliplr', p).by('rotate', p, [-10, 10]))
plot_eval = PlotLines((0, 1), layout=(2, 1))
print('creating network...')
network = create_network()
print('loading data...')
train_samples, test_samples = load_samples()
train_samples, val_samples = train_samples >> SplitRandom(0.8)
print('training...', len(train_samples), len(val_samples))
for epoch in range(NUM_EPOCHS):
print('EPOCH:', epoch)
t_loss, t_acc = (train_samples >> PrintProgress(train_samples) >>
Pick(PICK) >> augment >> rerange >> Shuffle(100) >>
build_batch >> network.train() >> Unzip())
t_loss, t_acc = t_loss >> Mean(), t_acc >> Mean()
print("train loss : {:.6f}".format(t_loss))
print("train acc : {:.1f}".format(100 * t_acc))
v_loss, v_acc = (val_samples >> rerange >> build_batch >>
network.validate() >> Unzip())
v_loss, v_acc = v_acc >> Mean(), v_acc >> Mean()
print('val loss : {:.6f}'.format(v_loss))
print('val acc : {:.1f}'.format(100 * v_acc))
network.save_best(v_acc, isloss=False)
plot_eval((t_acc, v_acc))
print('testing...', len(test_samples))
e_acc = (test_samples >> rerange >> build_batch >> network.evaluate(
[categorical_accuracy]))
print('test acc : {:.1f}'.format(100 * e_acc))
def get_connection(self, command_name, shard_hint=None):
host_id = shard_hint
if host_id is None:
raise RuntimeError('The routing pool requires the host id '
'as shard hint')
real_pool = self.cluster.get_pool_for_host(host_id)
# When we check something out from the real underlying pool it's
# very much possible that the connection is stale. This is why we
# check out up to 10 connections which are either not connected
# yet or verified alive.
for _ in six.range(10):
con = real_pool.get_connection(command_name)
if con._sock is None or not is_closed(con._sock):
con.__creating_pool = weakref(real_pool)
return con
raise ConnectionError('Failed to check out a valid connection '
'(host %s)' % host_id)
def create_block_gradient_alpha(self, val):
"""Create a box with variable alpha channel
Create a box of size `__box_size x __box_size` with its transparancy
scaled to `val`
Args:
val:
Scaling factor ranging from 0 to `q_max`. Sets the value of the
alpha channel of the current box. The scaling is done using
`val / q_max`. The color is set to `fg`
Returns:
A single color box with transparancy set to `val/q_max`
"""
color = self.fg[:]
# self.q_max = float('inf')
if type(color) is int or type(color) is float:
color = [color]*4
elif type(color) is list:
while len(color) < 4:
color += [0] # color = color + [0]
color[3] = val*1. / self.q_max
return [[color]*self.get_box_size() for _ in six.range(self.get_box_size())]
def _parse_abst(cls, bootstrap_bs, header):
abst = BootStrapInfoBox()
abst.header = header
box_bs = bootstrap_bs.read(abst.header.box_size * 8)
abst.version, abst.profile_raw, abst.live, abst.update, \
abst.time_scale, abst.current_media_time, abst.smpte_timecode_offset = \
box_bs.readlist("""pad:8, pad:24, uint:32, uint:2, bool, bool,
pad:4,
uint:32, uint:64, uint:64""")
abst.movie_identifier = cls._read_string(box_bs)
abst.server_entry_table = cls._read_count_and_string_table(box_bs)
abst.quality_entry_table = cls._read_count_and_string_table(box_bs)
abst.drm_data = cls._read_string(box_bs)
abst.meta_data = cls._read_string(box_bs)
abst.segment_run_tables = []
segment_count = box_bs.read("uint:8")
log.debug("segment_count: %d" % segment_count)
for _ in six.range(0, segment_count):
abst.segment_run_tables.append( cls._parse_asrt(box_bs) )
abst.fragment_tables = []
fragment_count = box_bs.read("uint:8")
log.debug("fragment_count: %d" % fragment_count)
for _ in xrange(0, fragment_count):
abst.fragment_tables.append( cls._parse_afrt(box_bs) )
log.debug("Finished parsing abst")
return abst
def make_zset_data(x):
return [(str(i), float(i)) for i in six.range(x, x + 10)]
def create_line(self, rot=0, base = None):
"""Create a line on a 8x8 grid box
Args:
rot: Rotation of the line. 0 represents vertical line (range 0-15)
Returns:
`__box_size x __box_size` np.array() with a line
NOTE:
The rotation is in the increments of 11.25 degrees clockwise
"""
if rot == -1:
return [[self.bg]*self.get_box_size() for _ in six.range(self.get_box_size())]
assert (0 <= rot <= 15)
rot = rot*11.25
# In case we want to draw something else:
if base is None:
# res = np.full((self.get_box_size(), self.get_box_size()), self.bg)
res = [[self.bg]*self.get_box_size() for _ in six.range(self.get_box_size())]
def bound_point(p):
while p[0] < 0: p[0] += 1
while p[0] >= self.get_box_size(): p[0] -= 1
while p[1] < 0: p[1] += 1
while p[1] >= self.get_box_size(): p[1] -= 1
return p
# conv = lambda x: map(int, np.floor(x))
def conv(x): # Readability
return map(int, np.floor(x))
## The reason the indeces are extending to -2 and 10 is because the
## rotation is calculated as sin/cos, meaning, we need a long line to
## fill the diagonals
if self.get_box_size() % 2 == 0:
## 4 different origins for rotation
# First quadrant
origin = [(self.get_box_size() - 1) / 2, (self.get_box_size() - 1) / 2]
for idx in six.range(-2, origin[1] + 1):
#for jdx in six.range(4):
jdx = origin[1]
point = self.rotate_point(origin, [idx, jdx], rot)
point = conv(point)
point = bound_point(point)
res[point[0]][point[1]] = self.fg
# Second quadrant
origin[1] += 1
for idx in six.range(-2, origin[1] + 1):
jdx = origin[1]
point = self.rotate_point(origin, [idx, jdx], rot)
point = conv(point)
point = bound_point(point)
res[point[0]][point[1]] = self.fg
# Third quadrant
origin[0] += 1
for idx in six.range(origin[1] + 1, self.get_box_size() + 2):
jdx = origin[1]
point = self.rotate_point(origin, [idx, jdx], rot)
point = conv(point)
point = bound_point(point)
res[point[0]][point[1]] = self.fg
# Fourth quadrant
origin[1] -= 1
for idx in six.range(origin[1] + 1, self.get_box_size() + 2):
jdx = origin[1]
point = self.rotate_point(origin, [idx, jdx], rot)
point = conv(point)
point = bound_point(point)
res[point[0]][point[1]] = self.fg
else:
## Single rotation origin
origin = [self.get_box_size() / 2, self.get_box_size() / 2]
for idx in six.range(self.get_box_size()):
jdx = origin[1]
point = self.rotate_point(origin, [idx, jdx], rot)
point = conv(point)
point = bound_point(point)
res[point[0]][point[1]] = self.fg
return res
def _parse_afra(cls, bs, header):
afra = FragmentRandomAccessBox()
afra.header = header
# read the entire box in case there's padding
afra_bs = bs.read(header.box_size * 8)
# skip Version and Flags
afra_bs.pos += 8 + 24
long_ids, long_offsets, global_entries, afra.time_scale, local_entry_count = \
afra_bs.readlist("bool, bool, bool, pad:5, uint:32, uint:32")
if long_ids:
id_bs_type = "uint:32"
else:
id_bs_type = "uint:16"
if long_offsets:
offset_bs_type = "uint:64"
else:
offset_bs_type = "uint:32"
log.debug("local_access_entries entry count: %s", local_entry_count)
afra.local_access_entries = []
for _ in six.range(0, local_entry_count):
time = cls._parse_time_field(afra_bs, afra.time_scale)
offset = afra_bs.read(offset_bs_type)
afra_entry = \
FragmentRandomAccessBox.FragmentRandomAccessBoxEntry(time=time,
offset=offset)
afra.local_access_entries.append(afra_entry)
afra.global_access_entries = []
if global_entries:
global_entry_count = afra_bs.read("uint:32")
log.debug("global_access_entries entry count: %s", global_entry_count)
for _ in six.range(0, global_entry_count):
time = cls._parse_time_field(afra_bs, afra.time_scale)
segment_number = afra_bs.read(id_bs_type)
fragment_number = afra_bs.read(id_bs_type)
afra_offset = afra_bs.read(offset_bs_type)
sample_offset = afra_bs.read(offset_bs_type)
afra_global_entry = \
FragmentRandomAccessBox.FragmentRandomAccessBoxGlobalEntry(
time=time,
segment_number=segment_number,
fragment_number=fragment_number,
afra_offset=afra_offset,
sample_offset=sample_offset)
afra.global_access_entries.append(afra_global_entry)
return afra
