def _calibrate_adj_bat(batln_x, batln_y, factor, reverse_factor):
"""Employs substitution and substitution choice rules to calibrate between two adjacent battalions."""
units_to_batln_y = _utils.round_up(factor *
batln_x.get_deficient_units())
batln_y.add_to_required_units(units_to_batln_y)
batln_x.update_required_units(batln_x.base_units)
if batln_y.is_deficient():
units_to_batln_x = _utils.round_up(batln_y.get_deficient_units() *
reverse_factor)
batln_x.add_to_required_units(units_to_batln_x)
batln_y.remove_from_required_units(batln_y.get_deficient_units())
def elastic_possible(self, node, task):
# Check that the task is not an MRAM container:
if task.type is YarnContainerType.MRAM:
return False
# Check if the task has an associated penalty function.
if task.penalty is None:
LOG.error("Task " + str(task) + " has no associated penalty function. ELASTIC disabled.")
return False
# Check if elastic behavior has been disabled as part of symbex or PEEK.
if self.disable_elastic:
return False
# Check that if node pools are set, and the current node is part
# of a non-elastic pool, we don't attempt to be elastic.
if self.elastic_pool is not None and node.node_id > (self.node_count * 1.0 * self.elastic_pool / 100):
return False
# Check that sufficient cores exist on this node.
if node.available.vcores < task.resource.vcores:
return False
# Next check that the minimum memory required exists
if node.available.memory_mb < round_up(task.resource.memory_mb * ELASTIC_MEM_MIN_RATIO / 100, MEM_INCREMENT):
return False
return True
def elastic_possible(self, node, task):
# Check that the task is not an MRAM container:
if task.type is YarnContainerType.MRAM:
return False
# Check if the task has an associated penalty function.
if task.penalty is None:
LOG.error("Task " + str(task) +
" has no associated penalty function. ELASTIC disabled.")
return False
# Check if elastic behavior has been disabled as part of symbex or PEEK.
if self.disable_elastic:
return False
# Check that if node pools are set, and the current node is part
# of a non-elastic pool, we don't attempt to be elastic.
if self.elastic_pool is not None and node.node_id > (
self.node_count * 1.0 * self.elastic_pool / 100):
return False
# Check that sufficient cores exist on this node.
if node.available.vcores < task.resource.vcores:
return False
# Next check that the minimum memory required exists
if node.available.memory_mb < round_up(
task.resource.memory_mb * ELASTIC_MEM_MIN_RATIO / 100,
MEM_INCREMENT):
return False
return True
def create_container_from_task(self, node, allocated, job, task):
# XXX: This code duplicates the code from YarnScheduler. Needs refactoring.
err_adjusted_duration = duration = task.duration
if task.penalty is not None:
mem_error_adjustment = self.mem_errors[job.job_id] if job.job_id in self.mem_errors else 0
ib_error_adjustment = self.ib_errors[job.job_id] if job.job_id in self.ib_errors else 0
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_ELASTIC_SCHEDULER: MEM_ERR_ADJUST: {}, IB_ERR_ADJUST: {}".format(mem_error_adjustment,
ib_error_adjustment))
resource = task.ideal_resource
if mem_error_adjustment != 0:
resource = YarnResource(task.ideal_resource.memory_mb, task.ideal_resource.vcores)
resource.memory_mb += mem_error_adjustment
resource.memory_mb = round_up(resource.memory_mb, MEM_INCREMENT)
if allocated < resource:
err_adjusted_duration = task.penalty.get_penalized_runtime(resource, allocated, task.duration,
error_offset=ib_error_adjustment)
duration = task.penalty.get_penalized_runtime(task.ideal_resource, allocated, task.duration)
# Create YarnRunningContainer from YarnPrototypeContainer
yarn_container = YarnRunningContainer(
container_id=job.get_new_container_id(),
duration=err_adjusted_duration,
resource=allocated,
priority=task.priority,
container_type=task.type,
job=task.job,
node=node,
task=task)
yarn_container.duration_error = err_adjusted_duration - duration
if yarn_container.is_elastic:
# Update decision count
self.stats_elastic_decisions_inc(task.job.job_id)
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_ELASTIC_SCHEDULER: Allocated {} elasticly for {}:{} on node {} "
"with penalized runtime of {} (err adjusted: {}) for a request of {} "
"and a duration of {}".format(
allocated.memory_mb, job.get_name(), job.next_container_id, node, duration,
err_adjusted_duration, task.resource.memory_mb, task.duration
))
else:
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_ELASTIC_SCHEDULER: Allocated {} regularly for {}:{} on node {} "
"with a duration of: {} (err adjusted: {})".format(
allocated.memory_mb, job.get_name(), job.next_container_id, node, task.duration,
err_adjusted_duration
))
return yarn_container
def prepare_battalions(self, counter_attack):
"""Prepares all the battalions in order using power rule."""
if self.validate_counter_attack(counter_attack):
# Helps to know what the counter attack is if needed later.
self.counter_attack = counter_attack
for batln_name in self.counter_attack:
required_units = _utils.round_up(counter_attack[batln_name],
half_it=True)
batln = Battalion(
army_name=self.army_name,
battalion_initials=batln_name,
**army_data['army'][self.army_name][batln_name],
required_units=required_units)
self.battalions.append(batln)
def find_closest():
how_long = 2.8
start = time.time()
first_distance = monster_distance()
if first_distance is None:
return None
time_dist = [utils.round_up(time.time() - start, 1), first_distance]
while True:
keyboard.press('d')
new_distance = monster_distance()
if new_distance is None:
return None
if time_dist[1] > new_distance:
time_dist = [utils.round_up(time.time() - start, 1), new_distance]
continue
if utils.round_up(time.time() - start, 1) == how_long:
keyboard.release('d')
break
# turn to target
turn_to_closest = time_dist[1]
if time_dist[0] == 0.0:
print('at begining')
return time_dist[1]
second_start = time.time()
keyboard.press('d')
while True:
print(utils.round_up(time.time() - second_start, 1))
second_new_distance = monster_distance()
if second_new_distance is None:
return None
if utils.round_up(second_new_distance,
1) == utils.round_up(turn_to_closest, 1):
print('found')
time.sleep(0.2)
keyboard.release('d')
return time_dist[1]
if utils.round_up(time.time() - second_start, 1) >= how_long:
print('not found in time')
keyboard.release('d')
return find_closest()
def allocate_on_node(self, node, task):
# First try being regular
regular = YarnRegularScheduler.allocate_on_node(self, node, task)
if regular is not None:
return regular
# Check if elasticity is possible
if not self.elastic_possible(node, task):
return None
# Check what is the best running time in these resources
min_mem_allocatable = round_up(
task.resource.memory_mb * ELASTIC_MEM_MIN_RATIO / 100,
MEM_INCREMENT)
if min_mem_allocatable < YARN_MIN_ALLOCATION_MB:
min_mem_allocatable = YARN_MIN_ALLOCATION_MB
min_mem_for_runtime = node.available.memory_mb
tmp_resource = YarnResource(min_mem_for_runtime, task.resource.vcores)
min_runtime = YarnElasticScheduler.get_penalized_runtime(
task.penalty, self.state.user_config.mem_overestimate_assume,
task.resource, tmp_resource, task.duration)
for mem in xrange(min_mem_allocatable, node.available.memory_mb,
MEM_INCREMENT):
tmp_resource.memory_mb = mem
penalized_runtime = YarnElasticScheduler.get_penalized_runtime(
task.penalty, self.state.user_config.mem_overestimate_assume,
task.resource, tmp_resource, task.duration)
if penalized_runtime < min_runtime:
min_runtime = penalized_runtime
min_mem_for_runtime = mem
elif penalized_runtime == min_runtime and mem < min_mem_for_runtime:
min_mem_for_runtime = mem
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_SMARTG_SCHEDULER: SMARTG possible with " +
str(min_mem_for_runtime) + " for " +
task.job.get_name() + ":" +
str(task.job.next_container_id) + " on node " +
str(node) + " with penalized runtime of " +
str(min_runtime) + " for a request of " +
str(task.resource.memory_mb) + " and a duration of: " +
str(task.duration))
return YarnResource(min_mem_for_runtime,
task.resource.vcores), min_runtime
def allocate_on_node(self, node, task):
# First try being regular
regular = YarnRegularScheduler.allocate_on_node(self, node, task)
if regular is not None:
return regular
# Check if elasticity is possible
if not self.elastic_possible(node, task):
return None
# Check what is the best running time in these resources
min_mem_allocatable = round_up(task.resource.memory_mb * ELASTIC_MEM_MIN_RATIO / 100, MEM_INCREMENT)
if min_mem_allocatable < YARN_MIN_ALLOCATION_MB:
min_mem_allocatable = YARN_MIN_ALLOCATION_MB
min_mem_for_runtime = node.available.memory_mb
tmp_resource = YarnResource(min_mem_for_runtime, task.resource.vcores)
min_runtime = YarnElasticScheduler.get_penalized_runtime(task.penalty,
self.state.user_config.mem_overestimate_assume,
task.resource, tmp_resource, task.duration)
for mem in xrange(min_mem_allocatable, node.available.memory_mb, MEM_INCREMENT):
tmp_resource.memory_mb = mem
penalized_runtime = YarnElasticScheduler.get_penalized_runtime(
task.penalty, self.state.user_config.mem_overestimate_assume, task.resource, tmp_resource, task.duration
)
if penalized_runtime < min_runtime:
min_runtime = penalized_runtime
min_mem_for_runtime = mem
elif penalized_runtime == min_runtime and mem < min_mem_for_runtime:
min_mem_for_runtime = mem
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_SMARTG_SCHEDULER: SMARTG possible with " +
str(min_mem_for_runtime) + " for " + task.job.get_name() +
":" + str(task.job.next_container_id) + " on node " +
str(node) + " with penalized runtime of " +
str(min_runtime) + " for a request of " +
str(task.resource.memory_mb) + " and a duration of: " +
str(task.duration))
return YarnResource(min_mem_for_runtime, task.resource.vcores), min_runtime
def create_container_from_task(self, node, allocated, job, task):
err_adjusted_duration = duration = task.duration
if task.penalty is not None:
mem_error_adjustment = self.mem_errors[
job.job_id] if job.job_id in self.mem_errors else 0
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug("YARN_REGULAR_SCHEDULER: MEM_ERR_ADJUST: {}".format(
mem_error_adjustment))
resource = task.ideal_resource
if mem_error_adjustment != 0:
resource = YarnResource(task.ideal_resource.memory_mb,
task.ideal_resource.vcores)
resource.memory_mb += mem_error_adjustment
resource.memory_mb = round_up(resource.memory_mb,
MEM_INCREMENT)
if allocated < resource:
err_adjusted_duration = task.penalty.get_penalized_runtime(
resource, allocated, task.duration)
duration = task.penalty.get_penalized_runtime(
task.ideal_resource, allocated, task.duration)
# Create YarnRunningContainer from YarnPrototypeContainer
yarn_container = YarnRunningContainer(
container_id=job.get_new_container_id(),
duration=err_adjusted_duration,
resource=task.resource,
priority=task.priority,
container_type=task.type,
job=task.job,
node=node,
task=task)
yarn_container.duration_error = err_adjusted_duration - duration
return yarn_container
def run_pp(self, etdata, pp=True, **kwargs):
'''Performs post-processing sanity check and hard-removes or replaces events
Parameters:
etdata -- instance of ETData()
pp -- if True, post-processing is performed. Otherwise only counts cases
Returns:
Numpy array of updated event stream
Numpy array of updated status
Dictionary with event status accumulator
'''
_evt = etdata.calc_evt(fast=True)
status = etdata.data['status']
fs = etdata.fs
check = self.check_accum
#%%pp
### Saccade sanity check
_sacc = _evt.query('evt==2')
check['saccades']+=len(_sacc)
#check isi
_isi = (_sacc[1:]['s'].values-_sacc[:-1]['e'].values)/fs
_isi_inds = np.where(_isi<kwargs['thres_isi']/1000.)[0]
check['sacc_isi']+=len(_isi_inds)
self.check_inds['sacc_isi'] = _sacc.index[_isi_inds].values
#TODO: implement isi merging
# if pp:
#
# _etdata = copy.deepcopy(etdata)
# _evt_unfold = [_e for _, e in _evt.iterrows()
# for _e in itertools.repeat(e['evt'],
# int(np.diff(e[['s', 'e']])))]
#
# _etdata.data['evt'] = _evt_unfold
# _etdata.calc_evt(fast=True)
# _evt = _etdata.evt
#pp: remove short saccades
# _sdur_thres = max([0.006, float(3/etdata.fs)])
# _sdur = _evt.query('evt==2 and dur<@_sdur_thres')
thres_sd_lo=kwargs['thres_sd_lo']/1000.
thres_sd_lo_s = round_up(thres_sd_lo*fs)
_sdur = _evt.query('evt==2 and (dur<@thres_sd_lo or dur_s<@thres_sd_lo_s)')
check['short_saccades']+=len(_sdur)
self.check_inds['short_saccades'] = _sdur.index.values
if pp:
_evt.loc[_sdur.index, 'evt'] = 0
#check long saccades.
thres_sd_hi=kwargs['thres_sd_hi']/1000.
thres_sd_hi_s = round_up(thres_sd_hi*fs)
_sdur = _evt.query('evt==2 and dur_s>@thres_sd_hi_s')
check['long_saccades']+=len(_sdur)
self.check_inds['long_saccades'] = _sdur.index.values
if pp:
_evt.loc[_sdur.index, 'evt'] = 0
#pp: find saccades surrounding undef;
_sacc_check={'202':0, '20':0, '02':0}
seq=''.join(map(str, _evt['evt']))
for pattern in _sacc_check.keys():
_check = np.array([m.start() for m in re.finditer('(?=%s)'%pattern, seq)])
if not (len(_check)):
continue
_sacc_check[pattern]+=len(_check)
self.check_inds['sacc%s'%pattern] = _check
# #pp: remove saccades surrounding undef; not used anymore
# if pp:
# if (pattern=='202'):
# assert ((_evt.loc[_check+1, 'evt']==0).all() and
# (_evt.loc[_check+2, 'evt']==2).all())
# _evt.loc[_check, 'evt'] = 0
# _evt.loc[_check+2, 'evt'] = 0
#
## if (pattern=='20'):
## assert (_evt.loc[_check+1, 'evt']==0).all()
## _evt.loc[_check, 'evt'] = 0
## if (pattern=='02'):
## assert (_evt.loc[_check+1, 'evt']==2).all()
## _evt.loc[_check+1, 'evt'] = 0
# seq=''.join(map(str, _evt['evt']))
check['sacc202']+=_sacc_check['202']
check['sacc20']+=_sacc_check['20']
check['sacc02']+=_sacc_check['02']
###PSO sanity check
check['pso']+=len(_evt.query('evt==3'))
#pp: change short PSOs to fixations; not used
# thres_pd = kwargs['thres_pd']/1000.
# thres_pd_s = round_up(thres_pd*fs)
# _pdur = _evt.query('evt==3 and (dur<@thres_pd or dur_s<@thres_pd_s)')
# check['short_pso']+=len(_pdur)
# self.check_inds['short_pso'] = _pdur.index.values
# if pp:
# _evt.loc[_pdur.index, 'evt'] = 1
#pp: remove unreasonable psos
_pso_check={'13':0, '03':0, '23':0 }
seq=''.join(map(str, _evt['evt']))
for pattern in _pso_check.keys():
_check = np.array([m.start() for m in re.finditer('(?=%s)'%pattern, seq)])
if not (len(_check)):
continue
_pso_check[pattern]+=len(_check)
self.check_inds['pso%s'%pattern] = _check
#pp: change PSOs after fixations to fixations
if pp:
if (pattern=='13'):
assert (_evt.loc[_check+1, 'evt']==3).all()
_evt.loc[_check+1, 'evt'] = 1
#pp: change PSOs after undef to undef
if (pattern=='03'):
assert (_evt.loc[_check+1, 'evt']==3).all()
_evt.loc[_check+1, 'evt'] = 0
check['pso23']+=_pso_check['23']
check['pso13']+=_pso_check['13']
check['pso03']+=_pso_check['03']
###fixation sanity check
#unfold and recalculate event data
_evt_unfold = [_e for _, e in _evt.iterrows()
for _e in itertools.repeat(e['evt'],
int(np.diff(e[['s', 'e']])))]
_etdata = copy.deepcopy(etdata)
_etdata.data['evt'] = _evt_unfold
_etdata.calc_evt(fast=True)
_evt = _etdata.evt
check['fixations']+=len(_evt.query('evt==1'))
#pp: remove short fixations
thres_fd=kwargs['thres_fd']/1000.
thres_fd_s = round_up(thres_fd*fs)
_fdur = _evt.query('evt==1 and (dur<@thres_fd or dur_s<@thres_fd_s)')
check['short_fixations']+=len(_fdur)
self.check_inds['short_fixations'] = _fdur.index.values
#TODO:
#check fixation merge
if pp:
_inds = np.array(_fdur.index)
_evt.loc[_inds, 'evt'] = 0
# #check if there are saccades or psos left around newly taged undef
# #so basically +- 2 events around small fixation
# _inds = np.unique(np.concatenate((_inds, _inds+1, _inds-1, _inds+2, _inds-2)))
# _inds = _inds[(_inds>-1) & (_inds<len(_evt))]
# _mask =_evt.loc[_inds, 'evt'].isin([2, 3])
# _evt.loc[_inds[_mask.values], 'evt'] = 0
##return result
_evt_unfold = [_e for _, e in _evt.iterrows()
for _e in itertools.repeat(e['evt'], int(np.diff(e[['s', 'e']])))]
assert len(_evt_unfold) == len(status)
status[np.array(_evt_unfold)==0] = False
return np.array(_evt_unfold), status, check, self.check_inds
def make_signature(arch_macho, arch_offset, arch_size, cmds, f,
entitlements_file, codesig_data_length, signer, ident):
# NB: arch_offset is absolute in terms of file start. Everything else is relative to arch_offset!
# sign from scratch
log.debug("signing from scratch")
drs = None
drs_lc = cmds.get('LC_DYLIB_CODE_SIGN_DRS')
if drs_lc:
drs = drs_lc.data.blob
codesig_offset = utils.round_up(arch_size, 16)
# generate code hashes
log.debug("codesig offset: {}".format(codesig_offset))
codeLimit = codesig_offset
log.debug("new cL: {}".format(hex(codeLimit)))
nCodeSlots = int(math.ceil(float(codesig_offset) / 0x1000))
log.debug("new nCS: {}".format(nCodeSlots))
# generate placeholder LC_CODE_SIGNATURE (like what codesign_allocate does)
fake_hashes = ["\x00" * 20] * nCodeSlots
codesig_cons = make_basic_codesig(entitlements_file, drs, codeLimit,
fake_hashes, signer, ident)
codesig_data = macho_cs.Blob.build(codesig_cons)
cmd_data = construct.Container(dataoff=codesig_offset,
datasize=codesig_data_length)
cmd = construct.Container(cmd='LC_CODE_SIGNATURE',
cmdsize=16,
data=cmd_data,
bytes=macho.CodeSigRef.build(cmd_data))
log.debug("CS blob before: {}".format(
utils.print_structure(codesig_cons, macho_cs.Blob)))
log.debug("len(codesig_data): {}".format(len(codesig_data)))
codesig_length = codesig_data_length
log.debug("codesig length: {}".format(codesig_length))
log.debug("old ncmds: {}".format(arch_macho.ncmds))
arch_macho.ncmds += 1
log.debug("new ncmds: {}".format(arch_macho.ncmds))
log.debug("old sizeofcmds: {}".format(arch_macho.sizeofcmds))
arch_macho.sizeofcmds += cmd.cmdsize
log.debug("new sizeofcmds: {}".format(arch_macho.sizeofcmds))
arch_macho.commands.append(cmd)
hashes = []
if codesig_data_length > 0:
# Patch __LINKEDIT
for lc in arch_macho.commands:
if lc.cmd == 'LC_SEGMENT_64' or lc.cmd == 'LC_SEGMENT':
if lc.data.segname == '__LINKEDIT':
log.debug(
"found __LINKEDIT, old filesize {}, vmsize {}".format(
lc.data.filesize, lc.data.vmsize))
lc.data.filesize = utils.round_up(lc.data.filesize,
16) + codesig_length
if (lc.data.filesize > lc.data.vmsize):
lc.data.vmsize = utils.round_up(lc.data.filesize, 4096)
if lc.cmd == 'LC_SEGMENT_64':
lc.bytes = macho.Segment64.build(lc.data)
else:
lc.bytes = macho.Segment.build(lc.data)
log.debug("new filesize {}, vmsize {}".format(
lc.data.filesize, lc.data.vmsize))
actual_data = macho.MachO.build(arch_macho)
log.debug("actual_data length with codesig LC {}".format(
len(actual_data)))
# Now seek to the start of the actual data and read until the end of the arch.
f.seek(arch_offset + len(actual_data))
bytes_to_read = codesig_offset + arch_offset - f.tell()
file_slice = f.read(bytes_to_read)
if len(file_slice) < bytes_to_read:
log.warn("expected {} bytes but got {}, zero padding.".format(
bytes_to_read, len(file_slice)))
file_slice += ("\x00" * (bytes_to_read - len(file_slice)))
actual_data += file_slice
for i in xrange(nCodeSlots):
actual_data_slice = actual_data[(0x1000 * i):(0x1000 * i + 0x1000)]
actual = hashlib.sha1(actual_data_slice).digest()
log.debug("Slot {} (File page @{}): {}".format(
i, hex(0x1000 * i), actual.encode('hex')))
hashes.append(actual)
else:
hashes = fake_hashes
# Replace placeholder with real one.
codesig_cons = make_basic_codesig(entitlements_file, drs, codeLimit,
hashes, signer, ident)
codesig_data = macho_cs.Blob.build(codesig_cons)
cmd_data = construct.Container(dataoff=codesig_offset,
datasize=len(codesig_data))
cmd = construct.Container(cmd='LC_CODE_SIGNATURE',
cmdsize=16,
data=cmd_data,
bytes=macho.CodeSigRef.build(cmd_data))
arch_macho.commands[-1] = cmd
cmds['LC_CODE_SIGNATURE'] = cmd
return codesig_data
def resblock_up_cond_deep(x_init,
z,
channels_out,
opt,
upscale=True,
use_bias=True,
scope='deep_resblock'):
channels_in = int(x_init.get_shape()[-1])
inner_channels = round_up((channels_in + channels_out) // 6, 8)
with tf.variable_scope(scope):
with tf.variable_scope('bottleneck'):
x = cond_bn(x_init, z, opt=opt)
x = opt["act"](x)
x = conv(x,
inner_channels,
kernel=1,
stride=1,
use_bias=False,
opt=opt)
with tf.variable_scope('upscale'):
x = cond_bn(x, z, opt=opt)
x = opt["act"](x)
if upscale:
x = upconv(x, inner_channels, use_bias=False, opt=opt)
with tf.variable_scope('inner1'):
x = g_conv(x, inner_channels, use_bias=False, opt=opt)
x = cond_bn(x, z, opt=opt)
x = opt["act"](x)
with tf.variable_scope('inner2'):
x = g_conv(x, inner_channels, use_bias=False, opt=opt)
x = bn(x, opt=opt)
x = opt["act"](x)
with tf.variable_scope('proj'):
x = conv(x,
channels_out,
kernel=1,
stride=1,
use_bias=use_bias,
opt=opt)
with tf.variable_scope('skip'):
if channels_in != channels_out:
print(inner_channels, channels_in, channels_out,
channels_in - channels_out)
kept, dropped = tf.split(x_init,
num_or_size_splits=[
channels_out,
channels_in - channels_out
],
axis=-1)
else:
kept = x_init
if upscale:
x_init = upconv(kept, channels_out, use_bias=use_bias, opt=opt)
return x + x_init
def write_register_data(reg, len, data):
writer.write32(0xA2000000 | reg)
writer.write32(((len - 1) << 8) | 0x20)
writer.writeBits(data)
#
# Enumerate config chains
#
for chain_id in range(0, len(chip.configChain)):
chain = chain_with_id(chain_id)
chain_bits = chain['bits']
chain_len = len(chain_bits)
num_padding_bits = round_up(chain_len, 32) - chain_len
write_register_data(0x20 | chain_id, chain_len,
chain_bits + ([0] * num_padding_bits))
#
# Write the bitstream
#
bitstream = []
for tile_row in range(chip.rows - 1, -1, -1):
row_height = chip.max_row_height(tile_row)
for row in range(0, row_height):
row_bits = []
for tile_col in range(chip.columns - 1, -1, -1):
column_width = chip.column_width(tile_col)
def create_container_from_task(self, node, allocated, job, task):
# XXX: This code duplicates the code from YarnScheduler. Needs refactoring.
err_adjusted_duration = duration = task.duration
if task.penalty is not None:
mem_error_adjustment = self.mem_errors[
job.job_id] if job.job_id in self.mem_errors else 0
ib_error_adjustment = self.ib_errors[
job.job_id] if job.job_id in self.ib_errors else 0
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug(
"YARN_ELASTIC_SCHEDULER: MEM_ERR_ADJUST: {}, IB_ERR_ADJUST: {}"
.format(mem_error_adjustment, ib_error_adjustment))
resource = task.ideal_resource
if mem_error_adjustment != 0:
resource = YarnResource(task.ideal_resource.memory_mb,
task.ideal_resource.vcores)
resource.memory_mb += mem_error_adjustment
resource.memory_mb = round_up(resource.memory_mb,
MEM_INCREMENT)
if allocated < resource:
err_adjusted_duration = task.penalty.get_penalized_runtime(
resource,
allocated,
task.duration,
error_offset=ib_error_adjustment)
duration = task.penalty.get_penalized_runtime(
task.ideal_resource, allocated, task.duration)
# Create YarnRunningContainer from YarnPrototypeContainer
yarn_container = YarnRunningContainer(
container_id=job.get_new_container_id(),
duration=err_adjusted_duration,
resource=allocated,
priority=task.priority,
container_type=task.type,
job=task.job,
node=node,
task=task)
yarn_container.duration_error = err_adjusted_duration - duration
if yarn_container.is_elastic:
# Update decision count
self.stats_elastic_decisions_inc(task.job.job_id)
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug(
"YARN_ELASTIC_SCHEDULER: Allocated {} elasticly for {}:{} on node {} "
"with penalized runtime of {} (err adjusted: {}) for a request of {} "
"and a duration of {}".format(allocated.memory_mb,
job.get_name(),
job.next_container_id, node,
duration,
err_adjusted_duration,
task.resource.memory_mb,
task.duration))
else:
if LOG.isEnabledFor(logging.DEBUG):
LOG.debug(
"YARN_ELASTIC_SCHEDULER: Allocated {} regularly for {}:{} on node {} "
"with a duration of: {} (err adjusted: {})".format(
allocated.memory_mb, job.get_name(),
job.next_container_id, node, task.duration,
err_adjusted_duration))
return yarn_container
#######
# getActualIntervals.py: calculate the actual time interval in between each of the events in microseconds
# argv[1]: recordedEvents.txt
#######
import sys
from utils import round_up
recorded = open(sys.argv[1])
lines = recorded.readlines()
times = [] # array to store all of the times
for line in lines:
if line[0] == "[":
timeWithBraketAndSpace = line.split("]")
timeWithBraket = timeWithBraketAndSpace[0].replace(" ", "")
time = timeWithBraket.replace("[", "")
times.append(float(time))
for i, time in enumerate(times):
if i != 0:
print("interval " + str(i) + ": " +
str(round_up((times[i] - times[i - 1]) * 1000000)))
def make_signature(arch_macho, arch_offset, arch_size, cmds, f, entitlements_file, codesig_data_length, signer, ident):
# NB: arch_offset is absolute in terms of file start. Everything else is relative to arch_offset!
# sign from scratch
log.debug("signing from scratch")
drs = None
drs_lc = cmds.get('LC_DYLIB_CODE_SIGN_DRS')
if drs_lc:
drs = drs_lc.data.blob
codesig_offset = utils.round_up(arch_size, 16)
# generate code hashes
log.debug("codesig offset: {}".format(codesig_offset))
codeLimit = codesig_offset
log.debug("new cL: {}".format(hex(codeLimit)))
nCodeSlots = int(math.ceil(float(codesig_offset) / 0x1000))
log.debug("new nCS: {}".format(nCodeSlots))
# generate placeholder LC_CODE_SIGNATURE (like what codesign_allocate does)
fake_hashes = ["\x00" * 20]*nCodeSlots
codesig_cons = make_basic_codesig(entitlements_file,
drs,
codeLimit,
fake_hashes,
signer,
ident)
codesig_data = macho_cs.Blob.build(codesig_cons)
cmd_data = construct.Container(dataoff=codesig_offset,
datasize=codesig_data_length)
cmd = construct.Container(cmd='LC_CODE_SIGNATURE',
cmdsize=16,
data=cmd_data,
bytes=macho.CodeSigRef.build(cmd_data))
log.debug("CS blob before: {}".format(utils.print_structure(codesig_cons, macho_cs.Blob)))
log.debug("len(codesig_data): {}".format(len(codesig_data)))
codesig_length = codesig_data_length
log.debug("codesig length: {}".format(codesig_length))
log.debug("old ncmds: {}".format(arch_macho.ncmds))
arch_macho.ncmds += 1
log.debug("new ncmds: {}".format(arch_macho.ncmds))
log.debug("old sizeofcmds: {}".format(arch_macho.sizeofcmds))
arch_macho.sizeofcmds += cmd.cmdsize
log.debug("new sizeofcmds: {}".format(arch_macho.sizeofcmds))
arch_macho.commands.append(cmd)
hashes = []
if codesig_data_length > 0:
# Patch __LINKEDIT
for lc in arch_macho.commands:
if lc.cmd == 'LC_SEGMENT_64' or lc.cmd == 'LC_SEGMENT':
if lc.data.segname == '__LINKEDIT':
log.debug("found __LINKEDIT, old filesize {}, vmsize {}".format(lc.data.filesize, lc.data.vmsize))
lc.data.filesize = utils.round_up(lc.data.filesize, 16) + codesig_length
if (lc.data.filesize > lc.data.vmsize):
lc.data.vmsize = utils.round_up(lc.data.filesize, 4096)
if lc.cmd == 'LC_SEGMENT_64':
lc.bytes = macho.Segment64.build(lc.data)
else:
lc.bytes = macho.Segment.build(lc.data)
log.debug("new filesize {}, vmsize {}".format(lc.data.filesize, lc.data.vmsize))
actual_data = macho.MachO.build(arch_macho)
log.debug("actual_data length with codesig LC {}".format(len(actual_data)))
# Now seek to the start of the actual data and read until the end of the arch.
f.seek(arch_offset + len(actual_data))
bytes_to_read = codesig_offset + arch_offset - f.tell()
file_slice = f.read(bytes_to_read)
if len(file_slice) < bytes_to_read:
log.warn("expected {} bytes but got {}, zero padding.".format(bytes_to_read, len(file_slice)))
file_slice += ("\x00" * (bytes_to_read - len(file_slice)))
actual_data += file_slice
for i in xrange(nCodeSlots):
actual_data_slice = actual_data[(0x1000 * i):(0x1000 * i + 0x1000)]
actual = hashlib.sha1(actual_data_slice).digest()
log.debug("Slot {} (File page @{}): {}".format(i, hex(0x1000 * i), actual.encode('hex')))
hashes.append(actual)
else:
hashes = fake_hashes
# Replace placeholder with real one.
codesig_cons = make_basic_codesig(entitlements_file,
drs,
codeLimit,
hashes,
signer,
ident)
codesig_data = macho_cs.Blob.build(codesig_cons)
cmd_data = construct.Container(dataoff=codesig_offset,
datasize=len(codesig_data))
cmd = construct.Container(cmd='LC_CODE_SIGNATURE',
cmdsize=16,
data=cmd_data,
bytes=macho.CodeSigRef.build(cmd_data))
arch_macho.commands[-1] = cmd
cmds['LC_CODE_SIGNATURE'] = cmd
return codesig_data
def test_check_a_number_is_halfed_rouded_to_the_next_whole_number(self):
observed = utils.round_up(5, half_it=True)
self.assertEqual(3, observed)
def resblock_down_deep(x_init,
channels_out,
opt,
downscale=True,
use_bias=True,
scope='deep_resblock'):
channels_in = x_init.get_shape()[-1]
inner_channels = round_up((channels_in + channels_out) // 6, 8)
with tf.variable_scope(scope):
with tf.variable_scope('bottleneck'):
x = x_init
if (opt["bn_in_d"]): x = bn(x, opt=opt)
x = opt["act"](x)
x = conv(x,
inner_channels,
kernel=1,
stride=1,
pad=0,
use_bias=use_bias,
opt=opt)
with tf.variable_scope('inner1'):
if (opt["bn_in_d"]): x = bn(x, opt=opt)
x = opt["act"](x)
x = conv(x,
inner_channels,
kernel=3,
stride=1,
pad=1,
use_bias=use_bias,
opt=opt)
with tf.variable_scope('inner2'):
if (opt["bn_in_d"]): x = bn(x, opt=opt)
x = opt["act"](x)
x = conv(x,
inner_channels,
kernel=3,
stride=1,
pad=1,
use_bias=use_bias,
opt=opt)
with tf.variable_scope('downscale'):
x = opt["act"](x)
if downscale:
x = downconv(x,
inner_channels,
use_bias=use_bias,
opt=opt,
method='pool_only')
with tf.variable_scope('proj'):
x = conv(x,
channels_out,
kernel=1,
stride=1,
pad=0,
use_bias=use_bias,
opt=opt)
with tf.variable_scope('skip'):
if downscale:
x_init = downconv(x_init,
channels_in,
use_bias=use_bias,
opt=opt,
method='pool_only')
if channels_in != channels_out:
conv_ch = channels_out - channels_in
dense = conv(x_init,
conv_ch,
kernel=1,
stride=1,
pad=0,
use_bias=use_bias,
opt=opt)
x_init = tf.concat([x_init, dense], axis=-1)
return x + x_init
def test_check_a_number_and_a_half_is_rouded_to_the_next_whole_number(
self):
observed = utils.round_up(6.5)
self.assertEqual(7, observed)
# Enumerate config chains
#
for chain_id in range(0, len(chip.configChain)):
chain = chain_with_id(chain_id)
chain_bits = chain['bits']
chain_len = len(chain_bits)
exp_len = len(chip.configChain[chain_id].empty_bits())
if chip.device_id == 0x01500010 and exp_len == chain_len:
chain_bits = [
chain_bits[int(x / 4)] for x in range(0,
len(chain_bits) * 4)
]
chain_len = len(chain_bits)
num_padding_bits = round_up(chain_len, 32) - chain_len
write_register_data(0x20 | chain_id, chain_len,
chain_bits + ([0] * num_padding_bits))
#
# Write the bitstream
#
bitstream = []
for tile_row in range(chip.rows - 1, -1, -1):
row_height = chip.bitstream_height_for_row(tile_row)
for row in range(0, row_height):
row_bits = []
for tile_col in range(chip.columns - 1, -1, -1):
column_width = chip.bitstream_width_for_column(tile_col)
return sd
# read in recordedEvent and calculate the total time of the events recorded
recorded = open(sys.argv[1])
lines = recorded.readlines()
times = [] # array to store all of the times
for line in lines:
if line[0] == "[":
timeWithBraketAndSpace = line.split("]")
timeWithBraket = timeWithBraketAndSpace[0].replace(" ", "")
time = timeWithBraket.replace("[", "")
times.append(float(time))
recordInterval = times[len(times) - 1] - times[0]
print("record time is " + str(round_up(recordInterval)))
# read in all of the replaytime and calculate the mean and standard diviation
allReplay = open(sys.argv[2])
lines = allReplay.readlines()
replayTimes = []
for line in lines:
line = line.replace(" ", "")
time = line.split("real")[0]
replayTimes.append(float(time))
replayMean = getMean(replayTimes)
replaySD = getStandardDiviation(replayTimes)
print("mean: " + str(round_up(replayMean)))
print("standard diviation: " + str(round_up(replaySD)))
print("error: " + str(round_up(replayMean / recordInterval)))
def sign(self, app, signer):
temp = tempfile.NamedTemporaryFile('wb', delete=False)
# If signing fat binary from scratch, need special handling
# TODO: we assume that if any slice is unsigned, all slices are. This should be true in practice but
# we should still guard against this.
if self.sign_from_scratch and 'FatArch' in self.m.data:
assert len(self.arches) >= 2
# todo(markwang): Update fat headers and mach_start for each slice if needewd
log.debug('signing fat binary from scratch')
sorted_archs = sorted(self.arches, key=lambda arch: arch['arch_offset'])
prev_arch_end = 0
for arch in sorted_archs:
fatentry = arch['macho'] # has pointert to container
codesig_arch_offset, new_codesig_data = self._sign_arch(arch, app, signer)
codesig_file_offset = arch['arch_offset'] + codesig_arch_offset
log.debug('existing arch slice: cputype {}, cpusubtype {}, offset {}, size {}'
.format(fatentry.cputype, fatentry.cpusubtype, arch['arch_offset'], arch['arch_size']))
log.debug("codesig arch offset: {2}, file offset: {0}, len: {1}"
.format(codesig_file_offset, len(new_codesig_data), codesig_arch_offset))
assert codesig_file_offset >= (arch['arch_offset'] + arch['arch_size'])
# Store the old slice offset/sizes because we need them when we copy the data slices from self.f to temp
arch['old_arch_offset'] = arch['arch_offset']
arch['old_arch_size'] = arch['arch_size']
arch['codesig_arch_offset'] = codesig_arch_offset
arch['codesig_data'] = new_codesig_data
new_arch_size = codesig_arch_offset + len(new_codesig_data)
if prev_arch_end > arch['arch_offset']:
arch['arch_offset'] = utils.round_up(prev_arch_end, 16384)
prev_arch_end = arch['arch_offset'] + new_arch_size
arch['arch_size'] = new_arch_size
log.debug('new arch slice after codesig: offset {}, size {}'.format(arch['arch_offset'],
arch['arch_size']))
# write slices and code signatures in reverse order
for arch in reversed(sorted_archs):
self.f.seek(arch['old_arch_offset'])
temp.seek(arch['arch_offset'])
temp.write(self.f.read(arch['old_arch_size']))
temp.seek(arch['arch_offset'] + arch['codesig_arch_offset'])
temp.write(arch['codesig_data'])
fatarch_info = self.m.data.FatArch[arch['fat_index']]
fatarch_info.size = arch['arch_size']
fatarch_info.offset = arch['arch_offset']
else:
# copy self.f into temp, reset to beginning of file
self.f.seek(0)
temp.write(self.f.read())
temp.seek(0)
# write new codesign blocks for each arch
offset_fmt = ("offset: {2}, write offset: {0}, "
"new_codesig_data len: {1}")
for arch in self.arches:
offset, new_codesig_data = self._sign_arch(arch, app, signer)
write_offset = arch['macho'].macho_start + offset
log.debug(offset_fmt.format(write_offset,
len(new_codesig_data),
offset))
temp.seek(write_offset)
temp.write(new_codesig_data)
# write new headers
temp.seek(0)
macho.MachoFile.build_stream(self.m, temp)
temp.close()
# make copy have same permissions
mode = os.stat(self.path).st_mode
os.chmod(temp.name, mode)
# log.debug("moving temporary file to {0}".format(self.path))
shutil.move(temp.name, self.path)
def postProcess(etdata, pred, pred_mask, events = [1, 2, 3], dev = False, **kwargs):
status = pred_mask
fs = etdata.fs
#prepare array for storing event data
events_pp = np.zeros((len(etdata.data), len(events)+1))
events_pp[pred_mask, 1:] = pred
#filter raw probabilities
events_pp = gaussian_filter1d(events_pp, 1, axis=0)
events_pp[~pred_mask, 0] = 1
# #1. mark short interpolation sequences as valid,
# #i.e. remove short interpolation (or other "invalid data") events
# thres_id_s = round_up_to_odd(kwargs['thres_id']*fs/1000.+1)
# status_aggr = np.array(aggr_events(pred_mask))
# events_interp = status_aggr[status_aggr[:,-1]==False]
# md = events_interp[:,1] - events_interp[:,0]
# mask_rem_interp = md<thres_id_s
# ind_rem_interp=[i for s, e in events_interp[mask_rem_interp, :2]
# for i in range(s, e)]
# status[ind_rem_interp] = True
#
# ind_leave_interp=[i for s, e in events_interp[~mask_rem_interp, :2]
# for i in range(s, e)]
# events_pp[ind_leave_interp, 0]=1
# events_pp[ind_leave_interp, 1:]=-1
#2. merge fixations; can be implemented as hpp
thres_ifa=kwargs['thres_ifa']
thres_ifi=kwargs['thres_ifi']
thres_ifi_s = round_up(thres_ifi*fs/1000.)
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
_events_fix = _events_aggr[_events_aggr[:,-1]==1]
_fix_pos = calc_fixPos(etdata, _events_fix)
#inter-fixation amplitudes
ifa = np.pad(np.hypot(_fix_pos[:-1, 1]-_fix_pos[1:, 0],
_fix_pos[:-1, 3]-_fix_pos[1:, 2]),
(0,1), 'constant', constant_values=thres_ifa+1e-5)
#inter-fixation intervals
ifi = np.pad(_events_fix[1:,0]-_events_fix[:-1,1],
(0,1), 'constant', constant_values=thres_ifi_s+1)
mask_merge_fix=(ifa<thres_ifa) & (ifi<thres_ifi_s)
ind_merge_fix=[i for s, e in zip(_events_fix[mask_merge_fix,1],
_events_fix[np.roll(mask_merge_fix, 1),0])
for i in range(s, e)]
events_pp[ind_merge_fix, 0]=-1
events_pp[ind_merge_fix, 1]=1
events_pp[ind_merge_fix, 2:]=-1
#3.1 expand saccades; can be implemented as hpp
thres_sd_s=kwargs['thres_sd_s'] #make saccades to be at least 3 samples
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
_events_sacc = _events_aggr[_events_aggr[:,-1]==2]
_sd = _events_sacc[:,1]-_events_sacc[:,0]
mask_expand_sacc = _sd < thres_sd_s
ind_mid_sacc=(_events_sacc[mask_expand_sacc][:,1]-_events_sacc[mask_expand_sacc][:,0])/2+_events_sacc[mask_expand_sacc][:,0]
ind_rem_fix=[i for s, e in zip(ind_mid_sacc-(thres_sd_s/2+thres_sd_s%2), ind_mid_sacc+(thres_sd_s/2)) for i in range(s, e)]
events_pp[ind_rem_fix, :2]=-1
events_pp[ind_rem_fix, 3]=-1
events_pp[ind_rem_fix, 2]=1
#3.2 merge nearby saccades; can be implemented as hpp
thres_isi = kwargs['thres_isi']
thres_isi_s = round_up(thres_isi*fs/1000.)
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
_events_sacc = _events_aggr[_events_aggr[:,-1]==2]
#inter-saccade intervals
isi = np.pad(_events_sacc[1:,0]-_events_sacc[:-1,1],
(0,1), 'constant', constant_values=thres_isi_s+1)
mask_merge_sacc=isi<thres_isi_s
ind_merge_sacc=[i for s, e in zip(_events_sacc[mask_merge_sacc,1],
_events_sacc[np.roll(mask_merge_sacc, 1),0])
for i in range(s, e)]
events_pp[ind_merge_sacc, 2]=1
events_pp[ind_merge_sacc, :2]=-1
#3.3. remove too short or too long saccades.
#+ for too long
#- for too short; give a chance to become fixation samples
#leave too long for hpp
thres_sd_lo=kwargs['thres_sd_lo']
# thres_sd_hi=kwargs['thres_sd_hi']
thres_sd_lo_s = round_up(thres_sd_lo*fs/1000.)
# thres_sd_hi_s = round_up(thres_sd_hi*fs/1000.)
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
_events_sacc = _events_aggr[_events_aggr[:,-1]==2]
fd = _events_sacc[:,1]-_events_sacc[:,0]
mask_rem_sacc=(fd<thres_sd_lo_s) # | (fd>thres_sd_hi_s)
ind_rem_sacc=[i for s, e in _events_sacc[mask_rem_sacc, :2]
for i in range(s, e)]
events_pp[ind_rem_sacc, 2:]=-1
#4. remove unreasonable PSOs; give a chance to become other classes
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
mask_pso = _events_aggr[:,-1]==3
#remove too short PSOs; not used
# thres_pd = kwargs['thres_pd']
# thres_pd_s = round_up(thres_pd*fs/1000.)
# pso_dur = _events_aggr[:,1]-_events_aggr[:,0]
# mask_pso_dur = pso_dur < thres_pd_s
#remove PSOs not after saccade
seq = ''.join(map(str, _events_aggr[:,-1]))
mask_pso_after_sacc = np.ones_like(mask_pso)
pso_after_sacc = [_m.start()+1 for _m in re.finditer('(?=23)', seq) ]
mask_pso_after_sacc[pso_after_sacc]=False
#mask_inv_pso = mask_pso & mask_pso_dur & mask_pso_after_sacc
mask_inv_pso = mask_pso & mask_pso_after_sacc
ind_inv_pso=[i for s, e in _events_aggr[mask_inv_pso, :2]
for i in range(s, e)]
events_pp[ind_inv_pso, 2:]=-1 #can't be neither pso, neither saccade
#5. remove too short fixations; +
#leave for hpp
# thres_fd=kwargs['thres_fd']
# thres_fd_s = round_up(thres_fd*fs/1000.)
# _events = np.argmax(np.around(events_pp, 3), axis=1)
# _events_aggr = np.array(aggr_events(_events))
# _events_fix = _events_aggr[_events_aggr[:,-1]==1]
# fd = _events_fix[:,1]-_events_fix[:,0]
# mask_rem_fix=fd<thres_fd_s
# ind_rem_fix=[i for s, e in _events_fix[mask_rem_fix, :2]
# for i in range(s, e)]
# events_pp[ind_rem_fix, 0]=1
# events_pp[ind_rem_fix, 1:]=-1
'''legacy code
#6.1 blink detection: remove saccade-like events between missing data
#leave for hpp
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
seq = ''.join(map(str, _events_aggr[:,-1]))
patterns = ['20', '02'] # !!! only works for patterns ['20', '02'] !!!
_blinks = [_m.start() for _pattern in patterns
for _m in re.finditer('(?=%s)'%_pattern, seq) ]
if len(_blinks):
_blinks = np.array(_blinks)
_blinks = np.unique(np.concatenate([_blinks, _blinks+1]))
ind_blink=[i for s, e in _events_aggr[_blinks, :2]
for i in range(s, e)]
events_pp[ind_blink, 0]=1
events_pp[ind_blink, 1:]=-1
#6.2 remove PSOs again, because some of saccades might been removed; +
#leave for hpp
_events = np.argmax(np.around(events_pp, 3), axis=1)
_events_aggr = np.array(aggr_events(_events))
mask_pso = _events_aggr[:,-1]==3
seq = ''.join(map(str, _events_aggr[:,-1]))
mask_pso_after_sacc = np.ones_like(mask_pso)
pso_after_sacc = [_m.start()+1 for _m in re.finditer('(?=23)', seq) ]
mask_pso_after_sacc[pso_after_sacc]=False
mask_inv_pso = mask_pso & mask_pso_after_sacc
ind_inv_pso=[i for s, e in _events_aggr[mask_inv_pso, :2]
for i in range(s, e)]
events_pp[ind_inv_pso, 1:]=-1 #remove event completely
'''
#7. Final events
events = np.argmax(np.around(events_pp, 3), axis=1)
status = ~(events==0)
return events, status, events_pp
