def SyncDn(self, base_dn=None, ldap_filter=None, skip_usn_changed=None):
"""Queries a base DN using an LDAP filter and escrows results to CauliflowerVest.
Args:
base_dn: str, base Distinguished Name to load to CauliflowerVest.
ldap_filter: str, LDAP filter.
skip_usn_changed: int, uSNChanged number to skip loading.
Returns:
Int. The highest uSNChanged number of all loaded hosts.
"""
top_usn_changed = None
for result in self._QueryLdap(base_dn, ldap_filter):
current_usn_changed = int(result['uSNChanged'][0])
# If the current host was loaded previously, skip it.
if current_usn_changed == skip_usn_changed:
continue
try:
self._ProcessHost(result)
except InvalidDistinguishedName as e:
logging.debug('Skipping object with invalid DN: %r', str(e))
continue
except InvalidGuid as e:
logging.debug(
'Skipping object with invalid GUID %r', str(e))
continue
top_usn_changed = max(top_usn_changed, current_usn_changed)
self._DisconnectFromAd()
return top_usn_changed
def _ConnectToAd(self):
"""Establish a connection to the Active Directory server."""
# To enable verbose debug logging, uncomment the following line.
# ldap.set_option(ldap.OPT_DEBUG_LEVEL, 255)
ldap.set_option(ldap.OPT_REFERRALS, 0)
ldap.set_option(ldap.OPT_X_TLS_ALLOW, 1)
ldap.set_option(ldap.OPT_X_TLS_REQUIRE_CERT, ldap.OPT_X_TLS_DEMAND)
logging.debug('Connecting to Active Directory: %s', self.ldap_url)
failures = 0
while True:
try:
self.conn = ldap.initialize(self.ldap_url)
self.conn.protocol_version = ldap.VERSION3
self.conn.simple_bind_s(self.auth_user, self.auth_password)
break
except ldap.LDAPError, e:
failures += 1
if failures == MAX_CONNECTION_FAILURES:
raise
logging.exception('LDAPError in ConnectToAd().')
if e.args and e.args[0].get('desc') == 'Can\'t contact LDAP server':
time.sleep(5 * failures)
continue
raise
def testNamingFieldRejectsBad(self):
bads = (
'corp_internal',
'CORP+INTERNAL',
)
for bad in bads:
logging.debug('Testing bad "%s".', bad)
self.assertRaises(ValueError, policy_simple.NamingField, bad)
def _Comment(self, addr, exclude=False, line_length=132):
rval = []
if self.noverbose:
return rval
# indentation, for multi-line comments, ensures that subsquent lines
# are correctly alligned with the first line of the comment.
indentation = 0
if exclude:
# len('1.1.1.1/32 except;') == 21
indentation = 21 + self._DEFAULT_INDENT
else:
# len('1.1.1.1/32;') == 14
indentation = 14 + self._DEFAULT_INDENT
# length_eol is the width of the line; b/c of the addition of the space
# and the /* characters, it needs to be a little less than the actual width
# to keep from wrapping
length_eol = 77 - indentation
if isinstance(addr, (nacaddr.IPv4, nacaddr.IPv6)):
if addr.text:
if line_length == 0:
# line_length of 0 means that we don't want to truncate the comment.
line_length = len(addr.text)
# There should never be a /* or */, but be safe and ignore those
# comments
if addr.text.find('/*') >= 0 or addr.text.find('*/') >= 0:
logging.debug('Malformed comment [%s] ignoring', addr.text)
else:
text = addr.text[:line_length]
comment = ' /*'
while text:
# split the line
if len(text) > length_eol:
new_length_eol = text[:length_eol].rfind(' ')
if new_length_eol <= 0:
new_length_eol = length_eol
else:
new_length_eol = length_eol
# what line am I gunna output?
line = comment + ' ' + text[:new_length_eol].strip()
# truncate what's left
text = text[new_length_eol:]
# setup the comment and indentation for the next go-round
comment = ' ' * indentation + '**'
rval.append(line)
rval[-1] += ' */'
else:
# should we be paying attention to any other addr type?
logging.debug('Ignoring non IPv4 or IPv6 address: %s', addr)
return rval
def __eq__(self, o):
if type(self) != type(o):
return False
if len(self.fields) != len(o.fields):
return False
for mine, theirs in zip(self.fields, o.fields):
logging.debug('testing "%s" vs "%s"', mine, theirs)
if mine != theirs:
return False
return True
def testNamingFieldAcceptsGood(self):
goods = (
'CORP_INTERNAL',
'RFC1918',
'FOO_BAR102.BAZ101',
)
for good in goods:
try:
logging.debug('Testing good "%s".', good)
_ = policy_simple.NamingField(good)
except ValueError:
self.fail('Rejected good NamingField value "%s".' % good)
def Parse(self):
"""Do the needful."""
self.policy = Policy(self.identifier)
for line in self.data.split('\n'):
line = line.strip()
logging.debug('Processing line: "%s"', line)
if self.block_in_progress:
self.ParseInBlock(line)
else:
self.ParseTopLevel(line)
if self.block_in_progress:
raise ValueError('Unexpected EOF reading "%s"', self.block_in_progress)
return self.policy
def RenderACL(acl_text, acl_suffix, output_directory, input_file, write_files,
binary=False):
"""Write the ACL string out to file if appropriate.
Args:
acl_text: Rendered output of an ACL Generator.
acl_suffix: File suffix to append to output filename.
output_directory: The directory to write the output file.
input_file: The name of the policy file that was used to render ACL.
write_files: A list of file tuples, (output_file, acl_text), to write.
binary: Boolean if the rendered ACL is in binary format.
"""
output_file = os.path.join(output_directory, '%s%s') % (
os.path.splitext(os.path.basename(input_file))[0], acl_suffix)
if FilesUpdated(output_file, acl_text, binary):
logging.info('file changed: %s', output_file)
write_files.append((output_file, acl_text))
else:
logging.debug('file not changed: %s', output_file)
def testHttpLib2(self):
logging.debug('httplib2 loaded from %s' % str(httplib2))
def main(unused_argv):
if FLAGS.verbose:
logging.set_verbosity(logging.INFO)
if FLAGS.debug:
logging.set_verbosity(logging.DEBUG)
logging.debug('binary: %s\noptimize: %d\nbase_directory: %s\n'
'policy_file: %s\nrendered_acl_directory: %s',
str(sys.argv[0]),
int(FLAGS.optimize),
str(FLAGS.base_directory),
str(FLAGS.policy_file),
str(FLAGS.output_directory))
definitions = None
try:
definitions = naming.Naming(FLAGS.definitions_directory)
except naming.NoDefinitionsError:
err_msg = 'bad definitions directory: %s' % FLAGS.definitions_directory
logging.fatal(err_msg)
# thead-safe list for storing files to write
manager = multiprocessing.Manager()
write_files = manager.list()
with_errors = False
if FLAGS.policy_file:
# render just one file
logging.info('rendering one file')
RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions,
FLAGS.exp_info, write_files)
else:
# render all files in parallel
logging.info('finding policies...')
pols = []
pols.extend(DescendRecursively(FLAGS.base_directory, FLAGS.output_directory,
definitions))
pool = multiprocessing.Pool(processes=FLAGS.max_renderers)
results = []
for x in pols:
results.append(pool.apply_async(RenderFile,
args=(x.get('in_file'),
x.get('out_dir'),
definitions,
FLAGS.exp_info,
write_files)))
pool.close()
pool.join()
for result in results:
try:
result.get()
except (ACLParserError, ACLGeneratorError) as e:
with_errors = True
logging.warn('\n\nerror encountered in rendering process:\n%s\n\n', e)
# actually write files to disk
WriteFiles(write_files)
if with_errors:
logging.warn('done, with errors.')
sys.exit(1)
else:
logging.info('done.')
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
ret_str = []
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or
(self.af == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.af))
return ''
# Term verbatim output - this will skip over most normal term
# creation code by returning early. Warnings provided in policy.py
if self.term.verbatim:
for next_verbatim in self.term.verbatim:
if next_verbatim.value[0] == self._PLATFORM:
ret_str.append(str(next_verbatim.value[1]))
return '\n'.join(ret_str)
# Create a new term
self._SetDefaultAction()
if self._TERM_FORMAT:
ret_str.append(self._TERM_FORMAT.substitute(term=self.term_name))
if self._PREJUMP_FORMAT:
ret_str.append(self._PREJUMP_FORMAT.substitute(filter=self.filter,
term=self.term_name))
if self.verbose:
if self.term.owner:
self.term.comment.append('Owner: %s' % self.term.owner)
# reformat long comments, if needed
#
# iptables allows individual comments up to 256 chars.
# But our generator will limit a single comment line to < 120, using:
# max = 119 - 27 (static chars in comment command) - [length of term name]
comment_max_width = 92 - len(self.term_name)
if comment_max_width < 40:
comment_max_width = 40
comments = aclgenerator.WrapWords(self.term.comment, comment_max_width)
# append comments to output
if comments and comments[0]:
for line in comments:
if not line:
continue # iptables-restore does not like 0-length comments.
# term comments
# Strip out quotes as iptables cant have nested quotes
ret_str.append(self._COMMENT_FORMAT.substitute(
filter=self.filter,
term=self.term_name,
comment=str(line).replace('\"', '')))
# Unsupported configuration; in the case of 'accept' or 'next', we
# skip the rule. In other cases, we blow up (raise an exception)
# to ensure that this is not considered valid configuration.
if self.term.source_prefix or self.term.destination_prefix:
if str(self.term.action[0]) not in set(['accept', 'next']):
raise UnsupportedFilterError('%s %s %s %s %s %s %s %s' % (
'\nTerm', self.term.name, 'has action', str(self.term.action[0]),
'with source_prefix or destination_prefix,',
' which is unsupported in', self._PLATFORM, 'iptables output.'))
return ('# skipped %s due to source or destination prefix rule' %
self.term.name)
# protocol
if self.term.protocol:
protocol = self.term.protocol
else:
protocol = ['all']
(term_saddr, exclude_saddr,
term_daddr, exclude_daddr) = self._CalculateAddresses(
self.term.source_address, self.term.source_address_exclude,
self.term.destination_address, self.term.destination_address_exclude)
if not term_saddr:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
if not term_daddr:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
# ports
source_port = []
destination_port = []
if self.term.source_port:
source_port = self.term.source_port
if self.term.destination_port:
destination_port = self.term.destination_port
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, protocol,
self.af)
source_interface = ''
if self.term.source_interface:
source_interface = self.term.source_interface
destination_interface = ''
if self.term.destination_interface:
destination_interface = self.term.destination_interface
log_hits = False
if self.term.logging:
# Iptables sends logs to hosts configured syslog
log_hits = True
# options
tcp_flags = []
tcp_track_options = []
for next_opt in [str(x) for x in self.term.option]:
#
# Sanity checking and high-ports are added as appropriate in
# pre-processing that is done in __str__ within class Iptables.
# Option established will add destination port high-ports if protocol
# contains only tcp, udp or both. This is done earlier in class Iptables.
#
if ((next_opt.find('established') == 0 or
next_opt.find('tcp-established') == 0)
and 'ESTABLISHED' not in [x.strip() for x in self.options]):
if next_opt.find('tcp-established') == 0 and protocol != ['tcp']:
raise TcpEstablishedError('%s %s %s' % (
'\noption tcp-established can only be applied for proto tcp.',
'\nError in term:', self.term.name))
if self.trackstate:
# Use nf_conntrack to track state -- works with any proto
self.options.append('-m state --state ESTABLISHED,RELATED')
elif protocol == ['tcp']:
# Simple established-only rule for TCP: Must have ACK field
# (SYN/ACK or subsequent ACK), or RST and no other flags.
tcp_track_options = [(['ACK'], ['ACK']),
(['SYN', 'FIN', 'ACK', 'RST'], ['RST'])]
# Iterate through flags table, and create list of tcp-flags to append
for next_flag in self._TCP_FLAGS_TABLE:
if next_opt.find(next_flag) == 0:
tcp_flags.append(self._TCP_FLAGS_TABLE.get(next_flag))
if next_opt in self._KNOWN_OPTIONS_MATCHERS:
self.options.append(self._KNOWN_OPTIONS_MATCHERS[next_opt])
if self.term.packet_length:
# Policy format is "#-#", but iptables format is "#:#"
self.options.append('-m length --length %s' %
self.term.packet_length.replace('-', ':'))
if self.term.fragment_offset:
self.options.append('-m u32 --u32 4&0x1FFF=%s' %
self.term.fragment_offset.replace('-', ':'))
icmp_code = ['']
if self.term.icmp_code:
icmp_code = self.term.icmp_code
for saddr in exclude_saddr:
ret_str.extend(self._FormatPart(
'', saddr, '', '', '', '', '', '', '', '', '', '', '',
self._ACTION_TABLE.get('next')))
for daddr in exclude_daddr:
ret_str.extend(self._FormatPart(
'', '', '', daddr, '', '', '', '', '', '', '', '', '',
self._ACTION_TABLE.get('next')))
for saddr in term_saddr:
for daddr in term_daddr:
for icmp in icmp_types:
for code in icmp_code:
for proto in protocol:
for tcp_matcher in tcp_track_options or (([], []),):
ret_str.extend(self._FormatPart(
str(proto),
saddr,
source_port,
daddr,
destination_port,
self.options,
tcp_flags,
icmp,
code,
tcp_matcher,
source_interface,
destination_interface,
log_hits,
self._ACTION_TABLE.get(str(self.term.action[0]))
))
if self._POSTJUMP_FORMAT:
ret_str.append(self._POSTJUMP_FORMAT.substitute(filter=self.filter,
term=self.term_name))
return '\n'.join(str(v) for v in ret_str if v is not '')
def recv_controller_task(self):
"""Recv remote train data in sync mode."""
while True:
# recv, data will deserialize with pyarrow default
# recv_data = self.recv_controller_q.recv()
ctr_info, data = self.recv_controller_q.recv_bytes()
recv_data = {
'ctr_info': deserialize(ctr_info),
'data': deserialize(data)
}
cmd = get_msg_info(recv_data, "cmd")
if cmd in ["close"]:
self.close(recv_data)
if cmd in ["create_explorer"]:
config_set = recv_data["data"]
# setup plasma only one times!
self._setup_share_qs_firstly(config_set)
config_set.update({"share_path": self._buf.get_path()})
self.create_explorer(config_set)
# update the buffer live attribute, explorer num as default.
# self._buf.plus_one_live()
continue
if cmd in ["create_evaluator"]:
# evaluator share single plasma.
config_set = recv_data["data"]
if not self.explorer_share_qs["EVAL0"]:
use_pbt, _, _, _ = get_pbt_set(config_set)
if use_pbt:
plasma_size = config_set.get("plasma_size", 100000000)
plasma_path = "/tmp/plasma{}EVAL0".format(os.getpid())
self.explorer_share_qs["EVAL0"] = UniComm(
"ShareByPlasma",
size=plasma_size,
path=plasma_path)
config_set.update({"share_path": self._buf.get_path()})
logging.debug(
"create evaluator with config:{}".format(config_set))
self.create_evaluator(config_set)
# self._buf.plus_one_live()
continue
if cmd in ["increase", "decrease"]:
self.alloc(cmd)
continue
if cmd in ("eval", ): # fixme: merge into explore
test_id = get_msg_info(recv_data, "test_id")
self.send_explorer_q[test_id].put(recv_data)
continue
# last job, distribute weights/model_name from controller
explorer_id = get_msg_info(recv_data, "explorer_id")
if not isinstance(explorer_id, list):
explorer_id = [explorer_id]
_t0 = time.time()
if cmd in ("explore", ): # todo: could mv to first priority.
# here, only handle explore weights
buf_id = self._buf.put(data)
# replace weight with id
recv_data.update({"data": buf_id})
# predict_reply
# e.g, {'ctr_info': {'broker_id': 0, 'explorer_id': 4, 'agent_id': -1,
# 'cmd': 'predict_reply'}, 'data': 0}
for _eid in explorer_id:
if _eid > -1:
self.send_explorer_q[_eid].put(recv_data)
elif _eid > -2: # -1 # whole explorer, contains evaluator!
for qid, send_q in self.send_explorer_q.items():
if isinstance(qid, str) and "test" in qid:
# logging.info("continue test: ", qid, send_q)
continue
send_q.put(recv_data)
else:
raise KeyError("invalid explore id: {}".format(_eid))
self._metric.append(send=time.time() - _t0)
self._metric.report_if_need()
def setUp(self):
logging.debug('======> %s <======', self.id())
def __init__(self,
use_pid=False,
obs_stack=True,
sub_seteps=50,
port=None):
"""
Args:
use_pid (bool): use pid or direct force to control
port: Gazebo port, need to specify when run multiple environment in parallel
obs_stack (bool): Use staked multi step observation if True
sub_seteps (int): take how many simulator substeps during one gym step
"""
super(ICubWalk, self).__init__(port=port)
self._sub_seteps = sub_seteps
self._obs_stack = obs_stack
self._world = gazebo.new_world_from_file(
os.path.join(social_bot.get_world_dir(), "icub.world"))
self._agent = self._world.get_agent('icub')
logging.debug(self._world.info())
self._agent_joints = [
'icub::iCub::l_leg::l_hip_pitch',
'icub::iCub::l_leg::l_hip_roll',
'icub::iCub::l_leg::l_hip_yaw',
'icub::iCub::l_leg::l_knee',
'icub::iCub::l_leg::l_ankle_pitch',
'icub::iCub::l_leg::l_ankle_roll',
'icub::iCub::r_leg::r_hip_pitch',
'icub::iCub::r_leg::r_hip_roll',
'icub::iCub::r_leg::r_hip_yaw',
'icub::iCub::r_leg::r_knee',
'icub::iCub::r_leg::r_ankle_pitch',
'icub::iCub::r_leg::r_ankle_roll',
'icub::iCub::torso_yaw',
'icub::iCub::torso_roll',
'icub::iCub::torso_pitch',
'icub::iCub::l_shoulder_pitch',
'icub::iCub::l_shoulder_roll',
'icub::iCub::l_shoulder_yaw',
'icub::iCub::l_elbow',
'icub::iCub::neck_pitch',
'icub::iCub::neck_roll',
'icub::iCub::r_shoulder_pitch',
'icub::iCub::r_shoulder_roll',
'icub::iCub::r_shoulder_yaw',
'icub::iCub::r_elbow',
]
joint_states = list(
map(lambda s: self._agent.get_joint_state(s), self._agent_joints))
self._joints_limits = list(
map(lambda s: s.get_effort_limits()[0], joint_states))
if use_pid:
for joint_index in range(len(self._agent_joints)):
self._agent.set_pid_controller(
self._agent_joints[joint_index],
'velocity',
p=0.02,
d=0.00001,
max_force=self._joints_limits[joint_index])
self._agent_control_range = 3.0 * np.pi
else:
self._agent_control_range = np.array(self._joints_limits)
logging.info("joints to control: %s" % self._agent_joints)
self.action_space = gym.spaces.Box(low=-1.0,
high=1.0,
shape=[len(self._agent_joints)],
dtype=np.float32)
self.reset()
obs = self._get_observation()
if self._obs_stack:
obs = np.concatenate((obs, obs), axis=0)
self.observation_space = gym.spaces.Box(low=-100,
high=100,
shape=obs.shape,
dtype=np.float32)
def SetAddress(self, address: int) -> None:
logging.debug("Setting address to 0x%x" % address)
self.address = address
def _sample_negative_patches(image,
image_filepath,
abnormalities_masks,
abnormalities_areas,
patch_size,
number_of_patches=10,
min_breast_overlap_threshold=0.75,
max_abnorm_overlap_threshold=0.35,
max_number_of_trials_per_threshold=100):
"""Sample background patches from the image.
The relative area of breast tissue in the patch must be, at least,
`min_breast_overlap_threshold` of the total patch area. This is to prevent
too easy negative examples.
Similarly, the relative area of the abnormal tissue in the patch must be,
at most, `max_abnorm_overlap_threshold`
The relative area of the patch must overlap with the breast tissue with,
at least, `min_breast_overlap_threshold` (relative) pixels.
In addition, it must also overlap with abnormal tissue with, at most,
`max_abnorm_overlap_threshold` (relative) pixels.
Args:
image: Image to patch from.
image_filepath: Only used for logging.
abnormalities_masks: List of binary mask of each abnormality in the image.
abnormalities_areas: List of precomputed area of each abnormality.
patch_size: Size of the patch to extract.
number_of_patches: Number of negative patches to sample from the image.
min_breast_overlap_threshold: Minimum (relative) number of breast pixels in
the patch.
max_abnorm_overlap_threshold: Maximum (relative) number of abnormal pixels
in the patch.
max_number_of_trials_per_threshold: Maximum number of random samples to try
before reducing the `min_breast_overlap_threshold` by 5% and increasing
the `max_abnorm_overlap_threshold` by 5%.
Yields:
The patch cropped from the input image.
"""
cv2 = tfds.core.lazy_imports.cv2
breast_mask = _get_breast_mask(image)
def patch_overlapping_breast_is_feasible(y, x):
"""Return True if the patch contains enough breast pixels."""
breast_in_patch = breast_mask[y:(y + patch_size[0]),
x:(x + patch_size[1])]
return (np.sum(breast_in_patch > 0) /
(patch_size[0] * patch_size[1]) > min_breast_overlap_threshold)
breast_roi = _get_roi_from_mask(breast_mask)
breast_x, breast_y, breast_w, breast_h = cv2.boundingRect(breast_roi)
number_of_yielded_patches = 0
while (min_breast_overlap_threshold > 0.1
and max_abnorm_overlap_threshold < 0.9):
# Determine the region where random samples should be sampled from.
max_h, min_h = max(breast_h,
patch_size[0]), min(breast_h, patch_size[0])
max_w, min_w = max(breast_w,
patch_size[1]), min(breast_w, patch_size[1])
min_y = breast_y - int((1.0 - min_breast_overlap_threshold) * max_h)
min_x = breast_x - int((1.0 - min_breast_overlap_threshold) * max_w)
max_y = breast_y + breast_h - int(min_breast_overlap_threshold * min_h)
max_x = breast_x + breast_w - int(min_breast_overlap_threshold * min_w)
# Ensure that all sampled batches are within the image.
min_y = max(min_y, 0)
min_x = max(min_x, 0)
max_y = max(min(max_y, image.shape[0] - patch_size[0] - 1), min_y)
max_x = max(min(max_x, image.shape[1] - patch_size[1] - 1), min_x)
# Cap the number of trials if the sampling region is too small.
effective_range_size = max_number_of_trials_per_threshold
if (max_y - min_y + 1) * (max_x - min_x + 1) < effective_range_size:
logging.debug(
'The sampling region for negative patches of size %r with '
'min_breast_overlap_threshold=%f contains less possible patches '
'than max_number_of_trials_per_threshold=%d, in mammography %s',
patch_size, min_breast_overlap_threshold,
max_number_of_trials_per_threshold, image_filepath)
effective_range_size = (max_y - min_y + 1) * (max_x - min_x + 1)
for _ in range(effective_range_size):
patch_y = np.random.randint(min_y, max_y + 1)
patch_x = np.random.randint(min_x, max_x + 1)
if (patch_overlapping_breast_is_feasible(patch_y, patch_x)
and not _patch_overlaps_any_abnormality_above_threshold(
patch_y, patch_x, patch_size, abnormalities_masks,
abnormalities_areas, max_abnorm_overlap_threshold)):
number_of_yielded_patches += 1
yield image[patch_y:(patch_y + patch_size[0]),
patch_x:(patch_x + patch_size[1])]
# If we have yielded all requested patches return.
if number_of_yielded_patches >= number_of_patches:
return
# We failed to produce patches with the given overlapping requirements.
# Relaxate the requirements and try again.
min_breast_overlap_threshold = min_breast_overlap_threshold * 0.95
max_abnorm_overlap_threshold = max_abnorm_overlap_threshold * 1.05
logging.debug(
'Overlapping constraints relaxed to min_breast_overlap_threshold=%f '
'and max_abnorm_overlap_threshold=%f while sampling negative '
'patches for the mammography %s', min_breast_overlap_threshold,
max_abnorm_overlap_threshold,
image_filepath) # Filepath to the abnormality mask image.
# This should not happen ever.
raise ValueError(
'Only %d negative patches of size %r could be sampled satisfying the '
'current conditions (min. relative overlapping area with breast = %f, '
'max. relative overlapping area with abnormalities = %f) for the '
'mammography %s' %
(number_of_yielded_patches, patch_size, min_breast_overlap_threshold,
max_abnorm_overlap_threshold, image_filepath))
def close(self):
for p, v in self._target_executors.items():
for e in v:
logging.debug('Closing child executor for placement: %s', p)
e.close()
def get_state(self):
"""Returns the global state.
NOTE: This functon should not be used during decentralised execution.
"""
if self.obs_instead_of_state:
obs_concat = np.concatenate(self.get_obs(),
axis=0).astype(np.float32)
return obs_concat
nf_al = 4 + self.shield_bits_ally + self.unit_type_bits
nf_en = 3 + self.shield_bits_enemy + self.unit_type_bits
ally_state = np.ones((self.n_agents, nf_al)) * -1
enemy_state = np.ones((self.n_enemies, nf_en)) * -1
center_x = self.map_x / 2
center_y = self.map_y / 2
for al_id, al_unit in self.agents.items():
if al_unit.health > 0:
x = al_unit.pos.x
y = al_unit.pos.y
max_cd = self.unit_max_cooldown(al_unit)
ally_state[al_id,
0] = (al_unit.health / al_unit.health_max) # health
if (self.map_type == "MMM"
and al_unit.unit_type == self.medivac_id):
ally_state[al_id, 1] = al_unit.energy / max_cd # energy
else:
ally_state[al_id, 1] = (al_unit.weapon_cooldown / max_cd
) # cooldown
ally_state[al_id,
2] = (x -
center_x) / self.max_distance_x # relative X
ally_state[al_id,
3] = (y -
center_y) / self.max_distance_y # relative Y
ind = 4
if self.shield_bits_ally > 0:
max_shield = self.unit_max_shield(al_unit)
ally_state[al_id,
ind] = (al_unit.shield / max_shield) # shield
ind += 1
if self.unit_type_bits > 0:
type_id = self.get_unit_type_id(al_unit, True)
ally_state[al_id, ind + type_id] = 1
for e_id, e_unit in self.enemies.items():
if e_unit.health > 0:
x = e_unit.pos.x
y = e_unit.pos.y
enemy_state[e_id,
0] = (e_unit.health / e_unit.health_max) # health
enemy_state[e_id,
1] = (x -
center_x) / self.max_distance_x # relative X
enemy_state[e_id,
2] = (y -
center_y) / self.max_distance_y # relative Y
ind = 3
if self.shield_bits_enemy > 0:
max_shield = self.unit_max_shield(e_unit)
enemy_state[e_id,
ind] = (e_unit.shield / max_shield) # shield
ind += 1
if self.unit_type_bits > 0:
type_id = self.get_unit_type_id(e_unit, False)
enemy_state[e_id, ind + type_id] = 1
state = np.append(ally_state.flatten(), enemy_state.flatten())
if self.state_last_action:
state = np.append(state, self.last_action.flatten())
state = state.astype(dtype=np.float32)
if self.debug:
logging.debug("STATE".center(60, "-"))
logging.debug("Ally state {}".format(ally_state))
logging.debug("Enemy state {}".format(enemy_state))
if self.state_last_action:
logging.debug("Last actions {}".format(self.last_action))
return state
def _lazy_init(self):
"""Lazily initialize the underlying gRPC stream."""
if self._is_initialized:
return
logging.debug('Initializing bidi stream')
self._request_queue = queue.Queue()
self._response_event_dict = {}
self._stream_closed_event = threading.Event()
def request_iter():
"""Iterator that blocks on the request Queue."""
for seq in itertools.count():
logging.debug('Request thread: blocking for next request')
val = self._request_queue.get()
if val:
py_typecheck.check_type(val[0],
executor_pb2.ExecuteRequest)
py_typecheck.check_type(val[1], threading.Event)
req = val[0]
req.sequence_number = seq
logging.debug(
'Request thread: processing request of type %s, seq_no %s',
val[0].WhichOneof('request'), seq)
self._response_event_dict[seq] = val[1]
yield val[0]
else:
logging.debug(
'Request thread: Final request received. Stream will close.'
)
# None means we are done processing
return
response_iter = self._stub.Execute(request_iter())
def response_thread_fn():
"""Consumes response iter and exposes the value on corresponding Event."""
try:
logging.debug('Response thread: blocking for next response')
for response in response_iter:
logging.debug(
'Response thread: processing response of type %s, seq_no %s',
response.WhichOneof('response'),
response.sequence_number)
# Get the corresponding response Event
response_event = self._response_event_dict[
response.sequence_number]
# Attach the response as an attribute on the Event
response_event.response = response
response_event.set()
# Set the event indicating the stream has been closed
self._stream_closed_event.set()
except grpc.RpcError as error:
logging.exception('Error calling remote executor: %s', error)
response_thread = threading.Thread(target=response_thread_fn)
response_thread.daemon = True
response_thread.start()
self._is_initialized = True
def get_obs_agent(self, agent_id):
"""Returns observation for agent_id.
NOTE: Agents should have access only to their local observations
during decentralised execution.
"""
unit = self.get_unit_by_id(agent_id)
nf_al = 4 + self.unit_type_bits
nf_en = 4 + self.unit_type_bits
if self.obs_all_health:
nf_al += 1 + self.shield_bits_ally
nf_en += 1 + self.shield_bits_enemy
if self.obs_last_action:
nf_al += self.n_actions
nf_own = self.unit_type_bits
if self.obs_own_health:
nf_own += 1 + self.shield_bits_ally
move_feats_len = self.n_actions_move
if self.obs_pathing_grid:
move_feats_len += self.n_obs_pathing
if self.obs_terrain_height:
move_feats_len += self.n_obs_height
move_feats = np.ones(move_feats_len, dtype=np.float32) * -1
enemy_feats = np.ones((self.n_enemies, nf_en), dtype=np.float32) * -1
ally_feats = np.ones((self.n_agents - 1, nf_al), dtype=np.float32) * -1
own_feats = np.ones(nf_own, dtype=np.float32) * -1
if unit.health > 0: # otherwise dead, return all zeros
x = unit.pos.x
y = unit.pos.y
sight_range = self.unit_sight_range(agent_id)
# Movement features
avail_actions = self.get_avail_agent_actions(agent_id)
for m in range(self.n_actions_move):
move_feats[m] = avail_actions[m + 2]
ind = self.n_actions_move
if self.obs_pathing_grid:
move_feats[ind:ind +
self.n_obs_pathing] = self.get_surrounding_pathing(
unit)
ind += self.n_obs_pathing
if self.obs_terrain_height:
move_feats[ind:] = self.get_surrounding_height(unit)
# Enemy features
for e_id, e_unit in self.enemies.items():
e_x = e_unit.pos.x
e_y = e_unit.pos.y
dist = self.distance(x, y, e_x, e_y)
if (dist < sight_range
and e_unit.health > 0): # visible and alive
# Sight range > shoot range
enemy_feats[e_id,
0] = avail_actions[self.n_actions_no_attack +
e_id] # available
enemy_feats[e_id, 1] = dist / sight_range # distance
enemy_feats[e_id,
2] = (e_x - x) / sight_range # relative X
enemy_feats[e_id,
3] = (e_y - y) / sight_range # relative Y
ind = 4
if self.obs_all_health:
enemy_feats[e_id,
ind] = (e_unit.health / e_unit.health_max
) # health
ind += 1
if self.shield_bits_enemy > 0:
max_shield = self.unit_max_shield(e_unit)
enemy_feats[e_id,
ind] = (e_unit.shield / max_shield
) # shield
ind += 1
if self.unit_type_bits > 0:
type_id = self.get_unit_type_id(e_unit, False)
enemy_feats[e_id, ind + type_id] = 1 # unit type
# Ally features
al_ids = [
al_id for al_id in range(self.n_agents) if al_id != agent_id
]
for i, al_id in enumerate(al_ids):
al_unit = self.get_unit_by_id(al_id)
al_x = al_unit.pos.x
al_y = al_unit.pos.y
dist = self.distance(x, y, al_x, al_y)
if (dist < sight_range
and al_unit.health > 0): # visible and alive
ally_feats[i, 0] = 1 # visible
ally_feats[i, 1] = dist / sight_range # distance
ally_feats[i, 2] = (al_x - x) / sight_range # relative X
ally_feats[i, 3] = (al_y - y) / sight_range # relative Y
ind = 4
if self.obs_all_health:
ally_feats[i,
ind] = (al_unit.health / al_unit.health_max
) # health
ind += 1
if self.shield_bits_ally > 0:
max_shield = self.unit_max_shield(al_unit)
ally_feats[i, ind] = (al_unit.shield / max_shield
) # shield
ind += 1
if self.unit_type_bits > 0:
type_id = self.get_unit_type_id(al_unit, True)
ally_feats[i, ind + type_id] = 1
ind += self.unit_type_bits
if self.obs_last_action:
ally_feats[i, ind:] = self.last_action[al_id]
# Own features
ind = 0
if self.obs_own_health:
own_feats[ind] = unit.health / unit.health_max
ind += 1
if self.shield_bits_ally > 0:
max_shield = self.unit_max_shield(unit)
own_feats[ind] = unit.shield / max_shield
ind += 1
if self.unit_type_bits > 0:
type_id = self.get_unit_type_id(unit, True)
own_feats[ind + type_id] = 1
agent_obs = np.concatenate((
move_feats.flatten(),
enemy_feats.flatten(),
ally_feats.flatten(),
own_feats.flatten(),
))
if self.debug:
logging.debug("Obs Agent: {}".format(agent_id).center(60, "-"))
logging.debug("Avail. actions {}".format(
self.get_avail_agent_actions(agent_id)))
logging.debug("Move feats {}".format(move_feats))
logging.debug("Enemy feats {}".format(enemy_feats))
logging.debug("Ally feats {}".format(ally_feats))
logging.debug("Own feats {}".format(own_feats))
return agent_obs
def get_agent_action(self, a_id, action):
"""Construct the action for agent a_id."""
avail_actions = self.get_avail_agent_actions(a_id)
# modify 去掉legal action的选择
# assert avail_actions[action] == 1, \
# "Agent {} cannot perform action {}".format(a_id, action)
if avail_actions[action] != 1:
return None
unit = self.get_unit_by_id(a_id)
tag = unit.tag
x = unit.pos.x
y = unit.pos.y
if action == 0:
# no-op (valid only when dead)
assert unit.health == 0, "No-op only available for dead agents."
if self.debug:
logging.debug("Agent {}: Dead".format(a_id))
return None
elif action == 1:
# stop
cmd = r_pb.ActionRawUnitCommand(ability_id=actions["stop"],
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {}: Stop".format(a_id))
elif action == 2:
# move north
cmd = r_pb.ActionRawUnitCommand(
ability_id=actions["move"],
target_world_space_pos=sc_common.Point2D(x=x,
y=y +
self._move_amount),
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {}: Move North".format(a_id))
elif action == 3:
# move south
cmd = r_pb.ActionRawUnitCommand(
ability_id=actions["move"],
target_world_space_pos=sc_common.Point2D(x=x,
y=y -
self._move_amount),
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {}: Move South".format(a_id))
elif action == 4:
# move east
cmd = r_pb.ActionRawUnitCommand(
ability_id=actions["move"],
target_world_space_pos=sc_common.Point2D(x=x +
self._move_amount,
y=y),
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {}: Move East".format(a_id))
elif action == 5:
# move west
cmd = r_pb.ActionRawUnitCommand(
ability_id=actions["move"],
target_world_space_pos=sc_common.Point2D(x=x -
self._move_amount,
y=y),
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {}: Move West".format(a_id))
else:
# attack/heal units that are in range
target_id = action - self.n_actions_no_attack
if self.map_type == "MMM" and unit.unit_type == self.medivac_id:
target_unit = self.agents[target_id]
action_name = "heal"
else:
target_unit = self.enemies[target_id]
action_name = "attack"
action_id = actions[action_name]
target_tag = target_unit.tag
cmd = r_pb.ActionRawUnitCommand(ability_id=action_id,
target_unit_tag=target_tag,
unit_tags=[tag],
queue_command=False)
if self.debug:
logging.debug("Agent {} {}s unit # {}".format(
a_id, action_name, target_id))
sc_action = sc_pb.Action(action_raw=r_pb.ActionRaw(unit_command=cmd))
return sc_action
def step(self, actions):
"""A single environment step. Returns reward, terminated, info."""
actions = [int(a) for a in actions]
self.last_action = np.eye(self.n_actions)[np.array(actions)]
# Collect individual actions
sc_actions = []
if self.debug:
logging.debug("Actions".center(60, "-"))
for a_id, action in enumerate(actions):
agent_action = self.get_agent_action(a_id, action)
if agent_action:
sc_actions.append(agent_action)
# Send action request
req_actions = sc_pb.RequestAction(actions=sc_actions)
try:
self._controller.actions(req_actions)
# Make step in SC2, i.e. apply actions
self._controller.step(self._step_mul)
# Observe here so that we know if the episode is over.
self._obs = self._controller.observe()
except (protocol.ProtocolError, protocol.ConnectionError):
self.full_restart()
return 0, True, {}
self._total_steps += 1
self._episode_steps += 1
# Update units
game_end_code = self.update_units()
terminated = False
reward = self.reward_battle()
info = {"battle_won": False}
if game_end_code is not None:
# Battle is over
terminated = True
self.battles_game += 1
if game_end_code == 1:
self.battles_won += 1
info["battle_won"] = True
if not self.reward_sparse:
reward += self.reward_win
else:
reward = 1
elif game_end_code == -1:
if not self.reward_sparse:
reward += self.reward_defeat
else:
reward = -1
elif self._episode_steps >= self.episode_limit:
# Episode limit reached
terminated = True
if self.continuing_episode:
info["episode_limit"] = True
self.battles_game += 1
self.timeouts += 1
if self.debug:
logging.debug("Reward = {}".format(reward).center(60, '-'))
if terminated:
self._episode_count += 1
if self.reward_scale:
reward /= self.max_reward / self.reward_scale_rate
return reward, terminated, info
def RenderFile(input_file, output_directory, definitions,
exp_info, write_files):
"""Render a single file.
Args:
input_file: the name of the input policy file.
output_directory: the directory in which we place the rendered file.
definitions: the definitions from naming.Naming().
exp_info: print a info message when a term is set to expire
in that many weeks.
write_files: a list of file tuples, (output_file, acl_text), to write
"""
logging.debug('rendering file: %s into %s', input_file,
output_directory)
pol = None
jcl = False
acl = False
asacl = False
aacl = False
bacl = False
eacl = False
gca = False
gcefw = False
ips = False
ipt = False
spd = False
nsx = False
pcap_accept = False
pcap_deny = False
pf = False
srx = False
jsl = False
nft = False
win_afw = False
xacl = False
paloalto = False
try:
conf = open(input_file).read()
logging.debug('opened and read %s', input_file)
except IOError as e:
logging.warn('bad file: \n%s', e)
raise
try:
pol = policy.ParsePolicy(
conf, definitions, optimize=FLAGS.optimize,
base_dir=FLAGS.base_directory, shade_check=FLAGS.shade_check)
except policy.ShadingError as e:
logging.warn('shading errors for %s:\n%s', input_file, e)
return
except (policy.Error, naming.Error):
raise ACLParserError('Error parsing policy file %s:\n%s%s' % (
input_file, sys.exc_info()[0], sys.exc_info()[1]))
platforms = set()
for header in pol.headers:
platforms.update(header.platforms)
if 'juniper' in platforms:
jcl = copy.deepcopy(pol)
if 'cisco' in platforms:
acl = copy.deepcopy(pol)
if 'ciscoasa' in platforms:
asacl = copy.deepcopy(pol)
if 'brocade' in platforms:
bacl = copy.deepcopy(pol)
if 'arista' in platforms:
eacl = copy.deepcopy(pol)
if 'aruba' in platforms:
aacl = copy.deepcopy(pol)
if 'ipset' in platforms:
ips = copy.deepcopy(pol)
if 'iptables' in platforms:
ipt = copy.deepcopy(pol)
if 'nsxv' in platforms:
nsx = copy.deepcopy(pol)
if 'packetfilter' in platforms:
pf = copy.deepcopy(pol)
if 'pcap' in platforms:
pcap_accept = copy.deepcopy(pol)
pcap_deny = copy.deepcopy(pol)
if 'speedway' in platforms:
spd = copy.deepcopy(pol)
if 'srx' in platforms:
srx = copy.deepcopy(pol)
if 'srxlo' in platforms:
jsl = copy.deepcopy(pol)
if 'windows_advfirewall' in platforms:
win_afw = copy.deepcopy(pol)
if 'ciscoxr' in platforms:
xacl = copy.deepcopy(pol)
if 'nftables' in platforms:
nft = copy.deepcopy(pol)
if 'gce' in platforms:
gcefw = copy.deepcopy(pol)
if 'paloalto' in platforms:
paloalto = copy.deepcopy(pol)
if 'cloudarmor' in platforms:
gca = copy.deepcopy(pol)
if not output_directory.endswith('/'):
output_directory += '/'
try:
if jcl:
acl_obj = juniper.Juniper(jcl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if srx:
acl_obj = junipersrx.JuniperSRX(srx, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if acl:
acl_obj = cisco.Cisco(acl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if asacl:
acl_obj = ciscoasa.CiscoASA(asacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if aacl:
acl_obj = aruba.Aruba(aacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if bacl:
acl_obj = brocade.Brocade(bacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if eacl:
acl_obj = arista.Arista(eacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if ips:
acl_obj = ipset.Ipset(ips, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if ipt:
acl_obj = iptables.Iptables(ipt, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if nsx:
acl_obj = nsxv.Nsxv(nsx, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if spd:
acl_obj = speedway.Speedway(spd, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pcap_accept:
acl_obj = pcap.PcapFilter(pcap_accept, exp_info)
RenderACL(str(acl_obj), '-accept' + acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pcap_deny:
acl_obj = pcap.PcapFilter(pcap_deny, exp_info, invert=True)
RenderACL(str(acl_obj), '-deny' + acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pf:
acl_obj = packetfilter.PacketFilter(pf, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if win_afw:
acl_obj = windows_advfirewall.WindowsAdvFirewall(win_afw, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if jsl:
acl_obj = srxlo.SRXlo(jsl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if xacl:
acl_obj = ciscoxr.CiscoXR(xacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if nft:
acl_obj = nftables.Nftables(nft, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if gcefw:
acl_obj = gce.GCE(gcefw, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if paloalto:
acl_obj = paloaltofw.PaloAltoFW(paloalto, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if gca:
acl_obj = cloudarmor.CloudArmor(gca, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
# TODO(robankeny) add additional errors.
except (juniper.Error, junipersrx.Error, cisco.Error, ipset.Error,
iptables.Error, speedway.Error, pcap.Error,
aclgenerator.Error, aruba.Error, nftables.Error, gce.Error,
cloudarmor.Error) as e:
raise ACLGeneratorError(
'Error generating target ACL for %s:\n%s' % (input_file, e))
def __str__(self):
output = []
# Don't render term if not in platforms or in excluded platforms
if self.term.platform and self._PLATFORM not in self.term.platform:
return ''
if (self.term.platform_exclude
and self._PLATFORM in self.term.platform_exclude):
return ''
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
# Does not currently support mixed family.
if ((self.af == 'inet6' and 'icmp' in self.term.protocol)
or (self.af == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(
self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.af))
return ''
# Term verbatim output - this will skip over most normal term
# creation code by returning early. Warnings provided in policy.py.
if self.term.verbatim:
for verbatim_line in self.term.verbatim:
platform, contents = verbatim_line.value
if platform == self._PLATFORM:
output.append(str(contents))
return '\n'.join(output)
# Source address
if self.term.source_address or self.term.source_address_exclude:
src_addrs = self._CalculateAddrs(self.term.source_address,
self.term.source_address_exclude)
if not src_addrs:
logging.warn(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
# TODO(castagno): Add support for ipv6
output.append('ip saddr %s' % self._FormatMatch(src_addrs))
# Destination address
if self.term.destination_address or self.term.source_address_exclude:
dst_addrs = self._CalculateAddrs(
self.term.destination_address,
self.term.destination_address_exclude)
if not dst_addrs:
logging.warn(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
# TODO(castagno): Add support for ipv6
output.append('ip daddr %s' % self._FormatMatch(dst_addrs))
# Protocol
#
# nft intepreter shortcuts protocol specification if there are more specific
# matches. At the moment, these are:
# * source port
# * destination port
# * ICMP type
if self.term.protocol and not (self.term.source_port
or self.term.destination_port
or self.term.icmp_type):
output.append('ip protocol %s' %
self._FormatMatch(self.term.protocol))
# Source port
if self.term.source_port:
output.append('%s sport %s' % (self._FormatMatch(
self.term.protocol), self._FormatMatch(self.term.source_port)))
# Destination port
if self.term.destination_port:
output.append('%s dport %s' %
(self._FormatMatch(self.term.protocol),
self._FormatMatch(self.term.destination_port)))
# Icmp type
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol, self.af)
if icmp_types != ['']:
# nft intepreter requires ICMP types to be spelled out
icmp_name_types = self.ICMP_TYPE[self.AF_MAP[self.af]]
icmp_type_names = dict(
(v, k) for k, v in icmp_name_types.iteritems())
output.append('icmp type %s' % self._FormatMatch(
[icmp_type_names[icmp_type] for icmp_type in icmp_types]))
# Counter
# This does not use the value that was passed in the term.
if self.term.counter:
output.append('counter')
# Log
# Setup logic so that only one log statement is printed.
if self.term.logging and not self.term.log_name:
output.append('log')
elif (self.term.logging and self.term.log_name) or self.term.log_name:
# Only supports log prefix's of 128 characters truncate to 126 to support
# the additional suffix that is being added
output.append('log prefix "%s: "' % self.term.log_name[:126])
# Action
output.append(self._ACTIONS[self.term.action[0]])
# Owner (implement as comment)
if self.term.owner:
self.term.comment.append('Owner: %s' % self.term.owner)
# Comment
if self.term.comment:
comment_data = ' '.join(self.term.comment)
# Have to truncate MAX_CHARACTERS characters due to NFTables limitation
if len(comment_data) > self.MAX_CHARACTERS:
# Have to use the first MAX_CHARACTERS characters
comment_data = comment_data[:self.MAX_CHARACTERS]
logging.warn(
'Term %s in policy is too long (>%d characters) '
'and will be truncated', self.term.name,
self.MAX_CHARACTERS)
output.append('comment "%s"' % comment_data)
return ' '.join(output)
def _Worker(self):
"""Threaded graph reader."""
graph_files = list(self.graph_path.iterdir())
logging.debug(
"Enumerated %s graph files to load", humanize.Commas(len(graph_files))
)
graph_count = 0
while not self.min_graph_count or graph_count < self.min_graph_count:
# Strip any graph files that we have earmarked for ignoring.
graph_files = [
f for f in graph_files if f not in self._excluded_graph_files
]
# We may have run out of files.
if not graph_files:
self._Done(graph_count)
return
if self.seed:
# If we are setting a reproducible seed, first sort the list of files
# since iterdir() order is undefined, then seed the RNG for the
# shuffle.
graph_files = sorted(graph_files, key=lambda x: x.name)
# Change the seed so that on the next execution of this loop we will
# chose a different random ordering.
self.seed += 1
random.Random(self.seed).shuffle(graph_files)
for graph_path in graph_files:
# Master thread requested a stop.
if self._stopped:
self._Done(graph_count)
return
stem = graph_path.name[: -len("ProgramGraph.pb")]
name = f"{stem}ProgramGraphFeaturesList.pb"
features_path = self.labels_path / name
# There is no guarantee that we have generated features for this
# program graph, so we check for its existence. As a *very* defensive
# measure, we also check for the existence of the graph file that we
# enumerated at the start of this function. This check can be removed
# later, it is only useful during development when you might be
# modifying the dataset at the same time as having test jobs running.
if not graph_path.is_file() or not features_path.is_file():
self.skip_count += 1
continue
# Read the graph from disk, maybe performing a cheeky wee conversion
# to CDFG format.
logging.debug("Read %s", features_path)
if self.use_cdfg:
graph = cdfg.FromProgramGraphFile(graph_path)
else:
graph = pbutil.FromFile(
graph_path, program_graph_pb2.ProgramGraph()
)
if not graph:
logging.debug("Failed to load graph %s", graph_path)
self._excluded_graph_files.add(graph_path)
continue
# Skip empty graphs.
if not len(graph.node) or len(graph.node) > FLAGS.max_graph_node_count:
logging.debug(
"Graph node count %s is not in range (1,%s]",
len(graph.node),
FLAGS.max_graph_node_count,
)
self._excluded_graph_files.add(graph_path)
continue
# Skip a graph without inst2vec
if self.require_inst2vec and not len(
graph.features.feature["inst2vec_annotated"].int64_list.value
):
logging.debug("Skipping graph without inst2vec annotations")
continue
features_list = pbutil.FromFile(
features_path, program_graph_features_pb2.ProgramGraphFeaturesList()
)
# Iterate over the features list to yield <graph, features> pairs.
skipped_all_features = True
for j, features in enumerate(features_list.graph):
step_count_feature = features.features.feature[
"data_flow_step_count"
].int64_list.value
step_count = step_count_feature[0] if len(step_count_feature) else 0
if self.data_flow_step_max and step_count > self.data_flow_step_max:
self.skip_count += 1
logging.debug(
"Skipped graph with data_flow_step_count %d > %d "
"(skipped %d / %d, %.2f%%)",
step_count,
self.data_flow_step_max,
self.skip_count,
(graph_count + self.skip_count),
(self.skip_count / (graph_count + self.skip_count)) * 100,
)
continue
graph_count += 1
if self.logfile:
self.logfile.write(f"{features_path} {j}\n")
self._outq.put((graph, features), block=True)
skipped_all_features = False
if self.max_graph_count and graph_count >= self.max_graph_count:
logging.debug("Stopping after reading %d graphs", graph_count)
self._Done(graph_count)
return
if skipped_all_features:
self._excluded_graph_files.add(graph_path)
self._Done(graph_count)
def test_english(self):
""" test NLU joint task of chiniese data, split sentences by space"""
config = utils.load_config(self.config_file)
max_len = config["model"]["net"]["structure"]["max_len"]
batch_size = config["data"]["task"]["batch_size"]
data_config = config["data"]
task_config = data_config["task"]
task_config["language"] = "english"
task_config["split_by_space"] = False
task_config["use_word"] = True
task_config[
"text_vocab"] = "egs/mock_text_nlu_joint_data/nlu-joint/v1/data/text_vocab.txt"
task_config["need_shuffle"] = False
# generate_mock_files(config)
task = TextNLUJointTask(config, utils.TRAIN)
# test offline data
data = task.dataset()
self.assertTrue("input_x_dict" in data
and "input_x" in data["input_x_dict"])
self.assertTrue("input_y_dict" in data
and "input_y" in data["input_y_dict"])
input_intent_y, input_slots_y = data["input_y_dict"]["input_y"]
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
sess.run(data["iterator"].initializer)
res = sess.run([
data["input_x_dict"]["input_x"], data["input_x_len"],
input_intent_y, input_slots_y
])
logging.debug(res[0][0][:5])
logging.debug(res[1][0])
logging.debug(res[2])
logging.debug(res[3])
self.assertAllEqual(res[0][0][:5], [5, 6, 8, 9, 0])
self.assertEqual(np.shape(res[0]), (batch_size, max_len))
self.assertEqual(np.shape(res[1]), (batch_size, ))
self.assertEqual(np.shape(res[2]), (batch_size, 2))
self.assertEqual(np.shape(res[3]), (batch_size, max_len))
# test online data
export_inputs = task.export_inputs()
self.assertTrue("export_inputs" in export_inputs
and "input_sentence" in export_inputs["export_inputs"])
input_sentence = export_inputs["export_inputs"]["input_sentence"]
input_x = export_inputs["model_inputs"]["input_x"]
with self.cached_session(use_gpu=False, force_gpu=False) as sess:
sess.run(data["iterator"].initializer)
res = sess.run(input_x, feed_dict={input_sentence: ["i am happy"]})
logging.debug(res[0][:5])
logging.debug(np.shape(res[0]))
self.assertAllEqual(res[0][:5], [2, 3, 7, 0, 0])
self.assertEqual(np.shape(res[0]), (max_len, ))
def main(_):
logging.set_verbosity(logging.DEBUG if FLAGS.debug else logging.INFO)
params = Config(FLAGS.config_path).params
trainer = Trainer(
run_mode='export', # noqa: F841
strategy=tf.distribute.OneDeviceStrategy(
device='/cpu:0'), # noqa: E501
model_fn=model_builder(params),
train_input_fn=None,
val_input_fn=None,
style_loss_weight=params.training.style_loss_weight,
content_loss_weight=params.training.content_loss_weight,
train_steps=params.training.train_steps,
val_steps=params.training.validation_steps,
val_freq=params.training.validation_freq,
steps_per_execution=params.training.steps_per_execution,
batch_size=params.training.batch_size,
model_dir=params.experiment.model_dir,
save_every=params.training.save_every,
restore_checkpoint=params.training.restore_checkpoint,
summary_dir=params.experiment.tensorboard_dir,
name=params.experiment.name)
inference_model = trainer.model
inference_model.build([(1, None, None, 3), (1, None, None, 3), ()])
inference_model.optimizer = None
inference_model.compiled_loss = None
inference_model.compiled_metrics = None
inference_model._metrics = []
input_signature = {
'style_images':
tf.TensorSpec(shape=[1, None, None, 3],
name='style_images',
dtype=tf.float32),
'content_images':
tf.TensorSpec(shape=[1, None, None, 3],
name='content_images',
dtype=tf.float32),
'alpha':
tf.TensorSpec(shape=[], name='alpha', dtype=tf.float32),
'resize':
tf.TensorSpec(shape=[], name='resize', dtype=tf.bool)
}
logging.debug(
'Signature for serving_default :\n{}'.format(input_signature))
inference_model._saved_model_inputs_spec = input_signature
preprocessing_pipeline = PreprocessingPipeline(params=params,
is_validation_dataset=True)
@tf.function
def serving_fn(sample):
style_images = sample['style_images']
content_images = sample['content_images']
if sample['resize']:
style_images = preprocessing_pipeline.inference_pipeline(
images=style_images)
content_images = preprocessing_pipeline.inference_pipeline(
images=content_images)
stylized_images = inference_model.call(
(style_images, content_images, sample['alpha']),
training=False)['synthesized_images']
stylized_images = preprocessing_pipeline.denormalize(stylized_images)
return {'stylized_images': stylized_images}
frozen_serving_fn, _ = convert_variables_to_constants_v2_as_graph(
serving_fn.get_concrete_function(input_signature),
aggressive_inlining=True)
class InferenceModule(tf.Module):
def __init__(self, inference_function):
super(InferenceModule, self).__init__(name='inference_module')
self.inference_function = inference_function
@tf.function
def run_inference(self, sample):
outputs = self.inference_function(**sample)
return {'stylized_images': outputs[0]}
inference_module = InferenceModule(inference_function=frozen_serving_fn)
signatures = {
'serving_default':
inference_module.run_inference.get_concrete_function(input_signature)
}
print(signatures['serving_default'].output_shapes)
for _signature_name, _concrete_fn in signatures.items():
input_shapes = {
x.name.split(':')[0]: x.shape.as_list()
for x in _concrete_fn.inputs
}
output_shapes = {
k: v.as_list()
for k, v in _concrete_fn.output_shapes.items()
}
logging.info(
'\nSignature: {}\n Input Shapes:\n {}\nOutput Shapes:\n{}'.format(
_signature_name, input_shapes, output_shapes))
logging.info('Exporting `saved_model` to {}'.format(FLAGS.export_dir))
tf.saved_model.save(obj=inference_module,
export_dir=FLAGS.export_dir,
signatures=signatures,
options=tf.saved_model.SaveOptions(
experimental_custom_gradients=False))
def extract_feat(*args, **kwargs):
''' pyfeat, extract feat from utt and dump it '''
logging.debug("extract_feat : {}".format(kwargs))
winlen = kwargs.get('winlen')
winstep = kwargs.get('winstep')
feature_size = kwargs.get('feature_size')
sr = kwargs.get('sr') #pylint: disable=invalid-name
nfft = kwargs.get('nfft')
lowfreq = kwargs.get('lowfreq')
highfreq = kwargs.get('highfreq')
preemph = kwargs.get('preemph')
save_feat_path = kwargs.get('save_feat_path')
dry_run = kwargs.get('dry_run')
feat_type = kwargs.get('feat_type')
del lowfreq, preemph
if save_feat_path and not os.path.exists(save_feat_path):
os.makedirs(save_feat_path)
for wavpath in args:
if save_feat_path:
filename = os.path.splitext(os.path.split(wavpath)[-1])[0] + '.npy'
savepath = os.path.join(save_feat_path, filename)
else:
savepath = os.path.splitext(wavpath)[0] + '.npy'
logging.debug('input: {}, output: {}'.format(wavpath, savepath))
sr_out, samples = load_wav(wavpath, sr=sr)
del sr_out
feat = powspec_feat(
samples, sr=sr, nfft=nfft, winlen=winlen, winstep=winstep)
logging.debug('apply power spectorgram')
if feat_type == 'spectrogram':
# shape: [T, F]
feat = psf.logpowerspec(feat)
if highfreq:
resolution = freq_resolution(sr, nfft)
ps = int(points(highfreq, resolution)) #pylint: disable=invalid-name
logging.debug("feat slice: {} {}".format(ps, type(ps)))
feat = feat[:, :ps]
logging.debug('apply log power spectorgram')
elif feat_type == 'logfbank':
feat = fbank_feat(feat, sr=sr, nfft=nfft, feature_size=feature_size)
logging.debug('apply fbank spectorgram')
else:
raise ValueError("not support feat method")
feat = feat.astype(np.float32)
if dry_run:
logging.info('save feat: path {} shape:{} dtype:{}'.format(
savepath, feat.shape, feat.dtype))
else:
np.save(savepath, feat)
def clear_queue(self):
logging.debug('Purging %d chunks from queue.',
self._raw_audio_queue.qsize())
while not self._raw_audio_queue.empty():
self._raw_audio_queue.get()
def _save_resource_version(resource_version: int):
logging.debug('Saving resource_version %d', resource_version)
with open(_resource_version_path(), 'w') as f:
f.write(str(resource_version))
def close(self):
if self._bidi_stream is not None:
logging.debug('Closing bidi stream')
self._bidi_stream.close()
def utt_to_samples(self, args):
'''
Utt to samples of (feat_chunk, spk_id, sample_key).
Will be run in a process pool so restrictions apply.
'''
result_queue, utt_info = args
logging.debug(utt_info)
# TODO: wrap into a function or something
utt_key, utt_meta = utt_info
# Load features and select voiced frames.
feat_scp = utt_meta['feat']
feat_mat = kaldiio.load_mat(feat_scp)
num_frames_feat, feat_dim = feat_mat.shape
vad_scp = utt_meta['vad']
if vad_scp:
vad_mat = kaldiio.load_mat(vad_scp)
num_frames_vad = vad_mat.shape[0]
logging.debug('feat_mat: %s, vad_mat: %s' %
(str(feat_mat.shape), str(vad_mat.shape)))
if num_frames_feat != num_frames_vad:
logging.debug('num_frames_feat != num_frames_vad: %d vs %d' %
(num_frames_feat, num_frames_vad))
return None
voiced_frames_index = np.where(vad_mat == 1)[0]
logging.debug('voiced_frames_index: %s' %
(str(voiced_frames_index.shape)))
feat_mat_voiced = feat_mat[voiced_frames_index, :]
else:
# If no VAD info was found, the entire utt will be used.
feat_mat_voiced = feat_mat
num_frames_voiced = feat_mat_voiced.shape[0]
logging.debug('feat_mat_voiced: %s' % (str(feat_mat_voiced.shape)))
spk_id = utt_meta['spkid']
logging.debug('Chunk size: %d' % (self.chunk_size))
results = []
chunk_idx = 0
if self.add_random_offset:
random_offset = np.random.randint(0, self.chunk_size)
else:
random_offset = 0
for offset in range(random_offset, num_frames_voiced, self.chunk_size):
if self.single_chunk:
available = num_frames_voiced - self.chunk_size
if available < 0:
# No padding.
logging.warn('Single chunk mode: available < 0.')
break
offset = random.randint(0, available)
logging.debug('offset = %d' % (offset))
feat_chunk = feat_mat_voiced[offset:offset + self.chunk_size, :]
unpadded_frames = feat_chunk.shape[0]
if self.pad_chunks and unpadded_frames < self.chunk_size:
rel_chunk_len = float(unpadded_frames) / self.chunk_size
if rel_chunk_len < self.drop_short_chunks:
continue
logging.debug('Padding chunk of frames %d ...' %
(unpadded_frames))
padded = np.zeros((self.chunk_size, feat_dim),
dtype=feat_chunk.dtype)
padded[:unpadded_frames, :] = feat_chunk
feat_chunk = padded
feat_chunk = np.expand_dims(feat_chunk, axis=2) # TODO: not here
sample_key = '%s_chunk%02d' % (utt_key, chunk_idx)
sample = (feat_chunk, spk_id, sample_key)
chunk_idx += 1
results.append(sample)
if self.single_chunk:
break
if result_queue:
# queue mode
result_queue.put(results)
return None
# imap mode
return results
def _sync_weights_and_state_across_hosts(self):
"""Sync weights and state across all the hosts in the computation."""
if logging.vlog_is_on(1):
logging.debug(
'Input training weights shape: %s',
fastmath.nested_map(lambda x: x.shape, self._model.weights))
logging.debug('Input training weights: %s', self._model.weights)
logging.debug('Input training state: %s', self._model.state)
logging.debug('Input eval weights: %s', self._eval_model.weights)
logging.debug('Input eval state: %s', self._eval_model.state)
(self._model.weights, self._model.state, self._eval_model.weights,
self._eval_model.state) = self._unreplicate(
_make_weights_and_state_same_across_hosts(
self._for_n_devices(
(self._model.weights, self._model.state,
self._eval_model.weights, self._eval_model.state))))
if logging.vlog_is_on(1):
logging.debug(
'Output training weights shape: %s',
fastmath.nested_map(lambda x: x.shape, self._model.weights))
logging.debug('Output training weights: %s', self._model.weights)
logging.debug('Output training state: %s', self._model.state)
logging.debug('Output eval weights: %s', self._eval_model.weights)
logging.debug('Output eval state: %s', self._eval_model.state)
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
ret_str = []
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.af == 'inet6' and 'icmp' in self.term.protocol)
or (self.af == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(
self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=', '.join(
self.term.protocol),
af=self.af))
return ''
# Term verbatim output - this will skip over most normal term
# creation code by returning early. Warnings provided in policy.py
if self.term.verbatim:
for next_verbatim in self.term.verbatim:
if next_verbatim[0] == self._PLATFORM:
ret_str.append(str(next_verbatim[1]))
return '\n'.join(ret_str)
# Create a new term
self._SetDefaultAction()
if self._TERM_FORMAT:
ret_str.append(self._TERM_FORMAT.substitute(term=self.term_name))
if self._PREJUMP_FORMAT:
ret_str.append(
self._PREJUMP_FORMAT.substitute(filter=self.filter,
term=self.term_name))
if self.verbose:
if self.term.owner:
self.term.comment.append('Owner: %s' % self.term.owner)
# reformat long comments, if needed
#
# iptables allows individual comments up to 256 chars.
# But our generator will limit a single comment line to < 120, using:
# max = 119 - 27 (static chars in comment command) - [length of term name]
comment_max_width = 92 - len(self.term_name)
if comment_max_width < 40:
comment_max_width = 40
comments = aclgenerator.WrapWords(self.term.comment,
comment_max_width)
# append comments to output
if comments and comments[0]:
for line in comments:
if not line:
continue # iptables-restore does not like 0-length comments.
# term comments
# Strip out quotes as iptables cant have nested quotes
ret_str.append(
self._COMMENT_FORMAT.substitute(
filter=self.filter,
term=self.term_name,
comment=str(line).replace('\"', '')))
# Unsupported configuration; in the case of 'accept' or 'next', we
# skip the rule. In other cases, we blow up (raise an exception)
# to ensure that this is not considered valid configuration.
if self.term.source_prefix or self.term.destination_prefix:
if str(self.term.action[0]) not in set(['accept', 'next']):
raise UnsupportedFilterError(
'%s %s %s %s %s %s %s %s' %
('\nTerm', self.term.name, 'has action',
str(self.term.action[0]),
'with source_prefix or destination_prefix,',
' which is unsupported in', self._PLATFORM,
'iptables output.'))
return ('# skipped %s due to source or destination prefix rule' %
self.term.name)
# protocol
if self.term.protocol:
protocol = self.term.protocol
else:
protocol = ['all']
if 'hopopt' in protocol and self.af == 'inet':
logging.warning('Term %s is using hopopt in IPv4 context.',
self.term_name)
return ''
(term_saddr, exclude_saddr, term_daddr,
exclude_daddr) = self._CalculateAddresses(
self.term.source_address, self.term.source_address_exclude,
self.term.destination_address,
self.term.destination_address_exclude)
if not term_saddr:
logging.debug(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
if not term_daddr:
logging.debug(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
# ports
source_port = []
destination_port = []
if self.term.source_port:
source_port = self.term.source_port
if self.term.destination_port:
destination_port = self.term.destination_port
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, protocol,
self.af)
source_interface = ''
if self.term.source_interface:
source_interface = self.term.source_interface
destination_interface = ''
if self.term.destination_interface:
destination_interface = self.term.destination_interface
log_hits = False
if self.term.logging:
# Iptables sends logs to hosts configured syslog
log_hits = True
# options
tcp_flags = []
tcp_track_options = []
for next_opt in [str(x) for x in self.term.option]:
#
# Sanity checking and high-ports are added as appropriate in
# pre-processing that is done in __str__ within class Iptables.
# Option established will add destination port high-ports if protocol
# contains only tcp, udp or both. This is done earlier in class Iptables.
#
if ((next_opt.find('established') == 0
or next_opt.find('tcp-established') == 0)
and 'ESTABLISHED' not in [x.strip()
for x in self.options]):
if next_opt.find(
'tcp-established') == 0 and protocol != ['tcp']:
raise TcpEstablishedError('%s %s %s' % (
'\noption tcp-established can only be applied for proto tcp.',
'\nError in term:', self.term.name))
if self.trackstate:
# Use nf_conntrack to track state -- works with any proto
self.options.append('-m state --state ESTABLISHED,RELATED')
elif protocol == ['tcp']:
# Simple established-only rule for TCP: Must have ACK field
# (SYN/ACK or subsequent ACK), or RST and no other flags.
tcp_track_options = [(['ACK'], ['ACK']),
(['SYN', 'FIN', 'ACK',
'RST'], ['RST'])]
# Iterate through flags table, and create list of tcp-flags to append
for next_flag in self._TCP_FLAGS_TABLE:
if next_opt.find(next_flag) == 0:
tcp_flags.append(self._TCP_FLAGS_TABLE.get(next_flag))
if next_opt in self._KNOWN_OPTIONS_MATCHERS:
self.options.append(self._KNOWN_OPTIONS_MATCHERS[next_opt])
if self.term.packet_length:
# Policy format is "#-#", but iptables format is "#:#"
self.options.append('-m length --length %s' %
self.term.packet_length.replace('-', ':'))
if self.term.fragment_offset:
self.options.append('-m u32 --u32 4&0x1FFF=%s' %
self.term.fragment_offset.replace('-', ':'))
icmp_code = ['']
if self.term.icmp_code:
icmp_code = self.term.icmp_code
for saddr in exclude_saddr:
ret_str.extend(
self._FormatPart('', saddr, '', '', '', '', '', '', '', '', '',
'', '', self._action_table.get('next')))
for daddr in exclude_daddr:
ret_str.extend(
self._FormatPart('', '', '', daddr, '', '', '', '', '', '', '',
'', '', self._action_table.get('next')))
for saddr in term_saddr:
for daddr in term_daddr:
for icmp in icmp_types:
for code in icmp_code:
for proto in protocol:
for tcp_matcher in tcp_track_options or (([],
[]), ):
ret_str.extend(
self._FormatPart(
str(proto), saddr, source_port, daddr,
destination_port, self.options,
tcp_flags, icmp, code, tcp_matcher,
source_interface,
destination_interface, log_hits,
self._action_table.get(
str(self.term.action[0]))))
if self._POSTJUMP_FORMAT:
ret_str.append(
self._POSTJUMP_FORMAT.substitute(filter=self.filter,
term=self.term_name))
return '\n'.join(str(v) for v in ret_str if v is not '')
def RunSetupCommand(self, timeout_seconds: int = 60) -> None:
if self.config.HasField('setup_cmd'):
cmd = self._MakeVariableSubstitution(self.config.setup_cmd)
cmd = f"timeout -s9 {timeout_seconds} bash -c '{cmd}'"
logging.debug('$ %s', cmd)
subprocess.check_call(cmd, shell=True)
def Step(self, step: random_opt_pb2.Step) -> random_opt_pb2.Step:
"""Run a step.
Args:
step: A step proto with field 'opt_pass' set.
Returns:
The input step.
"""
start_time = labdate.MillisecondsTimestamp()
step.start_time_epoch_ms = start_time
step.status = random_opt_pb2.Step.PASS
temp_bytecode = self.working_dir / 'temp_src.ll'
temp_binary = self.working_dir / 'temp_binary'
# Run the pass.
try:
opt.RunOptPassOnBytecode(self.working_bytecode_path, temp_bytecode,
list(step.opt_pass))
except llvm.LlvmError as e:
step.status = random_opt_pb2.Step.OPT_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
# Update bytecode file.
logging.debug('$ mv %s %s', temp_bytecode,
self.working_bytecode_path)
step.bytecode_changed = BytecodesAreEqual(
temp_bytecode, self.working_bytecode_path)
os.rename(str(temp_bytecode), str(self.working_bytecode_path))
# Compile a new binary.
try:
clang.Compile([self.working_bytecode_path],
temp_binary,
copts=['-O0'])
step.binary_changed = BinariesAreEqual(temp_binary,
self.binary_path)
os.rename(str(temp_binary), str(self.binary_path))
except llvm.LlvmError as e:
step.status = random_opt_pb2.Step.COMPILE_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
# Get the binary runtime.
try:
step.binary_runtime_ms.extend(self.GetRuntimes())
except ValueError as e:
step.status = random_opt_pb2.Step.EXEC_FAILED
step.status_msg = text.truncate(str(e), 255)
if step.status == random_opt_pb2.Step.PASS:
if self.BinaryIsValid():
step.speedup = (
(sum(self.episodes[-1].step[-1].binary_runtime_ms) /
len(self.episodes[-1].step[-1].binary_runtime_ms)) /
(sum(step.binary_runtime_ms) /
len(step.binary_runtime_ms)))
step.total_speedup = (
(sum(self.episodes[-1].step[0].binary_runtime_ms) /
len(self.episodes[-1].step[0].binary_runtime_ms)) /
(sum(step.binary_runtime_ms) /
len(step.binary_runtime_ms)))
else:
step.status = random_opt_pb2.Step.EVAL_FAILED
step.reward = self.Reward(step.status, step.speedup)
step.total_reward = self.episodes[-1].step[
-1].total_reward + step.reward
step.total_step_runtime_ms = labdate.MillisecondsTimestamp(
) - start_time
return step
def main(args):
FLAGS(args)
if FLAGS.verbose:
logging.basicConfig(level=logging.INFO)
if FLAGS.debug:
logging.basicConfig(level=logging.DEBUG)
logging.debug('binary: %s\noptimize: %d\nbase_directory: %s\n'
'policy_file: %s\nrendered_acl_directory: %s',
str(sys.argv[0]),
int(FLAGS.optimize),
str(FLAGS.base_directory),
str(FLAGS.policy_file),
str(FLAGS.output_directory))
definitions = None
try:
definitions = naming.Naming(FLAGS.definitions_directory)
except naming.NoDefinitionsError:
err_msg = 'bad definitions directory: %s', FLAGS.definitions_directory
logging.fatal(err_msg)
sys.exit(1)
# thead-safe list for storing files to write
manager = multiprocessing.Manager()
write_files = manager.list()
with_errors = False
if FLAGS.policy_file:
# render just one file
logging.info('rendering one file')
RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions,
FLAGS.exp_info, write_files)
else:
# render all files in parallel
logging.info('finding policies...')
pols = []
pols.extend(DescendRecursively(FLAGS.base_directory, FLAGS.output_directory,
definitions))
pool = multiprocessing.Pool(processes=FLAGS.max_renderers)
results = []
for x in pols:
results.append(pool.apply_async(RenderFile,
args=(x.get('in_file'),
x.get('out_dir'),
definitions,
FLAGS.exp_info,
write_files)))
pool.close()
pool.join()
for result in results:
try:
result.get()
except (ACLParserError, ACLGeneratorError) as e:
with_errors = True
logging.warn('\n\nerror encountered in rendering process:\n%s\n\n', e)
# actually write files to disk
WriteFiles(write_files)
if with_errors:
logging.warn('done, with errors.')
sys.exit(1)
else:
logging.info('done.')
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
config = Config(indent=self._DEFAULT_INDENT)
from_str = []
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.term_type == 'inet6' and 'icmp' in self.term.protocol) or
(self.term_type == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(
self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=', '.join(
self.term.protocol),
af=self.term_type))
return ''
# comment
# this deals just fine with multi line comments, but we could probably
# output them a little cleaner; do things like make sure the
# len(output) < 80, etc. Note, if 'noverbose' is set for the filter, skip
# all comment processing.
if self.term.owner and not self.noverbose:
self.term.comment.append('Owner: %s' % self.term.owner)
if self.term.comment and not self.noverbose:
config.Append('/*')
for comment in self.term.comment:
for line in comment.split('\n'):
config.Append('** ' + line)
config.Append('*/')
# Term verbatim output - this will skip over normal term creation
# code. Warning generated from policy.py if appropriate.
if self.term.verbatim:
for next_term in self.term.verbatim:
if next_term[0] == self._PLATFORM:
config.Append(str(next_term[1]), verbatim=True)
return str(config)
# Helper for per-address-family keywords.
family_keywords = self._TERM_TYPE.get(self.term_type)
# option
# this is going to be a little ugly b/c there are a few little messed
# up options we can deal with.
if self.term.option:
for opt in [str(x) for x in self.term.option]:
# there should be a better way to search the array of protocols
if opt.startswith('sample'):
self.extra_actions.append('sample')
# only append tcp-established for option established when
# tcp is the only protocol, otherwise other protos break on juniper
elif opt.startswith('established'):
if self.term.protocol == ['tcp']:
if 'tcp-established;' not in from_str:
from_str.append(family_keywords['tcp-est'] + ';')
# if tcp-established specified, but more than just tcp is included
# in the protocols, raise an error
elif opt.startswith('tcp-established'):
flag = family_keywords['tcp-est'] + ';'
if self.term.protocol == ['tcp']:
if flag not in from_str:
from_str.append(flag)
else:
raise TcpEstablishedWithNonTcpError(
'tcp-established can only be used with tcp protocol in term %s'
% self.term.name)
elif opt.startswith('rst'):
from_str.append('tcp-flags "rst";')
elif opt.startswith('initial') and 'tcp' in self.term.protocol:
from_str.append('tcp-initial;')
elif opt.startswith('first-fragment'):
from_str.append('first-fragment;')
# we don't have a special way of dealing with this, so we output it and
# hope the user knows what they're doing.
else:
from_str.append('%s;' % opt)
# if the term is inactive we have to set the prefix
if self.term.inactive:
term_prefix = 'inactive:'
else:
term_prefix = ''
# term name
config.Append('%s term %s {' % (term_prefix, self.term.name))
# a default action term doesn't have any from { clause
has_match_criteria = (
self.term.address or self.term.dscp_except or self.term.dscp_match
or self.term.destination_address or self.term.destination_port
or self.term.destination_prefix
or self.term.destination_prefix_except or self.term.ether_type
or self.term.flexible_match_range or self.term.forwarding_class
or self.term.forwarding_class_except or self.term.fragment_offset
or self.term.hop_limit or self.term.next_ip or self.term.port
or self.term.precedence or self.term.protocol
or self.term.protocol_except or self.term.source_address
or self.term.source_port or self.term.source_prefix
or self.term.source_prefix_except or self.term.traffic_type
or self.term.ttl)
if has_match_criteria:
config.Append('from {')
term_af = self.AF_MAP.get(self.term_type)
# address
address = self.term.GetAddressOfVersion('address', term_af)
if self.enable_dsmo:
address = summarizer.Summarize(address)
if address:
config.Append('%s {' % family_keywords['addr'])
for addr in address:
for comment in self._Comment(addr):
config.Append('%s' % comment)
if self.enable_dsmo:
config.Append(
'%s/%s;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
config.Append('%s;' % addr)
config.Append('}')
elif self.term.address:
logging.debug(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
af=self.term_type))
return ''
# source address
src_addr = self.term.GetAddressOfVersion('source_address', term_af)
src_addr_ex = self.term.GetAddressOfVersion(
'source_address_exclude', term_af)
if self.enable_dsmo:
src_addr = summarizer.Summarize(src_addr)
src_addr_ex = summarizer.Summarize(src_addr_ex)
else:
src_addr, src_addr_ex = self._MinimizePrefixes(
src_addr, src_addr_ex)
if src_addr:
config.Append('%s {' % family_keywords['saddr'])
for addr in src_addr:
for comment in self._Comment(addr):
config.Append('%s' % comment)
if self.enable_dsmo:
config.Append(
'%s/%s;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
config.Append('%s;' % addr)
for addr in src_addr_ex:
for comment in self._Comment(addr, exclude=True):
config.Append('%s' % comment)
if self.enable_dsmo:
config.Append(
'%s/%s except;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
config.Append('%s except;' % addr)
config.Append('}')
elif self.term.source_address:
logging.debug(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.term_type))
return ''
# destination address
dst_addr = self.term.GetAddressOfVersion('destination_address',
term_af)
dst_addr_ex = self.term.GetAddressOfVersion(
'destination_address_exclude', term_af)
if self.enable_dsmo:
dst_addr = summarizer.Summarize(dst_addr)
dst_addr_ex = summarizer.Summarize(dst_addr_ex)
else:
dst_addr, dst_addr_ex = self._MinimizePrefixes(
dst_addr, dst_addr_ex)
if dst_addr:
config.Append('%s {' % family_keywords['daddr'])
for addr in dst_addr:
for comment in self._Comment(addr):
config.Append('%s' % comment)
if self.enable_dsmo:
config.Append(
'%s/%s;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
config.Append('%s;' % addr)
for addr in dst_addr_ex:
for comment in self._Comment(addr, exclude=True):
config.Append('%s' % comment)
if self.enable_dsmo:
config.Append(
'%s/%s except;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
config.Append('%s except;' % addr)
config.Append('}')
elif self.term.destination_address:
logging.debug(
self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.term_type))
return ''
# forwarding-class
if self.term.forwarding_class:
config.Append(
'forwarding-class %s' %
self._Group(self.term.forwarding_class, lc=False))
# forwarding-class-except
if self.term.forwarding_class_except:
config.Append(
'forwarding-class-except %s' %
self._Group(self.term.forwarding_class_except, lc=False))
# source prefix <except> list
if self.term.source_prefix or self.term.source_prefix_except:
config.Append('source-prefix-list {')
for pfx in self.term.source_prefix:
config.Append(pfx + ';')
for epfx in self.term.source_prefix_except:
config.Append(epfx + ' except;')
config.Append('}')
# destination prefix <except> list
if self.term.destination_prefix or self.term.destination_prefix_except:
config.Append('destination-prefix-list {')
for pfx in self.term.destination_prefix:
config.Append(pfx + ';')
for epfx in self.term.destination_prefix_except:
config.Append(epfx + ' except;')
config.Append('}')
if self.term.ttl:
config.Append('ttl %s;' % self.term.ttl)
# protocol
if self.term.protocol:
# both are supported on JunOS, but only icmp6 is supported
# on SRX loopback stateless filter
config.Append(family_keywords['protocol'] + ' ' +
self._Group(self.term.protocol))
# protocol
if self.term.protocol_except:
# same as above
config.Append(family_keywords['protocol-except'] + ' ' +
self._Group(self.term.protocol_except))
# port
if self.term.port:
config.Append('port %s' % self._Group(self.term.port))
# source port
if self.term.source_port:
config.Append('source-port %s' %
self._Group(self.term.source_port))
# destination port
if self.term.destination_port:
config.Append('destination-port %s' %
self._Group(self.term.destination_port))
# append any options beloging in the from {} section
for next_str in from_str:
config.Append(next_str)
# packet length
if self.term.packet_length:
config.Append('packet-length %s;' % self.term.packet_length)
# fragment offset
if self.term.fragment_offset:
config.Append('fragment-offset %s;' %
self.term.fragment_offset)
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol,
self.term_type)
if icmp_types != ['']:
config.Append('icmp-type %s' % self._Group(icmp_types))
if self.term.icmp_code:
config.Append('icmp-code %s' %
self._Group(self.term.icmp_code))
if self.term.ether_type:
config.Append('ether-type %s' %
self._Group(self.term.ether_type))
if self.term.traffic_type:
config.Append('traffic-type %s' %
self._Group(self.term.traffic_type))
if self.term.precedence:
# precedence may be a single integer, or a space separated list
policy_precedences = set()
# precedence values may only be 0 through 7
for precedence in self.term.precedence:
if int(precedence) in range(0, 8):
policy_precedences.add(precedence)
else:
raise PrecedenceError(
'Precedence value %s is out of bounds in %s' %
(precedence, self.term.name))
config.Append('precedence %s' %
self._Group(sorted(policy_precedences)))
# DSCP Match
if self.term.dscp_match:
if self.term_type == 'inet6':
config.Append('traffic-class [ %s ];' %
(' '.join(self.term.dscp_match)))
else:
config.Append('dscp [ %s ];' %
' '.join(self.term.dscp_match))
# DSCP Except
if self.term.dscp_except:
if self.term_type == 'inet6':
config.Append('traffic-class-except [ %s ];' %
(' '.join(self.term.dscp_except)))
else:
config.Append('dscp-except [ %s ];' %
' '.join(self.term.dscp_except))
if self.term.hop_limit:
# Only generate a hop-limit if inet6, inet4 has not hop-limit.
if self.term_type == 'inet6':
config.Append('hop-limit %s;' % (self.term.hop_limit))
# flexible-match
if self.term.flexible_match_range:
config.Append('flexible-match-range {')
for fm_opt in self.term.flexible_match_range:
config.Append('%s %s;' % (fm_opt[0], fm_opt[1]))
config.Append('}')
config.Append('}') # end from { ... }
####
# ACTIONS go below here
####
# If the action is only one line, include it in the same line as "then "
# statement.
# For example, if the action is only accept, it should be:
# "then accept;" rather than:
# "then {
# accept;
# }"
#
unique_actions = set(self.extra_actions)
if not self.term.routing_instance:
unique_actions.update(self.term.action)
if len(unique_actions) <= 1:
for action in [
self.term.logging, self.term.routing_instance,
self.term.counter, self.term.policer, self.term.qos,
self.term.loss_priority, self.term.dscp_set,
self.term.next_ip, self.term.traffic_class_count
]:
if action:
try:
unique_actions.update(action)
except TypeError:
unique_actions.add(action)
if len(unique_actions) > 1:
break
if len(unique_actions) == 1:
# b/21795531: Juniper device treats a set of IPv4 actions differently
# than any other actions.
# For example, if the term is in IPv4 and the action is only discard,
# it should be:
# "then {
# discard;
# }" rather than:
# "then discard;"
current_action = self.ACTIONS.get(unique_actions.pop(), 'next_ip')
if (self.term_type == 'inet' and current_action in [
'discard', 'reject', 'reject tcp-reset'
]) or (self.term_type == 'inet6'
and current_action in ['reject', 'reject tcp-reset']):
config.Append('then {')
config.Append('%s;' % current_action)
config.Append('}')
elif current_action == 'next_ip':
self.NextIpCheck(self.term.next_ip, self.term.name)
config.Append('then {')
if self.term.next_ip[0].version == 4:
config.Append('next-ip %s;' % str(self.term.next_ip[0]))
else:
config.Append('next-ip6 %s;' % str(self.term.next_ip[0]))
config.Append('}')
else:
config.Append('then %s;' % current_action)
elif len(unique_actions) > 1:
config.Append('then {')
# logging
if self.term.logging:
for log_target in self.term.logging:
if str(log_target) == 'local':
config.Append('log;')
else:
config.Append('syslog;')
if self.term.routing_instance:
config.Append('routing-instance %s;' %
self.term.routing_instance)
if self.term.counter:
config.Append('count %s;' % self.term.counter)
if self.term.traffic_class_count:
config.Append('traffic-class-count %s;' %
self.term.traffic_class_count)
oid_length = 128
if self.term.policer:
config.Append('policer %s;' % self.term.policer)
if len(self.term.policer) > oid_length:
logging.warn(
'WARNING: %s is longer than %d bytes. Due to limitation'
' in JUNOS, OIDs longer than %dB can cause SNMP '
'timeout issues.', self.term.policer, oid_length,
oid_length)
if self.term.qos:
config.Append('forwarding-class %s;' % self.term.qos)
if self.term.loss_priority:
config.Append('loss-priority %s;' % self.term.loss_priority)
if self.term.next_ip:
self.NextIpCheck(self.term.next_ip, self.term.name)
if self.term.next_ip[0].version == 4:
config.Append('next-ip %s;' % str(self.term.next_ip[0]))
else:
config.Append('next-ip6 %s;' % str(self.term.next_ip[0]))
for action in self.extra_actions:
config.Append(action + ';')
# If there is a routing-instance defined, skip reject/accept/etc actions.
if not self.term.routing_instance:
for action in self.term.action:
config.Append(self.ACTIONS.get(action) + ';')
# DSCP SET
if self.term.dscp_set:
if self.term_type == 'inet6':
config.Append('traffic-class %s;' % self.term.dscp_set)
else:
config.Append('dscp %s;' % self.term.dscp_set)
config.Append('}') # end then{...}
config.Append('}') # end term accept-foo-to-bar { ... }
return str(config)
def __str__(self):
output = []
# Don't render term if not in platforms or in excluded platforms
if self.term.platform and self._PLATFORM not in self.term.platform:
return ''
if (self.term.platform_exclude and
self._PLATFORM in self.term.platform_exclude):
return ''
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
# Does not currently support mixed family.
if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or
(self.af == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.af))
return ''
# Term verbatim output - this will skip over most normal term
# creation code by returning early. Warnings provided in policy.py.
if self.term.verbatim:
for verbatim_line in self.term.verbatim:
platform, contents = verbatim_line.value
if platform == self._PLATFORM:
output.append(str(contents))
return '\n'.join(output)
# Source address
if self.term.source_address or self.term.source_address_exclude:
src_addrs = self._CalculateAddrs(self.term.source_address,
self.term.source_address_exclude)
if not src_addrs:
logging.warn(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
# TODO(castagno): Add support for ipv6
output.append('ip saddr %s' % self._FormatMatch(src_addrs))
# Destination address
if self.term.destination_address or self.term.source_address_exclude:
dst_addrs = self._CalculateAddrs(self.term.destination_address,
self.term.destination_address_exclude)
if not dst_addrs:
logging.warn(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
# TODO(castagno): Add support for ipv6
output.append('ip daddr %s' % self._FormatMatch(dst_addrs))
# Protocol
#
# nft intepreter shortcuts protocol specification if there are more specific
# matches. At the moment, these are:
# * source port
# * destination port
# * ICMP type
if self.term.protocol and not (self.term.source_port or
self.term.destination_port or
self.term.icmp_type):
output.append('ip protocol %s' % self._FormatMatch(self.term.protocol))
# Source port
if self.term.source_port:
output.append('%s sport %s' %
(self._FormatMatch(self.term.protocol),
self._FormatMatch(self.term.source_port)))
# Destination port
if self.term.destination_port:
output.append('%s dport %s' %
(self._FormatMatch(self.term.protocol),
self._FormatMatch(self.term.destination_port)))
# Icmp type
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol,
self.af)
if icmp_types != ['']:
# nft intepreter requires ICMP types to be spelled out
icmp_name_types = self.ICMP_TYPE[self.AF_MAP[self.af]]
icmp_type_names = dict((v, k) for k, v in icmp_name_types.iteritems())
output.append('icmp type %s' %
self._FormatMatch([icmp_type_names[icmp_type] for
icmp_type in icmp_types]))
# Counter
# This does not use the value that was passed in the term.
if self.term.counter:
output.append('counter')
# Log
# Setup logic so that only one log statement is printed.
if self.term.logging and not self.term.log_name:
output.append('log')
elif (self.term.logging and self.term.log_name) or self.term.log_name:
# Only supports log prefix's of 128 characters truncate to 126 to support
# the additional suffix that is being added
output.append('log prefix "%s: "' % self.term.log_name[:126])
# Action
output.append(self._ACTIONS[self.term.action[0]])
# Owner (implement as comment)
if self.term.owner:
self.term.comment.append('Owner: %s' % self.term.owner)
# Comment
if self.term.comment:
comment_data = ' '.join(self.term.comment)
# Have to truncate MAX_CHARACTERS characters due to NFTables limitation
if len(comment_data) > self.MAX_CHARACTERS:
# Have to use the first MAX_CHARACTERS characters
comment_data = comment_data[:self.MAX_CHARACTERS]
logging.warn(
'Term %s in policy is too long (>%d characters) '
'and will be truncated', self.term.name, self.MAX_CHARACTERS)
output.append('comment "%s"' % comment_data)
return ' '.join(output)
def __str__(self):
"""Render config output from this term object."""
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
conditions = []
# if terms does not specify action, use filter default action
if not self.term.action:
self.term.action[0].value = self.default_action
if str(self.term.action[0]) not in self._ACTION_TABLE:
raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % (
'\n', self.term.name, self.term.action[0],
'action not currently supported.'))
# source address
term_saddrs = self._CheckAddressAf(self.term.source_address)
if not term_saddrs:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
conditions.append(self._GenerateAddrStatement(
term_saddrs, self.term.source_address_exclude))
# destination address
term_daddrs = self._CheckAddressAf(self.term.destination_address)
if not term_daddrs:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
conditions.append(self._GenerateAddrStatement(
term_daddrs, self.term.destination_address_exclude))
# protocol
if self.term.protocol_except:
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'protocol_except logic not currently supported.'))
conditions.append(self._GenerateProtoStatement(self.term.protocol))
conditions.append(self._GeneratePortStatement(
self.term.source_port, 'src'))
conditions.append(self._GeneratePortStatement(
self.term.destination_port, 'dst'))
# icmp-type
icmp_types = ['']
if self.term.icmp_type:
if self.af != 'mixed':
af = self.af
elif self.term.protocol == ['icmp']:
af = 'inet'
elif self.term.protocol == ['icmp6']:
af = 'inet6'
else:
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'icmp protocol is not defined or not supported.'))
icmp_types = self.NormalizeIcmpTypes(
self.term.icmp_type, self.term.protocol, af)
if 'icmp' in self.term.protocol:
conditions.append(self._GenerateIcmpType(icmp_types,
self.term.icmp_code))
# tcp options
if 'tcp' in self.term.protocol:
conditions.append(self._GenerateTcpOptions(self.term.option))
cond = Term.JoinConditionals(conditions, 'and')
# Note that directionally-based pcap filter requires post-processing to
# replace 'localhost' with whatever the IP(s) of the local machine happen
# to be. This bit of logic ensure there's a placeholder with the
# appropriate booleans around it. We also have to check that there exists
# some form of condition already, else we'll end up with something overly
# broad like 'dst net localhost' (e.g., 'default-deny').
if cond and self.direction == 'in':
cond = Term.JoinConditionals(['dst net localhost', cond], 'and')
elif cond and self.direction == 'out':
cond = Term.JoinConditionals(['src net localhost', cond], 'and')
return cond + '\n'
def __str__(self):
"""Convert term to a rule string.
Returns:
A rule as a string.
Raises:
NsxvAclTermError: When unknown icmp-types are specified
"""
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if 'nsxv' not in self.term.platform:
return ''
if self.term.platform_exclude:
if 'nsxv' in self.term.platform_exclude:
return ''
ret_str = ['']
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.af == 6 and 'icmp' in self.term.protocol) or
(self.af == 4 and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.filter_type))
return ''
# Term verbatim is not supported
if self.term.verbatim:
raise NsxvAclTermError(
'Verbatim are not implemented in standard ACLs')
# Term option is not supported
if self.term.option:
for opt in [str(single_option) for single_option in self.term.option]:
if((opt.find('tcp-established') == 0)
or (opt.find('established') == 0)):
return ''
else:
raise NsxvAclTermError(
'Option are not implemented in standard ACLs')
# check for keywords Nsxv does not support
term_keywords = self.term.__dict__
unsupported_keywords = []
for key in term_keywords:
if term_keywords[key]:
# translated is obj attribute not keyword
if ('translated' not in key) and (key not in _NSXV_SUPPORTED_KEYWORDS):
unsupported_keywords.append(key)
if unsupported_keywords:
logging.warn('WARNING: The keywords %s in Term %s are not supported in '
'Nsxv ', unsupported_keywords, self.term.name)
name = '%s%s%s' % (_XML_TABLE.get('nameStart'), self.term.name,
_XML_TABLE.get('nameEnd'))
notes = ''
if self.term.comment:
for comment in self.term.comment:
notes = '%s%s' %(notes, comment)
notes = '%s%s%s' % (_XML_TABLE.get('noteStart'), notes,
_XML_TABLE.get('noteEnd'))
# protocol
protocol = None
if self.term.protocol:
protocol = map(self.PROTO_MAP.get, self.term.protocol, self.term.protocol)
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol,
self.af)
# for mixed filter type get both IPV4address and IPv6Address
af_list = []
if self.filter_type == 'mixed':
af_list = [4, 6]
else:
af_list = [self.af]
source_address = None
destination_address = None
source_addr = []
destination_addr = []
source_v4_addr = []
source_v6_addr = []
dest_v4_addr = []
dest_v6_addr = []
for af in af_list:
# source address
if self.term.source_address:
source_address = self.term.GetAddressOfVersion('source_address', af)
source_address_exclude = self.term.GetAddressOfVersion(
'source_address_exclude', af)
if source_address_exclude:
source_address = nacaddr.ExcludeAddrs(
source_address,
source_address_exclude)
if source_address:
if af == 4:
source_v4_addr = source_address
else:
source_v6_addr = source_address
source_addr = source_v4_addr + source_v6_addr
# destination address
if self.term.destination_address:
destination_address = self.term.GetAddressOfVersion(
'destination_address', af)
destination_address_exclude = self.term.GetAddressOfVersion(
'destination_address_exclude', af)
if destination_address_exclude:
destination_address = nacaddr.ExcludeAddrs(
destination_address,
destination_address_exclude)
if destination_address:
if af == 4:
dest_v4_addr = destination_address
else:
dest_v6_addr = destination_address
destination_addr = dest_v4_addr + dest_v6_addr
# Check for mismatch IP for source and destination address for mixed filter
if self.filter_type == 'mixed':
if source_addr and destination_addr:
if source_v4_addr and not dest_v4_addr:
source_addr = source_v6_addr
elif source_v6_addr and not dest_v6_addr:
source_addr = source_v4_addr
elif dest_v4_addr and not source_v4_addr:
destination_addr = dest_v6_addr
elif dest_v6_addr and not source_v6_addr:
destination_addr = dest_v4_addr
if not source_addr or not destination_addr:
logging.warn('Term %s will not be rendered as it has IPv4/IPv6 '
'mismatch for source/destination for mixed address '
'family.', self.term.name)
return ''
# ports
source_port = None
destination_port = None
if self.term.source_port:
source_port = self.term.source_port
if self.term.destination_port:
destination_port = self.term.destination_port
# logging
log = 'false'
if self.term.logging:
log = 'true'
sources = ''
if source_addr:
sources = '<sources excluded="false">'
for saddr in source_addr:
# inet4
if isinstance(saddr, nacaddr.IPv4):
if saddr.numhosts > 1:
saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv4Start'),
saddr.with_prefixlen,
_XML_TABLE.get('srcIpv4End'),)
else:
saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv4Start'),
saddr.ip,
_XML_TABLE.get('srcIpv4End'))
sources = '%s%s' %(sources, saddr)
# inet6
if isinstance(saddr, nacaddr.IPv6):
if saddr.numhosts > 1:
saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv6Start'),
saddr.with_prefixlen,
_XML_TABLE.get('srcIpv6End'),)
else:
saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv6Start'),
saddr.ip, _XML_TABLE.get('srcIpv6End'))
sources = '%s%s' %(sources, saddr)
sources = '%s%s' %(sources, '</sources>')
destinations = ''
if destination_addr:
destinations = '<destinations excluded="false">'
for daddr in destination_addr:
# inet4
if isinstance(daddr, nacaddr.IPv4):
if daddr.numhosts > 1:
daddr = '%s%s%s' % (_XML_TABLE.get('destIpv4Start'),
daddr.with_prefixlen,
_XML_TABLE.get('destIpv4End'),)
else:
daddr = '%s%s%s' % (_XML_TABLE.get('destIpv4Start'),
daddr.ip,
_XML_TABLE.get('destIpv4End'))
destinations = '%s%s' %(destinations, daddr)
# inet6
if isinstance(daddr, nacaddr.IPv6):
if daddr.numhosts > 1:
daddr = '%s%s%s' % (_XML_TABLE.get('destIpv6Start'),
daddr.with_prefixlen,
_XML_TABLE.get('destIpv6End'),)
else:
daddr = '%s%s%s' % (_XML_TABLE.get('destIpv6Start'),
daddr.ip,
_XML_TABLE.get('destIpv6End'))
destinations = '%s%s' %(destinations, daddr)
destinations = '%s%s' %(destinations, '</destinations>')
services = []
if protocol:
services.append('<services>')
for proto in protocol:
if proto != 'any':
services.append(self._ServiceToString(proto,
source_port,
destination_port,
icmp_types))
services.append('</services>')
service = ''
for s in services:
service = '%s%s' % (service, s)
# applied_to
applied_to_list = ''
if self.applied_to:
applied_to_list = '<appliedToList>'
applied_to_element = '%s%s%s' % (_XML_TABLE.get('appliedToStart'),
self.applied_to,
_XML_TABLE.get('appliedToEnd'))
applied_to_list = '%s%s' %(applied_to_list, applied_to_element)
applied_to_list = '%s%s' %(applied_to_list, '</appliedToList>')
# action
action = '%s%s%s' % (_XML_TABLE.get('actionStart'),
_ACTION_TABLE.get(str(self.term.action[0])),
_XML_TABLE.get('actionEnd'))
ret_lines = []
ret_lines.append('<rule logged="%s">%s%s%s%s%s%s%s</rule>' %
(log, name, action, sources, destinations, service,
applied_to_list, notes))
# remove any trailing spaces and replace multiple spaces with singles
stripped_ret_lines = [re.sub(r'\s+', ' ', x).rstrip() for x in ret_lines]
ret_str.extend(stripped_ret_lines)
return ''.join(ret_str)
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
config = Config(indent=self._DEFAULT_INDENT)
from_str = []
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.term_type == 'inet6' and 'icmp' in self.term.protocol) or
(self.term_type == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.term_type))
return ''
# comment
# this deals just fine with multi line comments, but we could probably
# output them a little cleaner; do things like make sure the
# len(output) < 80, etc. Note, if 'noverbose' is set for the filter, skip
# all comment processing.
if self.term.owner and not self.noverbose:
self.term.comment.append('Owner: %s' % self.term.owner)
if self.term.comment and not self.noverbose:
config.Append('/*')
for comment in self.term.comment:
for line in comment.split('\n'):
config.Append('** ' + line)
config.Append('*/')
# Term verbatim output - this will skip over normal term creation
# code. Warning generated from policy.py if appropriate.
if self.term.verbatim:
for next_term in self.term.verbatim:
if next_term.value[0] == self._PLATFORM:
config.Append(str(next_term.value[1]), verbatim=True)
return str(config)
# Helper for per-address-family keywords.
family_keywords = self._TERM_TYPE.get(self.term_type)
# option
# this is going to be a little ugly b/c there are a few little messed
# up options we can deal with.
if self.term.option:
for opt in [str(x) for x in self.term.option]:
# there should be a better way to search the array of protocols
if opt.startswith('sample'):
self.extra_actions.append('sample')
# only append tcp-established for option established when
# tcp is the only protocol, otherwise other protos break on juniper
elif opt.startswith('established'):
if self.term.protocol == ['tcp']:
if 'tcp-established;' not in from_str:
from_str.append(family_keywords['tcp-est'] + ';')
# if tcp-established specified, but more than just tcp is included
# in the protocols, raise an error
elif opt.startswith('tcp-established'):
flag = family_keywords['tcp-est'] + ';'
if self.term.protocol == ['tcp']:
if flag not in from_str:
from_str.append(flag)
else:
raise TcpEstablishedWithNonTcp(
'tcp-established can only be used with tcp protocol in term %s'
% self.term.name)
elif opt.startswith('rst'):
from_str.append('tcp-flags "rst";')
elif opt.startswith('initial') and 'tcp' in self.term.protocol:
from_str.append('tcp-initial;')
elif opt.startswith('first-fragment'):
from_str.append('first-fragment;')
# we don't have a special way of dealing with this, so we output it and
# hope the user knows what they're doing.
else:
from_str.append('%s;' % opt)
# term name
config.Append('term %s {' % self.term.name)
# a default action term doesn't have any from { clause
has_match_criteria = (self.term.address or
self.term.dscp_except or
self.term.dscp_match or
self.term.destination_address or
self.term.destination_port or
self.term.destination_prefix or
self.term.destination_prefix_except or
self.term.ether_type or
self.term.flexible_match_range or
self.term.forwarding_class or
self.term.forwarding_class_except or
self.term.fragment_offset or
self.term.hop_limit or
self.term.next_ip or
self.term.port or
self.term.precedence or
self.term.protocol or
self.term.protocol_except or
self.term.source_address or
self.term.source_port or
self.term.source_prefix or
self.term.source_prefix_except or
self.term.traffic_type or
self.term.ttl)
if has_match_criteria:
config.Append('from {')
term_af = self.AF_MAP.get(self.term_type)
# address
address = self.term.GetAddressOfVersion('address', term_af)
if self.enable_dsmo:
address = summarizer.Summarize(address)
if address:
config.Append('%s {' % family_keywords['addr'])
for addr in address:
if self.enable_dsmo:
config.Append('%s/%s;' % summarizer.ToDottedQuad(addr,
nondsm=True))
else:
for comment in self._Comment(addr):
config.Append('%s' % comment)
config.Append('%s;' % addr)
config.Append('}')
elif self.term.address:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
af=self.term_type))
return ''
# source address
src_addr = self.term.GetAddressOfVersion('source_address', term_af)
src_addr_ex = self.term.GetAddressOfVersion('source_address_exclude',
term_af)
if self.enable_dsmo:
src_addr = summarizer.Summarize(src_addr)
src_addr_ex = summarizer.Summarize(src_addr_ex)
else:
src_addr, src_addr_ex = self._MinimizePrefixes(src_addr, src_addr_ex)
if src_addr:
config.Append('%s {' % family_keywords['saddr'])
for addr in src_addr:
if self.enable_dsmo:
config.Append('%s/%s;' % summarizer.ToDottedQuad(addr,
nondsm=True))
else:
for comment in self._Comment(addr):
config.Append('%s' % comment)
config.Append('%s;' % addr)
for addr in src_addr_ex:
if self.enable_dsmo:
config.Append('%s/%s except;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
for comment in self._Comment(addr, exclude=True):
config.Append('%s' % comment)
config.Append('%s except;' % addr)
config.Append('}')
elif self.term.source_address:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.term_type))
return ''
# destination address
dst_addr = self.term.GetAddressOfVersion('destination_address', term_af)
dst_addr_ex = self.term.GetAddressOfVersion('destination_address_exclude',
term_af)
if self.enable_dsmo:
dst_addr = summarizer.Summarize(dst_addr)
dst_addr_ex = summarizer.Summarize(dst_addr_ex)
else:
dst_addr, dst_addr_ex = self._MinimizePrefixes(dst_addr, dst_addr_ex)
if dst_addr:
config.Append('%s {' % family_keywords['daddr'])
for addr in dst_addr:
if self.enable_dsmo:
config.Append('%s/%s;' % summarizer.ToDottedQuad(addr,
nondsm=True))
else:
for comment in self._Comment(addr):
config.Append('%s' % comment)
config.Append('%s;' % addr)
for addr in dst_addr_ex:
if self.enable_dsmo:
config.Append('%s/%s except;' %
summarizer.ToDottedQuad(addr, nondsm=True))
else:
for comment in self._Comment(addr, exclude=True):
config.Append('%s' % comment)
config.Append('%s except;' % addr)
config.Append('}')
elif self.term.destination_address:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.term_type))
return ''
# forwarding-class
if self.term.forwarding_class:
config.Append('forwarding-class %s' % self._Group(
self.term.forwarding_class, lc=False))
# forwarding-class-except
if self.term.forwarding_class_except:
config.Append('forwarding-class-except %s' % self._Group(
self.term.forwarding_class_except, lc=False))
# source prefix <except> list
if self.term.source_prefix or self.term.source_prefix_except:
config.Append('source-prefix-list {')
for pfx in self.term.source_prefix:
config.Append(pfx + ';')
for epfx in self.term.source_prefix_except:
config.Append(epfx + ' except;')
config.Append('}')
# destination prefix <except> list
if self.term.destination_prefix or self.term.destination_prefix_except:
config.Append('destination-prefix-list {')
for pfx in self.term.destination_prefix:
config.Append(pfx + ';')
for epfx in self.term.destination_prefix_except:
config.Append(epfx + ' except;')
config.Append('}')
if self.term.ttl:
config.Append('ttl %s;' % self.term.ttl)
# protocol
if self.term.protocol:
# both are supported on JunOS, but only icmp6 is supported
# on SRX loopback stateless filter
config.Append(family_keywords['protocol'] +
' ' + self._Group(self.term.protocol))
# protocol
if self.term.protocol_except:
# same as above
config.Append(family_keywords['protocol-except'] + ' '
+ self._Group(self.term.protocol_except))
# port
if self.term.port:
config.Append('port %s' % self._Group(self.term.port))
# source port
if self.term.source_port:
config.Append('source-port %s' % self._Group(self.term.source_port))
# destination port
if self.term.destination_port:
config.Append('destination-port %s' %
self._Group(self.term.destination_port))
# append any options beloging in the from {} section
for next_str in from_str:
config.Append(next_str)
# packet length
if self.term.packet_length:
config.Append('packet-length %s;' % self.term.packet_length)
# fragment offset
if self.term.fragment_offset:
config.Append('fragment-offset %s;' % self.term.fragment_offset)
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol, self.term_type)
if icmp_types != ['']:
config.Append('icmp-type %s' % self._Group(icmp_types))
if self.term.icmp_code:
config.Append('icmp-code %s' % self._Group(self.term.icmp_code))
if self.term.ether_type:
config.Append('ether-type %s' %
self._Group(self.term.ether_type))
if self.term.traffic_type:
config.Append('traffic-type %s' %
self._Group(self.term.traffic_type))
if self.term.precedence:
# precedence may be a single integer, or a space separated list
policy_precedences = set()
# precedence values may only be 0 through 7
for precedence in self.term.precedence:
if int(precedence) in range(0, 8):
policy_precedences.add(precedence)
else:
raise PrecedenceError('Precedence value %s is out of bounds in %s' %
(precedence, self.term.name))
config.Append('precedence %s' % self._Group(sorted(policy_precedences)))
# DSCP Match
if self.term.dscp_match:
if self.term_type == 'inet6':
config.Append('traffic-class [ %s ];' % (
' '.join(self.term.dscp_match)))
else:
config.Append('dscp [ %s ];' % ' '.join(self.term.dscp_match))
# DSCP Except
if self.term.dscp_except:
if self.term_type == 'inet6':
config.Append('traffic-class-except [ %s ];' % (
' '.join(self.term.dscp_except)))
else:
config.Append('dscp-except [ %s ];' % ' '.join(self.term.dscp_except))
if self.term.hop_limit:
# Only generate a hop-limit if inet6, inet4 has not hop-limit.
if self.term_type == 'inet6':
config.Append('hop-limit %s;' % (self.term.hop_limit))
# flexible-match
if self.term.flexible_match_range:
config.Append('flexible-match-range {')
for fm_opt in self.term.flexible_match_range:
config.Append('%s %s;' % (fm_opt[0], fm_opt[1]))
config.Append('}')
config.Append('}') # end from { ... }
####
# ACTIONS go below here
####
# If the action is only one line, include it in the same line as "then "
# statement.
# For example, if the action is only accept, it should be:
# "then accept;" rather than:
# "then {
# accept;
# }"
#
unique_actions = set(self.extra_actions)
if not self.term.routing_instance:
unique_actions.update(self.term.action)
if len(unique_actions) <= 1:
for action in [self.term.logging, self.term.routing_instance,
self.term.counter, self.term.policer, self.term.qos,
self.term.loss_priority, self.term.dscp_set,
self.term.next_ip, self.term.traffic_class_count]:
if action:
try:
unique_actions.update(action)
except TypeError:
unique_actions.add(action)
if len(unique_actions) > 1:
break
if len(unique_actions) == 1:
# b/21795531: Juniper device treats a set of IPv4 actions differently
# than any other actions.
# For example, if the term is in IPv4 and the action is only discard,
# it should be:
# "then {
# discard;
# }" rather than:
# "then discard;"
current_action = self.ACTIONS.get(unique_actions.pop(), 'next_ip')
if (self.term_type == 'inet' and
current_action in ['discard', 'reject', 'reject tcp-reset']
) or (self.term_type == 'inet6' and current_action in
['reject', 'reject tcp-reset']):
config.Append('then {')
config.Append('%s;' % current_action)
config.Append('}')
elif current_action == 'next_ip':
self.NextIpCheck(self.term.next_ip, self.term.name)
config.Append('then {')
if self.term.next_ip[0].version == 4:
config.Append('next-ip %s;' % str(self.term.next_ip[0]))
else:
config.Append('next-ip6 %s;' % str(self.term.next_ip[0]))
config.Append('}')
else:
config.Append('then %s;' % current_action)
elif len(unique_actions) > 1:
config.Append('then {')
# logging
if self.term.logging:
for log_target in self.term.logging:
if str(log_target) == 'local':
config.Append('log;')
else:
config.Append('syslog;')
if self.term.routing_instance:
config.Append('routing-instance %s;' % self.term.routing_instance)
if self.term.counter:
config.Append('count %s;' % self.term.counter)
if self.term.traffic_class_count:
config.Append('traffic-class-count %s;' % self.term.traffic_class_count)
oid_length = 128
if self.term.policer:
config.Append('policer %s;' % self.term.policer)
if len(self.term.policer) > oid_length:
logging.warn('WARNING: %s is longer than %d bytes. Due to limitation'
' in JUNOS, OIDs longer than %dB can cause SNMP '
'timeout issues.',
self.term.policer, oid_length, oid_length)
if self.term.qos:
config.Append('forwarding-class %s;' % self.term.qos)
if self.term.loss_priority:
config.Append('loss-priority %s;' % self.term.loss_priority)
if self.term.next_ip:
self.NextIpCheck(self.term.next_ip, self.term.name)
if self.term.next_ip[0].version == 4:
config.Append('next-ip %s;' % str(self.term.next_ip[0]))
else:
config.Append('next-ip6 %s;' % str(self.term.next_ip[0]))
for action in self.extra_actions:
config.Append(action + ';')
# If there is a routing-instance defined, skip reject/accept/etc actions.
if not self.term.routing_instance:
for action in self.term.action:
config.Append(self.ACTIONS.get(action) + ';')
# DSCP SET
if self.term.dscp_set:
if self.term_type == 'inet6':
config.Append('traffic-class %s;' % self.term.dscp_set)
else:
config.Append('dscp %s;' % self.term.dscp_set)
config.Append('}') # end then{...}
config.Append('}') # end term accept-foo-to-bar { ... }
return str(config)
def main(unused_argv):
"""Read an binary input file and output to a C/C++ source file as an array.
Raises:
Error: If an input file is not specified.
"""
input_file = FLAGS.input
if not input_file:
raise Error("No input file specified.")
input_file_base = FLAGS.input.rsplit(".", 1)[0]
output_source = FLAGS.output_source
if not output_source:
output_source = input_file_base + ".cc"
logging.debug("Using default --output_source='%s'", output_source)
output_header = FLAGS.output_header
if not output_header:
output_header = input_file_base + ".h"
logging.debug("Using default --output_header='%s'", output_header)
identifier_base = sub("[^0-9a-zA-Z]+", "_", path.basename(input_file_base))
array_name = FLAGS.array
if not array_name:
array_name = identifier_base + "_data"
logging.debug("Using default --array='%s'", array_name)
array_size_name = FLAGS.array_size
if not array_size_name:
array_size_name = identifier_base + "_size"
logging.debug("Using default --array_size='%s'", array_size_name)
fileid = FLAGS.filename_identifier
if not fileid:
fileid = identifier_base + "_filename"
logging.debug("Using default --filename_identifier='%s'", fileid)
filename = FLAGS.filename
if filename is None: # but not if it's the empty string
filename = path.basename(input_file)
logging.debug("Using default --filename='%s'", filename)
header_guard = FLAGS.header_guard
if not header_guard:
header_guard = sub("[^0-9a-zA-Z]+", "_", "_" + output_header).upper()
logging.debug("Using default --header_guard='%s'", header_guard)
namespace = FLAGS.cpp_namespace
namespaces = namespace.split("::") if namespace else []
with open(input_file, "rb") as infile:
input_bytes = bytearray(infile.read())
logging.debug("Read %d bytes from %s", len(input_bytes), input_file)
header_text = "\n".join(
header(header_guard, namespaces, array_name, array_size_name, fileid))
source_text = "\n".join(
source(namespaces, array_name, array_size_name, fileid, filename,
input_bytes))
with open(output_header, "w") as hdr:
hdr.write(header_text)
logging.debug("Wrote header file %s", output_header)
with open(output_source, "w") as src:
src.write(source_text)
logging.debug("Wrote source file %s", output_source)
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self.platform not in self.term.platform:
return ''
if self.term.platform_exclude:
if self.platform in self.term.platform_exclude:
return ''
ret_str = ['\n']
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.af == 6 and 'icmp' in self.term.protocol) or
(self.af == 4 and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.text_af))
return ''
if self.term_remark:
ret_str.append(' remark ' + self.term.name)
if self.term.owner:
self.term.comment.append('Owner: %s' % self.term.owner)
for comment in self.term.comment:
for line in comment.split('\n'):
ret_str.append(' remark ' + str(line)[:100].rstrip())
# Term verbatim output - this will skip over normal term creation
# code by returning early. Warnings provided in policy.py.
if self.term.verbatim:
for next_verbatim in self.term.verbatim:
if next_verbatim.value[0] == self.platform:
ret_str.append(str(next_verbatim.value[1]))
return '\n'.join(ret_str)
# protocol
if not self.term.protocol:
if self.af == 6:
protocol = ['ipv6']
elif self.platform == 'ciscoxr':
protocol = ['ipv4']
else:
protocol = ['ip']
elif self.term.protocol == ['hopopt']:
protocol = ['hbh']
elif self.proto_int:
protocol = [proto if proto in self.ALLOWED_PROTO_STRINGS
else self.PROTO_MAP.get(proto)
for proto in self.term.protocol]
else:
protocol = self.term.protocol
# source address
if self.term.source_address:
source_address = self.term.GetAddressOfVersion('source_address', self.af)
source_address_exclude = self.term.GetAddressOfVersion(
'source_address_exclude', self.af)
if source_address_exclude:
source_address = nacaddr.ExcludeAddrs(
source_address,
source_address_exclude)
if not source_address:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.text_af))
return ''
if self.enable_dsmo:
source_address = summarizer.Summarize(source_address)
else:
# source address not set
source_address = ['any']
# destination address
if self.term.destination_address:
destination_address = self.term.GetAddressOfVersion(
'destination_address', self.af)
destination_address_exclude = self.term.GetAddressOfVersion(
'destination_address_exclude', self.af)
if destination_address_exclude:
destination_address = nacaddr.ExcludeAddrs(
destination_address,
destination_address_exclude)
if not destination_address:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.text_af))
return ''
if self.enable_dsmo:
destination_address = summarizer.Summarize(destination_address)
else:
# destination address not set
destination_address = ['any']
# options
opts = [str(x) for x in self.term.option]
if ((self.PROTO_MAP['tcp'] in protocol or 'tcp' in protocol)
and ('tcp-established' in opts or 'established' in opts)):
if 'established' not in self.options:
self.options.append('established')
# ports
source_port = [()]
destination_port = [()]
if self.term.source_port:
source_port = self._FixConsecutivePorts(self.term.source_port)
if self.term.destination_port:
destination_port = self._FixConsecutivePorts(self.term.destination_port)
# logging
if self.term.logging:
self.options.append('log')
# dscp; unlike srx, cisco only supports single, non-except values
if self.term.dscp_match:
if len(self.term.dscp_match) > 1:
raise ExtendedACLTermError(
'Extended ACLs cannot specify more than one dscp match value')
else:
self.options.append('dscp %s' % ' '.join(self.term.dscp_match))
# icmp-types
icmp_types = ['']
if self.term.icmp_type:
icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type,
self.term.protocol, self.af)
icmp_codes = ['']
if self.term.icmp_code:
icmp_codes = self.term.icmp_code
fixed_src_addresses = [self._GetIpString(x) for x in source_address]
fixed_dst_addresses = [self._GetIpString(x) for x in destination_address]
fixed_opts = {}
for p in protocol:
fixed_opts[p] = self._FixOptions(p, self.options)
for saddr in fixed_src_addresses:
for daddr in fixed_dst_addresses:
for sport in source_port:
for dport in destination_port:
for proto in protocol:
opts = fixed_opts[proto]
for icmp_type in icmp_types:
for icmp_code in icmp_codes:
ret_str.extend(self._TermletToStr(
_ACTION_TABLE.get(str(self.term.action[0])),
proto,
saddr,
self._FormatPort(sport, proto),
daddr,
self._FormatPort(dport, proto),
icmp_type,
icmp_code,
opts))
return '\n'.join(ret_str)
def __str__(self):
"""Render config output from this term object."""
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
ret_str = []
self._SetDefaultAction()
# Create a new term
ret_str.append('\n# term %s' % self.term.name)
comments = aclgenerator.WrapWords(self.term.comment, 80)
# append comments to output
if comments and comments[0]:
for line in comments:
ret_str.append('# %s' % str(line))
if str(self.term.action[0]) not in self._ACTION_TABLE:
raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % (
'\n', self.term.name, self.term.action[0],
'action not currently supported.'))
if self.direction and str(self.direction) not in self._DIRECTION_TABLE:
raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % (
'\n', self.term.name, self.term.direction,
'direction not currently supported.'))
# protocol
if self.term.protocol:
protocol = self.term.protocol
else:
protocol = []
# source address
term_saddrs = self._CheckAddressAf(self.term.source_address)
if not term_saddrs:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='source',
af=self.af))
return ''
term_saddr = self._GenerateAddrStatement(
term_saddrs, self.term.source_address_exclude)
# destination address
term_daddrs = self._CheckAddressAf(self.term.destination_address)
if not term_daddrs:
logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name,
direction='destination',
af=self.af))
return ''
term_daddr = self._GenerateAddrStatement(
term_daddrs, self.term.destination_address_exclude)
# ports
source_port = []
destination_port = []
if self.term.source_port:
source_port = self._GeneratePortStatement(self.term.source_port)
if self.term.destination_port:
destination_port = self._GeneratePortStatement(self.term.destination_port)
# icmp-type
icmp_types = ['']
if self.term.icmp_type:
if self.af != 'mixed':
af = self.af
elif protocol == ['icmp']:
af = 'inet'
elif protocol == ['icmpv6']:
af = 'inet6'
else:
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'icmp protocol is not defined or not supported.'))
icmp_types = self.NormalizeIcmpTypes(
self.term.icmp_type, protocol, af)
# options
tcp_flags_set = []
tcp_flags_check = []
for next_opt in [str(x) for x in self.term.option]:
for next_flag in self._TCP_FLAGS_TABLE:
if next_opt.find(next_flag) == 0:
if protocol != ['tcp']:
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'tcp flags may only be specified with tcp protocol.'))
tcp_flags_set.append(self._TCP_FLAGS_TABLE.get(next_flag))
tcp_flags_check.append(self._TCP_FLAGS_TABLE.get(next_flag))
# If tcp-established is set, override any of the flags above with the
# S/SA flags. Issue an error if flags are specified with 'established'.
for opt in [str(x) for x in self.term.option]:
if opt == 'established' or opt == 'tcp-established':
if tcp_flags_set or tcp_flags_check:
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'tcp flags may not be specified with tcp-established.'))
# We need to set 'flags A/A' for established regardless of whether or
# not we're stateful:
# - if we stateful, the default is 'flags S/SA' which prevent writing
# rules for reply packets.
# - if we're stateless, this is the only way to do it.
if not protocol or 'tcp' in protocol:
tcp_flags_set.append(self._TCP_FLAGS_TABLE.get('ack'))
tcp_flags_check.append(self._TCP_FLAGS_TABLE.get('ack'))
# The default behavior of pf is 'keep state flags S/SA'. If we're not
# stateless, and if flags have not been specified explicitly via options,
# append that here. Note that pf allows appending flags for udp and icmp;
# they are just ignored, as long as TCP is in the proto. This lets you
# doing things like 'proto { tcp udp icmp } flags S/SA' and have the flags
# only applied to the tcp bits that match. However, the policy description
# language prohibits setting flags on non-TCP, since it doesn't make sense
# on all platforms.
if ((not protocol or protocol == ['tcp']) and self.stateful
and not tcp_flags_set and not tcp_flags_check):
tcp_flags_set.append(self._TCP_FLAGS_TABLE.get('syn'))
tcp_flags_check.append(self._TCP_FLAGS_TABLE.get('syn'))
tcp_flags_check.append(self._TCP_FLAGS_TABLE.get('ack'))
ret_str.extend(self._FormatPart(
self.term.action[0],
self.direction,
self.term.logging,
self.af,
protocol,
term_saddr,
source_port,
term_daddr,
destination_port,
tcp_flags_set,
tcp_flags_check,
icmp_types,
self.options,
self.stateful,))
return '\n'.join(str(v) for v in ret_str if v is not '')
def close(self) -> None:
"""Closes all the sensors."""
for name, sensor in self.sensors.items():
logging.debug("Destroys {} sensor".format(name))
sensor.close()
def __str__(self):
# Verify platform specific terms. Skip whole term if platform does not
# match.
if self.term.platform:
if self._PLATFORM not in self.term.platform:
return ''
if self.term.platform_exclude:
if self._PLATFORM in self.term.platform_exclude:
return ''
ret_str = []
# Don't render icmpv6 protocol terms under inet, or icmp under inet6
if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or
(self.af == 'inet' and 'icmpv6' in self.term.protocol)):
logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name,
proto=self.term.protocol,
af=self.af))
return ''
# append comments to output
ret_str.append(self._COMMENT_FORMAT.substitute(filter=self.filter,
term=self.term_name,
comment=self.term.comment))
# if terms does not specify action, use filter default action
if not self.term.action:
self.term.action[0].value = self.default_action
if self.term.action[0] == 'next':
return ''
if len(self.term.action) > 1:
raise aclgenerator.UnsupportedFilterError('\n%s %s %s %s' % (
'Multiple actions unsupported by', self._PLATFORM,
'\nError in term:', self.term.name))
# protocol
if self.term.protocol:
protocols = self.term.protocol
else:
protocols = ['any']
# addresses
src_addr = self.term.source_address
if not src_addr:
src_addr = [self._all_ips]
dst_addr = self.term.destination_address
if not dst_addr:
dst_addr = [self._all_ips]
if (self.term.source_address_exclude or
self.term.destination_address_exclude):
raise aclgenerator.UnsupportedFilterError('\n%s %s %s %s' % (
'address exclusions unsupported by', self._PLATFORM,
'\nError in term:', self.term.name))
# ports = Map the ports in a straight list since multiports aren't supported
(src_ports, dst_ports) = self._HandlePorts(self.term.source_port,
self.term.destination_port)
# The windows ipsec driver requires either 'tcp' or 'udp' to be specified
# if a srcport or dstport is specified. Fail if src or dst ports are
# specified and of the protocols are not exactly one or both of 'tcp'
# or 'udp'.
if ((not set(protocols).issubset(set(['tcp', 'udp']))) and
(len(src_ports) > 1 or len(dst_ports) > 1)):
raise aclgenerator.UnsupportedFilterError('%s %s %s' % (
'\n', self.term.name,
'src or dst ports may only be specified with "tcp" and/or "udp".'))
# icmp-types
(icmp_types, protocols) = self._HandleIcmpTypes(self.term.icmp_type,
protocols)
ret_str = []
self._HandlePreRule(ret_str)
self._CartesianProduct(src_addr, dst_addr, protocols, icmp_types, src_ports,
dst_ports, ret_str)
self._HandlePreRule(ret_str)
return '\n'.join(str(v) for v in ret_str if v is not '')
def log_msg(prefix, msg):
logging.debug("%s: len: %s, hash: %s, msg: 0x%s", prefix, len(msg),
hashlib.md5(msg).hexdigest()[:6], binascii.hexlify(msg[:25]))
def _sample_positive_patches(image,
abnormality_filepath,
abnormality_mask,
abnormality_area,
patch_size,
number_of_patches=10,
min_overlap_threshold=0.90,
max_number_of_trials_per_threshold=100):
"""Sample random patches from the image overlapping with the given abnormality.
The abnormal area of the patch with respect to either (a) the total area of
the patch, or (b) the total area of the abnormality, must be at least
`min_overlap_threshold` (i.e. 90% by default).
After `max_number_of_trials_per_threshold` samples, if not enough patches
meeting this requirement have been generated, the `min_overlap_threshold` is
reduced by 5%. This procedure is repeated until min_overlap_threshold < 0.1
(which should not happen ever, if the dataset is correct).
Args:
image: Image to patch from.
abnormality_filepath: Only used for logging.
abnormality_mask: Binary mask of the abnormality in the image.
abnormality_area: Precomputed area of the abnormality.
patch_size: Size of the patch to extract.
number_of_patches: Number of patches to sample around the abnormality ROI.
min_overlap_threshold: Minimum relative area of the patch overlapping with
the abnormality.
max_number_of_trials_per_threshold: Maximum number of random samples to try
before reducing the `min_overlap_threshold` by 5%.
Yields:
The patch cropped from the input image.
"""
cv2 = tfds.core.lazy_imports.cv2
# The paper trying to be reproduced states that 90% of the are of each
# positive patch should correspond to abnormal tissue. Thus if the total area
# of abnormality is smaller than 0.9 * patch_area, we are certain that no
# patch can meet this requirement. This happens indeed quite often.
#
# However, in a piece of code release by the authors of the paper
# (https://github.com/yuyuyu123456/CBIS-DDSM/blob/bf3abc6ac2890b9b51eb5125e00056e39295fa44/ddsm_train/sample_patches_combined.py#L26)
# the authors accept a patch if the total area of abnormality in the patch is
# greater than 75% OR if 75% of the total abnormal area is in the patch.
# In addition, they reduce the overlapping threholds every 1000 trials to
# handle some corner casses.
abnormality_roi = _get_roi_from_mask(abnormality_mask)
abnorm_x, abnorm_y, abnorm_w, abnorm_h = cv2.boundingRect(abnormality_roi)
number_of_yielded_patches = 0
while min_overlap_threshold > 0.1:
# Determine the region where random samples should be sampled from.
max_h, min_h = max(abnorm_h,
patch_size[0]), min(abnorm_h, patch_size[0])
max_w, min_w = max(abnorm_w,
patch_size[1]), min(abnorm_w, patch_size[1])
min_y = abnorm_y - int((1.0 - min_overlap_threshold) * max_h)
min_x = abnorm_x - int((1.0 - min_overlap_threshold) * max_w)
max_y = abnorm_y + abnorm_h - int(min_overlap_threshold * min_h)
max_x = abnorm_x + abnorm_w - int(min_overlap_threshold * min_w)
# Ensure that all sampled batches are within the image.
min_y = max(min_y, 0)
min_x = max(min_x, 0)
max_y = max(min(max_y, image.shape[0] - patch_size[0] - 1), min_y)
max_x = max(min(max_x, image.shape[1] - patch_size[1] - 1), min_x)
# Cap the number of trials if the sampling region is too small.
effective_range_size = max_number_of_trials_per_threshold
if (max_y - min_y + 1) * (max_x - min_x + 1) < effective_range_size:
logging.debug(
'The sampling region for patches of size %r with '
'min_overlap_threshold=%f contains less possible patches than '
'max_number_of_trials_per_threshold=%d, in abnormality %s',
patch_size, min_overlap_threshold,
max_number_of_trials_per_threshold, abnormality_filepath)
effective_range_size = (max_y - min_y + 1) * (max_x - min_x + 1)
for _ in range(effective_range_size):
patch_y = np.random.randint(min_y, max_y + 1)
patch_x = np.random.randint(min_x, max_x + 1)
if _patch_overlaps_any_abnormality_above_threshold(
patch_y, patch_x, patch_size, [abnormality_mask],
[abnormality_area], min_overlap_threshold):
number_of_yielded_patches += 1
yield image[patch_y:(patch_y + patch_size[0]),
patch_x:(patch_x + patch_size[1])]
# If we have yielded all requested patches return.
if number_of_yielded_patches >= number_of_patches:
return
# We failed to produce patches with the minimum overlapping requirements.
# Reduce those requirements and try again.
min_overlap_threshold = min_overlap_threshold * 0.95
logging.debug(
'Overlapping constraints relaxed to min_overlap_threshold=%f while '
'sampling positive patches for the abnormality %s',
min_overlap_threshold, abnormality_filepath)
# This should not happen ever.
raise ValueError(
'Only %d positive patches of size %r could be sampled satisfying the '
'current conditions (min. relative overlapping area = %f) for the '
'abnormality %s' % (number_of_yielded_patches, patch_size,
min_overlap_threshold, abnormality_filepath))
def RenderFile(input_file, output_directory, definitions,
exp_info, write_files):
"""Render a single file.
Args:
input_file: the name of the input policy file.
output_directory: the directory in which we place the rendered file.
definitions: the definitions from naming.Naming().
exp_info: print a info message when a term is set to expire
in that many weeks.
write_files: a list of file tuples, (output_file, acl_text), to write
"""
logging.debug('rendering file: %s into %s', input_file,
output_directory)
pol = None
jcl = False
acl = False
asacl = False
aacl = False
bacl = False
eacl = False
gcefw = False
ips = False
ipt = False
spd = False
nsx = False
pcap_accept = False
pcap_deny = False
pf = False
srx = False
jsl = False
nft = False
win_afw = False
xacl = False
paloalto = False
try:
conf = open(input_file).read()
logging.debug('opened and read %s', input_file)
except IOError as e:
logging.warn('bad file: \n%s', e)
raise
try:
pol = policy.ParsePolicy(
conf, definitions, optimize=FLAGS.optimize,
base_dir=FLAGS.base_directory, shade_check=FLAGS.shade_check)
except policy.ShadingError as e:
logging.warn('shading errors for %s:\n%s', input_file, e)
return
except (policy.Error, naming.Error):
raise ACLParserError('Error parsing policy file %s:\n%s%s' % (
input_file, sys.exc_info()[0], sys.exc_info()[1]))
platforms = set()
for header in pol.headers:
platforms.update(header.platforms)
if 'juniper' in platforms:
jcl = copy.deepcopy(pol)
if 'cisco' in platforms:
acl = copy.deepcopy(pol)
if 'ciscoasa' in platforms:
asacl = copy.deepcopy(pol)
if 'brocade' in platforms:
bacl = copy.deepcopy(pol)
if 'arista' in platforms:
eacl = copy.deepcopy(pol)
if 'aruba' in platforms:
aacl = copy.deepcopy(pol)
if 'ipset' in platforms:
ips = copy.deepcopy(pol)
if 'iptables' in platforms:
ipt = copy.deepcopy(pol)
if 'nsxv' in platforms:
nsx = copy.deepcopy(pol)
if 'packetfilter' in platforms:
pf = copy.deepcopy(pol)
if 'pcap' in platforms:
pcap_accept = copy.deepcopy(pol)
pcap_deny = copy.deepcopy(pol)
if 'speedway' in platforms:
spd = copy.deepcopy(pol)
if 'srx' in platforms:
srx = copy.deepcopy(pol)
if 'srxlo' in platforms:
jsl = copy.deepcopy(pol)
if 'windows_advfirewall' in platforms:
win_afw = copy.deepcopy(pol)
if 'ciscoxr' in platforms:
xacl = copy.deepcopy(pol)
if 'nftables' in platforms:
nft = copy.deepcopy(pol)
if 'gce' in platforms:
gcefw = copy.deepcopy(pol)
if 'paloalto' in platforms:
paloalto = copy.deepcopy(pol)
if not output_directory.endswith('/'):
output_directory += '/'
try:
if jcl:
acl_obj = juniper.Juniper(jcl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if srx:
acl_obj = junipersrx.JuniperSRX(srx, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if acl:
acl_obj = cisco.Cisco(acl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if asacl:
acl_obj = ciscoasa.CiscoASA(asacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if aacl:
acl_obj = aruba.Aruba(aacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if bacl:
acl_obj = brocade.Brocade(bacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if eacl:
acl_obj = arista.Arista(eacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if ips:
acl_obj = ipset.Ipset(ips, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if ipt:
acl_obj = iptables.Iptables(ipt, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if nsx:
acl_obj = nsxv.Nsxv(nsx, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if spd:
acl_obj = speedway.Speedway(spd, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pcap_accept:
acl_obj = pcap.PcapFilter(pcap_accept, exp_info)
RenderACL(str(acl_obj), '-accept' + acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pcap_deny:
acl_obj = pcap.PcapFilter(pcap_deny, exp_info, invert=True)
RenderACL(str(acl_obj), '-deny' + acl_obj.SUFFIX, output_directory,
input_file, write_files)
if pf:
acl_obj = packetfilter.PacketFilter(pf, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if win_afw:
acl_obj = windows_advfirewall.WindowsAdvFirewall(win_afw, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if jsl:
acl_obj = srxlo.SRXlo(jsl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if xacl:
acl_obj = ciscoxr.CiscoXR(xacl, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if nft:
acl_obj = nftables.Nftables(nft, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if gcefw:
acl_obj = gce.GCE(gcefw, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
if paloalto:
acl_obj = paloaltofw.PaloAltoFW(paloalto, exp_info)
RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory,
input_file, write_files)
# TODO(robankeny) add additional errors.
except (juniper.Error, junipersrx.Error, cisco.Error, ipset.Error,
iptables.Error, speedway.Error, pcap.Error,
aclgenerator.Error, aruba.Error, nftables.Error, gce.Error) as e:
raise ACLGeneratorError(
'Error generating target ACL for %s:\n%s' % (input_file, e))