def get_vars_to_restore(ckpt=None): """Returns list of variables that should be saved/restored. Args: ckpt: Path to existing checkpoint. If present, returns only the subset of variables that exist in given checkpoint. Returns: List of all variables that need to be saved/restored. """ model_vars = tf.trainable_variables() # Add batchnorm variables. bn_vars = [v for v in tf.global_variables() if 'moving_mean' in v.op.name or 'moving_variance' in v.op.name] model_vars.extend(bn_vars) model_vars = sorted(model_vars, key=lambda x: x.op.name) if ckpt is not None: ckpt_var_names = tf.contrib.framework.list_variables(ckpt) ckpt_var_names = [name for (name, unused_shape) in ckpt_var_names] for v in model_vars: if v.op.name not in ckpt_var_names: logging.warn('Missing var %s in checkpoint: %s', v.op.name, os.path.basename(ckpt)) model_vars = [v for v in model_vars if v.op.name in ckpt_var_names] return model_vars
def _WriteFile(output_file, file_string): try: with open(output_file, 'w') as output: logging.info('writing file: %s', output_file) output.write(file_string) except IOError: logging.warn('error while writing file: %s', output_file) raise
def _TranslatePolicy(self, pol, exp_info): self.juniper_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) filter_name = header.FilterName(self._PLATFORM) # Check for the position independent options and remove them from # the list. interface_specific = 'not-interface-specific' not in filter_options[1:] enable_dsmo = 'enable_dsmo' in filter_options[1:] noverbose = 'noverbose' in filter_options[1:] if not interface_specific: filter_options.remove('not-interface-specific') if enable_dsmo: filter_options.remove('enable_dsmo') # default to inet4 filters filter_type = 'inet' if len(filter_options) > 1: filter_type = filter_options[1] term_names = set() new_terms = [] for term in terms: term.name = self.FixTermLength(term.name) if term.name in term_names: raise JuniperDuplicateTermError('You have multiple terms named: %s' % term.name) term_names.add(term.name) term = self.FixHighPorts(term, af=filter_type) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue new_terms.append(self._TERM(term, filter_type, enable_dsmo, noverbose)) self.juniper_policies.append((header, filter_name, filter_type, interface_specific, new_terms))
def get_vars_to_save_and_restore(ckpt=None): """Returns list of variables that should be saved/restored. Args: ckpt: Path to existing checkpoint. If present, returns only the subset of variables that exist in given checkpoint. Returns: List of all variables that need to be saved/restored. """ model_vars = tf.trainable_variables() # Add batchnorm variables. bn_vars = [v for v in tf.global_variables() if 'moving_mean' in v.op.name or 'moving_variance' in v.op.name or 'mu' in v.op.name or 'sigma' in v.op.name or 'global_scale_var' in v.op.name] model_vars.extend(bn_vars) model_vars = sorted(model_vars, key=lambda x: x.op.name) mapping = {} if ckpt is not None: ckpt_var = tf.contrib.framework.list_variables(ckpt) ckpt_var_names = [name for (name, unused_shape) in ckpt_var] ckpt_var_shapes = [shape for (unused_name, shape) in ckpt_var] not_loaded = list(ckpt_var_names) for v in model_vars: if v.op.name not in ckpt_var_names: # For backward compatibility, try additional matching. v_additional_name = v.op.name.replace('egomotion_prediction/', '') if v_additional_name in ckpt_var_names: # Check if shapes match. ind = ckpt_var_names.index(v_additional_name) if ckpt_var_shapes[ind] == v.get_shape(): mapping[v_additional_name] = v not_loaded.remove(v_additional_name) continue else: logging.warn('Shape mismatch, will not restore %s.', v.op.name) logging.warn('Did not find var %s in checkpoint: %s', v.op.name, os.path.basename(ckpt)) else: # Check if shapes match. ind = ckpt_var_names.index(v.op.name) if ckpt_var_shapes[ind] == v.get_shape(): mapping[v.op.name] = v not_loaded.remove(v.op.name) else: logging.warn('Shape mismatch, will not restore %s.', v.op.name) if not_loaded: logging.warn('The following variables in the checkpoint were not loaded:') for varname_not_loaded in not_loaded: logging.info('%s', varname_not_loaded) else: # just get model vars. for v in model_vars: mapping[v.op.name] = v return mapping
def DescendRecursively(input_dirname, output_dirname, definitions, depth=1): """Recursively descend from input_dirname looking for policy files to render. Args: input_dirname: the base directory. output_dirname: where to place the rendered files. definitions: naming.Naming object. depth: integer, for outputting '---> rendering prod/corp-backbone.jcl'. Returns: the files that were found """ # p4 complains if you try to edit a file like ./corp//corp-isp.jcl input_dirname = input_dirname.rstrip('/') output_dirname = output_dirname.rstrip('/') files = [] # calling all directories for curdir in [x for x in dircache.listdir(input_dirname) if os.path.isdir(input_dirname + '/' + x)]: # be on the lookout for a policy directory if curdir == 'pol': for input_file in [x for x in dircache.listdir(input_dirname + '/pol') if x.endswith('.pol')]: files.append({'in_file': os.path.join(input_dirname, 'pol', input_file), 'out_dir': output_dirname, 'defs': definitions}) else: # so we don't have a policy directory, we should check if this new # directory has a policy directory if curdir in FLAGS.ignore_directories: continue logging.warn('-' * (2 * depth) + '> %s' % ( input_dirname + '/' + curdir)) files_found = DescendRecursively(input_dirname + '/' + curdir, output_dirname + '/' + curdir, definitions, depth + 1) logging.warn('-' * (2 * depth) + '> %s (%d pol files found)' % ( input_dirname + '/' + curdir, len(files_found))) files.extend(files_found) return files
def __str__(self): """Render the output of the Nsxv policy.""" target_header = [] target = [] # add the p4 tags target.append('<!--') target.extend(aclgenerator.AddRepositoryTags('\n')) target.append('\n') target.append('-->') for (_, _, _, terms) in self.nsxv_policies: section_name = self._FILTER_OPTIONS_DICT['section_name'] # check section id value section_id = self._FILTER_OPTIONS_DICT['section_id'] if not section_id or section_id == 0: logging.warn('WARNING: Section-id is 0. A new Section is created for%s.' ' If there is any existing section, it will remain ' 'unreferenced and should be removed manually.', section_name) target.append('<section name="%s">' % (section_name.strip(' \t\n\r'))) else: target.append('<section id="%s" name="%s">' % (section_id, section_name.strip(' \t\n\r'))) # now add the terms for term in terms: term_str = str(term) if term_str: target.append(term_str) # ensure that the header is always first target = target_header + target target.append('%s' % (_XML_TABLE.get('sectionEnd'))) target.append('\n') target_as_xml = xml.dom.minidom.parseString(''.join(target)) return target_as_xml.toprettyxml(indent=' ', encoding='UTF-8')
def __init__(self, term, filter_name, platform='cisco'): self.term = term self.filter_name = filter_name self.platform = platform self.options = [] self.logstring = '' self.dscpstring = '' # sanity checking for standard acls if self.term.protocol: raise StandardAclTermError( 'Standard ACLs cannot specify protocols') if self.term.icmp_type: raise StandardAclTermError( 'ICMP Type specifications are not permissible in standard ACLs') if (self.term.source_address or self.term.source_address_exclude or self.term.destination_address or self.term.destination_address_exclude): raise StandardAclTermError( 'Standard ACLs cannot use source or destination addresses') if self.term.option: raise StandardAclTermError( 'Standard ACLs prohibit use of options') if self.term.source_port or self.term.destination_port: raise StandardAclTermError( 'Standard ACLs prohibit use of port numbers') if self.term.logging: logging.warn( 'WARNING: Standard ACL logging is set in filter %s, term %s and ' 'may not implemented on all IOS versions', self.filter_name, self.term.name) self.logstring = ' log' if self.term.dscp_match: logging.warn( 'WARNING: dscp-match is set in filter %s, term %s and may not be ' 'implemented on all IOS version', self.filter_name, self.term_name) self.dscpstring = ' dscp' + self.term_dscp_match
def __str__(self): """Convert term to a rule string. Returns: A rule as a string. Raises: NsxvAclTermError: When unknown icmp-types are specified """ # Verify platform specific terms. Skip whole term if platform does not # match. if self.term.platform: if 'nsxv' not in self.term.platform: return '' if self.term.platform_exclude: if 'nsxv' in self.term.platform_exclude: return '' ret_str = [''] # Don't render icmpv6 protocol terms under inet, or icmp under inet6 if ((self.af == 6 and 'icmp' in self.term.protocol) or (self.af == 4 and 'icmpv6' in self.term.protocol)): logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name, proto=self.term.protocol, af=self.filter_type)) return '' # Term verbatim is not supported if self.term.verbatim: raise NsxvAclTermError( 'Verbatim are not implemented in standard ACLs') # Term option is not supported if self.term.option: for opt in [str(single_option) for single_option in self.term.option]: if((opt.find('tcp-established') == 0) or (opt.find('established') == 0)): return '' else: raise NsxvAclTermError( 'Option are not implemented in standard ACLs') # check for keywords Nsxv does not support term_keywords = self.term.__dict__ unsupported_keywords = [] for key in term_keywords: if term_keywords[key]: # translated is obj attribute not keyword if ('translated' not in key) and (key not in _NSXV_SUPPORTED_KEYWORDS): unsupported_keywords.append(key) if unsupported_keywords: logging.warn('WARNING: The keywords %s in Term %s are not supported in ' 'Nsxv ', unsupported_keywords, self.term.name) name = '%s%s%s' % (_XML_TABLE.get('nameStart'), self.term.name, _XML_TABLE.get('nameEnd')) notes = '' if self.term.comment: for comment in self.term.comment: notes = '%s%s' %(notes, comment) notes = '%s%s%s' % (_XML_TABLE.get('noteStart'), notes, _XML_TABLE.get('noteEnd')) # protocol protocol = None if self.term.protocol: protocol = map(self.PROTO_MAP.get, self.term.protocol, self.term.protocol) # icmp-types icmp_types = [''] if self.term.icmp_type: icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, self.term.protocol, self.af) # for mixed filter type get both IPV4address and IPv6Address af_list = [] if self.filter_type == 'mixed': af_list = [4, 6] else: af_list = [self.af] source_address = None destination_address = None source_addr = [] destination_addr = [] source_v4_addr = [] source_v6_addr = [] dest_v4_addr = [] dest_v6_addr = [] for af in af_list: # source address if self.term.source_address: source_address = self.term.GetAddressOfVersion('source_address', af) source_address_exclude = self.term.GetAddressOfVersion( 'source_address_exclude', af) if source_address_exclude: source_address = nacaddr.ExcludeAddrs( source_address, source_address_exclude) if source_address: if af == 4: source_v4_addr = source_address else: source_v6_addr = source_address source_addr = source_v4_addr + source_v6_addr # destination address if self.term.destination_address: destination_address = self.term.GetAddressOfVersion( 'destination_address', af) destination_address_exclude = self.term.GetAddressOfVersion( 'destination_address_exclude', af) if destination_address_exclude: destination_address = nacaddr.ExcludeAddrs( destination_address, destination_address_exclude) if destination_address: if af == 4: dest_v4_addr = destination_address else: dest_v6_addr = destination_address destination_addr = dest_v4_addr + dest_v6_addr # Check for mismatch IP for source and destination address for mixed filter if self.filter_type == 'mixed': if source_addr and destination_addr: if source_v4_addr and not dest_v4_addr: source_addr = source_v6_addr elif source_v6_addr and not dest_v6_addr: source_addr = source_v4_addr elif dest_v4_addr and not source_v4_addr: destination_addr = dest_v6_addr elif dest_v6_addr and not source_v6_addr: destination_addr = dest_v4_addr if not source_addr or not destination_addr: logging.warn('Term %s will not be rendered as it has IPv4/IPv6 ' 'mismatch for source/destination for mixed address ' 'family.', self.term.name) return '' # ports source_port = None destination_port = None if self.term.source_port: source_port = self.term.source_port if self.term.destination_port: destination_port = self.term.destination_port # logging log = 'false' if self.term.logging: log = 'true' sources = '' if source_addr: sources = '<sources excluded="false">' for saddr in source_addr: # inet4 if isinstance(saddr, nacaddr.IPv4): if saddr.numhosts > 1: saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv4Start'), saddr.with_prefixlen, _XML_TABLE.get('srcIpv4End'),) else: saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv4Start'), saddr.ip, _XML_TABLE.get('srcIpv4End')) sources = '%s%s' %(sources, saddr) # inet6 if isinstance(saddr, nacaddr.IPv6): if saddr.numhosts > 1: saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv6Start'), saddr.with_prefixlen, _XML_TABLE.get('srcIpv6End'),) else: saddr = '%s%s%s' % (_XML_TABLE.get('srcIpv6Start'), saddr.ip, _XML_TABLE.get('srcIpv6End')) sources = '%s%s' %(sources, saddr) sources = '%s%s' %(sources, '</sources>') destinations = '' if destination_addr: destinations = '<destinations excluded="false">' for daddr in destination_addr: # inet4 if isinstance(daddr, nacaddr.IPv4): if daddr.numhosts > 1: daddr = '%s%s%s' % (_XML_TABLE.get('destIpv4Start'), daddr.with_prefixlen, _XML_TABLE.get('destIpv4End'),) else: daddr = '%s%s%s' % (_XML_TABLE.get('destIpv4Start'), daddr.ip, _XML_TABLE.get('destIpv4End')) destinations = '%s%s' %(destinations, daddr) # inet6 if isinstance(daddr, nacaddr.IPv6): if daddr.numhosts > 1: daddr = '%s%s%s' % (_XML_TABLE.get('destIpv6Start'), daddr.with_prefixlen, _XML_TABLE.get('destIpv6End'),) else: daddr = '%s%s%s' % (_XML_TABLE.get('destIpv6Start'), daddr.ip, _XML_TABLE.get('destIpv6End')) destinations = '%s%s' %(destinations, daddr) destinations = '%s%s' %(destinations, '</destinations>') services = [] if protocol: services.append('<services>') for proto in protocol: if proto != 'any': services.append(self._ServiceToString(proto, source_port, destination_port, icmp_types)) services.append('</services>') service = '' for s in services: service = '%s%s' % (service, s) # applied_to applied_to_list = '' if self.applied_to: applied_to_list = '<appliedToList>' applied_to_element = '%s%s%s' % (_XML_TABLE.get('appliedToStart'), self.applied_to, _XML_TABLE.get('appliedToEnd')) applied_to_list = '%s%s' %(applied_to_list, applied_to_element) applied_to_list = '%s%s' %(applied_to_list, '</appliedToList>') # action action = '%s%s%s' % (_XML_TABLE.get('actionStart'), _ACTION_TABLE.get(str(self.term.action[0])), _XML_TABLE.get('actionEnd')) ret_lines = [] ret_lines.append('<rule logged="%s">%s%s%s%s%s%s%s</rule>' % (log, name, action, sources, destinations, service, applied_to_list, notes)) # remove any trailing spaces and replace multiple spaces with singles stripped_ret_lines = [re.sub(r'\s+', ' ', x).rstrip() for x in ret_lines] ret_str.extend(stripped_ret_lines) return ''.join(ret_str)
def _WarnIfCustomTarget(self, target): """Emit a warning if a policy's default target is not a built-in chain.""" if target not in self._GOOD_FILTERS: logging.warn('Filter is generating a non-standard chain that will not ' 'apply to traffic unless linked from INPUT, OUTPUT or ' 'FORWARD filters. New chain name is: %s', target)
def _TranslatePolicy(self, policy, expiration): """Translates policy contents to platform specific data structures. Args: policy: policy object to be transalted to platform specific data structures. expiration: integer number of weeks to be warned about term expiration in. Raises: InvalidTargetOption: if supplied target options are invalid. """ self.tables = collections.defaultdict(list) for header, terms in policy.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) if not filter_options: raise InvalidTargetOption('Chain name not specified.') if len(filter_options) > 4: raise InvalidTargetOption('Too many target options.') if len(filter_options) == 1: raise InvalidTargetOption( 'Must have at least hook name') # Chain name, mandatory chain_name = filter_options[0] # Hook name, mandatory hook_name = filter_options[1].lower() if hook_name not in self._VALID_HOOK_NAMES: raise InvalidTargetOption( 'Specified hook name (%s) is not a valid hook name.' % hook_name) # chain priority, mandatory chain_priority = None if len(filter_options) >= 3: try: chain_priority = str(int(filter_options[2])) except ValueError: raise InvalidTargetOption( 'Specified chain priority is not an integer (%s).' % filter_options[2]) # TODO(castagno): fix this. If you dont have hook name it never prints # anyways, so its not really optional if not hook_name or not chain_priority: logging.info('Chain %s is a non-base chain, make sure it is linked.', chain_name) raise InvalidTargetOption('A table name is required') # Address family, optional, defaults to capirca inet af = 'inet' if len(filter_options) == 4: af = filter_options[3] if af not in self._VALID_ADDRESS_FAMILIES: raise InvalidTargetOption( 'Specified address family (%s) is not supported.' % af) # Terms valid_terms = [] for term in terms: term = self.FixHighPorts(term, af) if not term: continue current_date = datetime.datetime.utcnow().date() expiration_date = current_date + datetime.timedelta(weeks=expiration) if term.expiration: if term.expiration < current_date: logging.warn( 'Term %s in policy %s is expired and will not be rendered.', term.name, chain_name) continue if term.expiration <= expiration_date: logging.info('Term %s in policy %s expires in less than %d weeks.', term.name, chain_name, expiration) valid_terms.append(self._TERM(term, af)) # Add to appropriate table self.tables[af].append((chain_name, hook_name, chain_priority, valid_terms))
def _TranslatePolicy(self, pol, exp_info): # pylint: disable=attribute-defined-outside-init """Transform a policy object into a JuniperSRX object. Args: pol: policy.Policy object exp_info: print a info message when a term is set to expire in that many weeks Raises: UnsupportedFilterError: An unsupported filter was specified UnsupportedHeader: A header option exists that is not understood/usable SRXDuplicateTermError: Two terms were found with same name in same filter ConflictingTargetOptions: Two target options are conflicting in the header MixedAddrBookTypes: Global and Zone address books in the same policy ConflictingApplicationSets: When two duplicate named terms have conflicting application entries """ self.srx_policies = [] self.addressbook = collections.OrderedDict() self.applications = [] self.ports = [] self.from_zone = '' self.to_zone = '' self.addr_book_type = set() current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) if (len(filter_options) < 4 or filter_options[0] != 'from-zone' or filter_options[2] != 'to-zone'): raise UnsupportedFilterError('SRX filter arguments must specify ' 'from-zone and to-zone.') # check if to-zone is not a supported target option if filter_options[1] in self._SUPPORTED_TARGET_OPTIONS: raise UnsupportedFilterError('to-zone %s cannot be the same as any ' 'valid SRX target-options' % (filter_options[1])) else: self.from_zone = filter_options[1] # check if from-zone is not a supported target option if filter_options[3] in self._SUPPORTED_TARGET_OPTIONS: raise UnsupportedFilterError('from-zone %s cannot be the same as any ' 'valid SRX target-options' % (filter_options[3])) else: self.to_zone = filter_options[3] # variables used to collect target-options and set defaults target_options = [] filter_type = '' # parse srx target options extra_options = filter_options[4:] if self._SUPPORTED_TARGET_OPTIONS.issubset(extra_options): raise ConflictingTargetOptions('only one address-book-type can ' 'be specified per header "%s"' % ' '.join(filter_options)) else: address_book_type = self._SUPPORTED_TARGET_OPTIONS.intersection( extra_options) if len(address_book_type) is 0: address_book_type = {self._GLOBAL_ADDR_BOOK} self.addr_book_type.update(address_book_type) if len(self.addr_book_type) > 1: raise MixedAddrBookTypes('Global and Zone address-book-types cannot ' 'be used in the same policy') if self.from_zone == 'all' and self.to_zone == 'all': if self._ZONE_ADDR_BOOK in self.addr_book_type: raise UnsupportedFilterError('Zone address books cannot be used ' 'with a global policy.') elif self.from_zone == 'all' or self.to_zone == 'all': raise UnsupportedFilterError('The zone name all is reserved for ' 'global policies.') for filter_opt in filter_options[4:]: # validate address families if filter_opt in self._SUPPORTED_AF: if not filter_type: filter_type = filter_opt else: raise ConflictingTargetOptions('only one address family can be ' 'specified per header "%s"' % ' '.join(filter_options)) elif filter_opt in self._SUPPORTED_TARGET_OPTIONS: target_options.append(filter_opt) else: raise UnsupportedHeader('SRX Generator currently does not support ' '%s as a header option "%s"' % (filter_opt, ' '.join(filter_options))) # if address-family and address-book-type have not been set then default if not filter_type: filter_type = 'mixed' term_dup_check = set() new_terms = [] self._FixLargePolices(terms, filter_type) for term in terms: term.name = self.FixTermLength(term.name) if term.name in term_dup_check: raise SRXDuplicateTermError('You have a duplicate term: %s' % term.name) term_dup_check.add(term.name) if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s>%s expires ' 'in less than two weeks.', term.name, self.from_zone, self.to_zone) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s>%s is expired.', term.name, self.from_zone, self.to_zone) continue # SRX address books leverage network token names for IPs. # When excluding addresses, we lose those distinct names so we need # to create a new unique name based off the term name before excluding. if term.source_address_exclude: # If we have a naked source_exclude, we need something to exclude from if not term.source_address: term.source_address = [nacaddr.IP('0.0.0.0/0', term.name.upper(), term.name.upper())] # Use the term name as the token & parent_token new_src_parent_token = term.name.upper() + '_SRC_EXCLUDE' new_src_token = new_src_parent_token for i in term.source_address_exclude: term.source_address = nacaddr.RemoveAddressFromList( term.source_address, i) for i in term.source_address: i.token = new_src_token i.parent_token = new_src_parent_token if term.destination_address_exclude: if not term.destination_address: term.destination_address = [nacaddr.IP('0.0.0.0/0', term.name.upper(), term.name.upper())] new_dst_parent_token = term.name.upper() + '_DST_EXCLUDE' new_dst_token = new_dst_parent_token for i in term.destination_address_exclude: term.destination_address = nacaddr.RemoveAddressFromList( term.destination_address, i) for i in term.destination_address: i.token = new_dst_token i.parent_token = new_dst_parent_token # SRX policies are controlled by addresses that are used within, so # policy can be at the same time inet and inet6. if self._GLOBAL_ADDR_BOOK in self.addr_book_type: for zone in self.addressbook: for unused_name, ips in sorted(self.addressbook[zone].iteritems()): ips = [i for i in ips] if term.source_address == ips: term.source_address = ips if term.destination_address == ips: term.destination_address = ips for addr in term.source_address: if addr.version in self._AF_MAP[filter_type]: self._BuildAddressBook(self.from_zone, addr) for addr in term.destination_address: if addr.version in self._AF_MAP[filter_type]: self._BuildAddressBook(self.to_zone, addr) new_term = Term(term, filter_options) new_terms.append(new_term) # Because SRX terms can contain inet and inet6 addresses. We have to # have ability to recover proper AF for ICMP type we need. # If protocol is empty or we cannot map to inet or inet6 we insert bogus # af_type name which will cause new_term.NormalizeIcmpTypes to fail. if not term.protocol: icmp_af_type = 'unknown_af_icmp' else: icmp_af_type = self._AF_ICMP_MAP.get( term.protocol[0], 'unknown_af_icmp') tmp_icmptype = new_term.NormalizeIcmpTypes( term.icmp_type, term.protocol, icmp_af_type) # NormalizeIcmpTypes returns [''] for empty, convert to [] for eval normalized_icmptype = tmp_icmptype if tmp_icmptype != [''] else [] # rewrites the protocol icmpv6 to icmp6 if 'icmpv6' in term.protocol: protocol = list(term.protocol) protocol[protocol.index('icmpv6')] = 'icmp6' else: protocol = term.protocol new_application_set = {'sport': self._BuildPort(term.source_port), 'dport': self._BuildPort(term.destination_port), 'protocol': protocol, 'icmp-type': normalized_icmptype, 'timeout': term.timeout} for application_set in self.applications: if all(item in application_set.items() for item in new_application_set.items()): new_application_set = '' term.replacement_application_name = application_set['name'] break if (term.name == application_set['name'] and new_application_set != application_set): raise ConflictingApplicationSets( 'Application set %s has a conflicting entry' % term.name) if new_application_set: new_application_set['name'] = term.name self.applications.append(new_application_set) self.srx_policies.append((header, new_terms, filter_options))
def main(args): FLAGS(args) if FLAGS.verbose: logging.basicConfig(level=logging.INFO) if FLAGS.debug: logging.basicConfig(level=logging.DEBUG) logging.debug( 'binary: %s\noptimize: %d\nbase_directory: %s\n' 'policy_file: %s\nrendered_acl_directory: %s', str(sys.argv[0]), int(FLAGS.optimize), str(FLAGS.base_directory), str(FLAGS.policy_file), str(FLAGS.output_directory)) definitions = None try: definitions = naming.Naming(FLAGS.definitions_directory) except naming.NoDefinitionsError: err_msg = 'bad definitions directory: %s', FLAGS.definitions_directory logging.fatal(err_msg) sys.exit(1) # thead-safe list for storing files to write manager = multiprocessing.Manager() write_files = manager.list() with_errors = False if FLAGS.policy_file: # render just one file logging.info('rendering one file') RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions, FLAGS.exp_info, write_files) else: # render all files in parallel logging.info('finding policies...') pols = [] pols.extend( DescendRecursively(FLAGS.base_directory, FLAGS.output_directory, definitions)) pool = multiprocessing.Pool(processes=FLAGS.max_renderers) results = [] for x in pols: results.append( pool.apply_async(RenderFile, args=(x.get('in_file'), x.get('out_dir'), definitions, FLAGS.exp_info, write_files))) pool.close() pool.join() for result in results: try: result.get() except (ACLParserError, ACLGeneratorError) as e: with_errors = True logging.warn( '\n\nerror encountered in rendering process:\n%s\n\n', e) # actually write files to disk WriteFiles(write_files) if with_errors: logging.warn('done, with errors.') sys.exit(1) else: logging.info('done.')
def _TranslatePolicy(self, pol, exp_info): self.pcap_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) good_afs = ['inet', 'inet6', 'mixed'] good_options = ['in', 'out'] direction = '' for header, terms in pol.filters: filter_type = None if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM)[1:] filter_name = header.FilterName(self._PLATFORM) # ensure all options after the filter name are expected for opt in filter_options: if opt not in good_afs + good_options: raise UnsupportedTargetOption('%s %s %s %s' % ( '\nUnsupported option found in', self._PLATFORM, 'target definition:', opt)) if 'in' in filter_options: direction = 'in' elif 'out' in filter_options: direction = 'out' # Check for matching af for address_family in good_afs: if address_family in filter_options: # should not specify more than one AF in options if filter_type is not None: raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % ( '\nMay only specify one of', good_afs, 'in filter options:', filter_options)) filter_type = address_family if filter_type is None: filter_type = 'mixed' # add the terms accept_terms = [] deny_terms = [] term_names = set() for term in terms: if term.name in term_names: raise aclgenerator.DuplicateTermError( 'You have a duplicate term: %s' % term.name) if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue if not term: continue if term.action[0] == 'accept': accept_terms.append(self._TERM(term, filter_name, filter_type, direction)) elif term.action[0] == 'deny' or term.action[0] == 'reject': deny_terms.append(self._TERM(term, filter_name, filter_type, direction)) self.pcap_policies.append((header, filter_name, filter_type, accept_terms, deny_terms))
def log_prob(self, x): """Redirects to log_prob_elbo with a warning.""" logging.warn('log_prob is actually a lower bound') return self.log_prob_elbo(x)
def RenderFile(input_file, output_directory, definitions, exp_info, write_files): """Render a single file. Args: input_file: the name of the input policy file. output_directory: the directory in which we place the rendered file. definitions: the definitions from naming.Naming(). exp_info: print a info message when a term is set to expire in that many weeks. write_files: a list of file tuples, (output_file, acl_text), to write """ logging.debug('rendering file: %s into %s', input_file, output_directory) pol = None jcl = False acl = False asacl = False aacl = False bacl = False eacl = False gcefw = False ips = False ipt = False spd = False nsx = False pcap_accept = False pcap_deny = False pf = False srx = False jsl = False nft = False win_afw = False xacl = False paloalto = False try: conf = open(input_file).read() logging.debug('opened and read %s', input_file) except IOError as e: logging.warn('bad file: \n%s', e) raise try: pol = policy.ParsePolicy(conf, definitions, optimize=FLAGS.optimize, base_dir=FLAGS.base_directory, shade_check=FLAGS.shade_check) except policy.ShadingError as e: logging.warn('shading errors for %s:\n%s', input_file, e) return except (policy.Error, naming.Error): raise ACLParserError( 'Error parsing policy file %s:\n%s%s' % (input_file, sys.exc_info()[0], sys.exc_info()[1])) platforms = set() for header in pol.headers: platforms.update(header.platforms) if 'juniper' in platforms: jcl = copy.deepcopy(pol) if 'cisco' in platforms: acl = copy.deepcopy(pol) if 'ciscoasa' in platforms: asacl = copy.deepcopy(pol) if 'brocade' in platforms: bacl = copy.deepcopy(pol) if 'arista' in platforms: eacl = copy.deepcopy(pol) if 'aruba' in platforms: aacl = copy.deepcopy(pol) if 'ipset' in platforms: ips = copy.deepcopy(pol) if 'iptables' in platforms: ipt = copy.deepcopy(pol) if 'nsxv' in platforms: nsx = copy.deepcopy(pol) if 'packetfilter' in platforms: pf = copy.deepcopy(pol) if 'pcap' in platforms: pcap_accept = copy.deepcopy(pol) pcap_deny = copy.deepcopy(pol) if 'speedway' in platforms: spd = copy.deepcopy(pol) if 'srx' in platforms: srx = copy.deepcopy(pol) if 'srxlo' in platforms: jsl = copy.deepcopy(pol) if 'windows_advfirewall' in platforms: win_afw = copy.deepcopy(pol) if 'ciscoxr' in platforms: xacl = copy.deepcopy(pol) if 'nftables' in platforms: nft = copy.deepcopy(pol) if 'gce' in platforms: gcefw = copy.deepcopy(pol) if 'paloalto' in platforms: paloalto = copy.deepcopy(pol) if not output_directory.endswith('/'): output_directory += '/' try: if jcl: acl_obj = juniper.Juniper(jcl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if srx: acl_obj = junipersrx.JuniperSRX(srx, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if acl: acl_obj = cisco.Cisco(acl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if asacl: acl_obj = ciscoasa.CiscoASA(asacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if aacl: acl_obj = aruba.Aruba(aacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if bacl: acl_obj = brocade.Brocade(bacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if eacl: acl_obj = arista.Arista(eacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if ips: acl_obj = ipset.Ipset(ips, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if ipt: acl_obj = iptables.Iptables(ipt, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if nsx: acl_obj = nsxv.Nsxv(nsx, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if spd: acl_obj = speedway.Speedway(spd, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if pcap_accept: acl_obj = pcap.PcapFilter(pcap_accept, exp_info) RenderACL(str(acl_obj), '-accept' + acl_obj.SUFFIX, output_directory, input_file, write_files) if pcap_deny: acl_obj = pcap.PcapFilter(pcap_deny, exp_info, invert=True) RenderACL(str(acl_obj), '-deny' + acl_obj.SUFFIX, output_directory, input_file, write_files) if pf: acl_obj = packetfilter.PacketFilter(pf, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if win_afw: acl_obj = windows_advfirewall.WindowsAdvFirewall(win_afw, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if jsl: acl_obj = srxlo.SRXlo(jsl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if xacl: acl_obj = ciscoxr.CiscoXR(xacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if nft: acl_obj = nftables.Nftables(nft, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if gcefw: acl_obj = gce.GCE(gcefw, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if paloalto: acl_obj = paloaltofw.PaloAltoFW(paloalto, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) # TODO(robankeny) add additional errors. except (juniper.Error, junipersrx.Error, cisco.Error, ipset.Error, iptables.Error, speedway.Error, pcap.Error, aclgenerator.Error, aruba.Error, nftables.Error, gce.Error) as e: raise ACLGeneratorError('Error generating target ACL for %s:\n%s' % (input_file, e))
def __init__(self, root_dir, train_step, agent, experience_dataset_fn=None, after_train_strategy_step_fn=None, triggers=None, checkpoint_interval=100000, summary_interval=1000, max_checkpoints_to_keep=3, use_kwargs_in_agent_train=False, strategy=None, run_optimizer_variable_init=True, use_reverb_v2=False, experience_dataset_options=None, strategy_run_options=None): """Initializes a Learner instance. Args: root_dir: Main directory path where checkpoints, saved_models, and summaries will be written to. train_step: a scalar tf.int64 `tf.Variable` which will keep track of the number of train steps. This is used for artifacts created like summaries, or outputs in the root_dir. agent: `tf_agent.TFAgent` instance to train with. experience_dataset_fn: a function that will create an instance of a tf.data.Dataset used to sample experience for training. Required for using the Learner as is. Optional for subclass learners which take a new iterator each time when `learner.run` is called. after_train_strategy_step_fn: (Optional) callable of the form `fn(sample, loss)` which can be used for example to update priorities in a replay buffer where sample is pulled from the `experience_iterator` and loss is a `LossInfo` named tuple returned from the agent. This is called after every train step. It runs using `strategy.run(...)`. triggers: List of callables of the form `trigger(train_step)`. After every `run` call every trigger is called with the current `train_step` value as an np scalar. checkpoint_interval: Number of train steps in between checkpoints. Note these are placed into triggers and so a check to generate a checkpoint only occurs after every `run` call. Set to -1 to disable (this is not recommended, because it means that if the pipeline gets preempted, all previous progress is lost). This only takes care of the checkpointing the training process. Policies must be explicitly exported through triggers. summary_interval: Number of train steps in between summaries. Note these are placed into triggers and so a check to generate a checkpoint only occurs after every `run` call. max_checkpoints_to_keep: Maximum number of checkpoints to keep around. These are used to recover from pre-emptions when training. use_kwargs_in_agent_train: If True the experience from the replay buffer is passed into the agent as kwargs. This requires samples from the RB to be of the form `dict(experience=experience, kwarg1=kwarg1, ...)`. This is useful if you have an agent with a custom argspec. strategy: (Optional) `tf.distribute.Strategy` to use during training. run_optimizer_variable_init: Specifies if the variables of the optimizer are initialized before checkpointing. This should be almost always `True` (default) to ensure that the state of the optimizer is checkpointed properly. The initialization of the optimizer variables happens by building the Tensorflow graph. This is done by calling a `get_concrete_function` on the agent's `train` method which requires passing some input. Since, no real data is available at this point we use the batched form of `training_data_spec` to achieve this (standard technique). The problem arises when the agent expects some agent specific batching of the input. In this case, there is no _general_ way at this point in the learner to batch the impacted specs properly. To avoid breaking the code in these specific cases, we recommend turning off initialization of the optimizer variables by setting the value of this field to `False`. use_reverb_v2: If True then we expect the dataset samples to return a named_tuple with a data and an info field. If False we expect a tuple(data, info). experience_dataset_options: (Optional) `tf.distribute.InputOptions` passed to `strategy.distribute_datasets_from_function`, used to control options on how this dataset is distributed. strategy_run_options: (Optional) `tf.distribute.RunOptions` passed to `strategy.run`. This is passed to every strategy.run invocation by the learner. """ if checkpoint_interval < 0: logging.warning( 'Warning: checkpointing the training process is manually disabled.' 'This means training progress will NOT be automatically restored ' 'if the job gets preempted.') self._train_dir = os.path.join(root_dir, TRAIN_DIR) self._use_reverb_v2 = use_reverb_v2 if summary_interval: self.train_summary_writer = tf.compat.v2.summary.create_file_writer( self._train_dir, flush_millis=10000) else: self.train_summary_writer = tf.summary.create_noop_writer() self.train_step = train_step self._agent = agent self.use_kwargs_in_agent_train = use_kwargs_in_agent_train self.strategy = strategy or tf.distribute.get_strategy() dataset = None if experience_dataset_fn: with self.strategy.scope(): dataset = self.strategy.distribute_datasets_from_function( lambda _: experience_dataset_fn(), options=experience_dataset_options) self._experience_iterator = iter(dataset) self.after_train_strategy_step_fn = after_train_strategy_step_fn self.triggers = triggers or [] # Prevent autograph from going into the agent. self._agent.train = tf.autograph.experimental.do_not_convert( agent.train) self._strategy_run_options = strategy_run_options checkpoint_dir = os.path.join(self._train_dir, POLICY_CHECKPOINT_DIR) with self.strategy.scope(): agent.initialize() if run_optimizer_variable_init: # Force a concrete function creation inside of the strategy scope to # ensure that all variables, including optimizer slot variables, are # created. This has to happen before the checkpointer is created. if dataset is not None: if use_reverb_v2: batched_specs = dataset.element_spec.data else: # Assumes (experience, sample_info) = next(iterator) batched_specs, _ = dataset.element_spec else: batched_specs = tensor_spec.add_outer_dims_nest( self._agent.training_data_spec, (None, self._agent.train_sequence_length)) if self.use_kwargs_in_agent_train: batched_specs = dict(experience=batched_specs) @common.function def _create_variables(specs): # TODO(b/170516529): Each replica has to be in the same graph. # This can be ensured by placing the `strategy.run(...)` call inside # the `tf.function`. if self.use_kwargs_in_agent_train: return self.strategy.run( self._agent.train, kwargs=specs, options=self._strategy_run_options) return self.strategy.run( self._agent.train, args=(specs, ), options=self._strategy_run_options) _create_variables.get_concrete_function(batched_specs) else: # TODO(b/186052656) Update clients. logging.warn( 'run_optimizer_variable_init = False is Deprecated') self._checkpointer = common.Checkpointer( checkpoint_dir, max_to_keep=max_checkpoints_to_keep, agent=self._agent, train_step=self.train_step) self._checkpointer.initialize_or_restore() # pytype: disable=attribute-error for trigger in self.triggers: if hasattr(trigger, 'set_start'): trigger.set_start(self.train_step.numpy()) self.triggers.append(self._get_checkpoint_trigger(checkpoint_interval)) self.summary_interval = tf.constant(summary_interval, dtype=tf.int64)
def __str__(self): output = [] # Don't render term if not in platforms or in excluded platforms if self.term.platform and self._PLATFORM not in self.term.platform: return '' if (self.term.platform_exclude and self._PLATFORM in self.term.platform_exclude): return '' # Don't render icmpv6 protocol terms under inet, or icmp under inet6 # Does not currently support mixed family. if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or (self.af == 'inet' and 'icmpv6' in self.term.protocol)): logging.debug( self.NO_AF_LOG_PROTO.substitute(term=self.term.name, proto=self.term.protocol, af=self.af)) return '' # Term verbatim output - this will skip over most normal term # creation code by returning early. Warnings provided in policy.py. if self.term.verbatim: for verbatim_line in self.term.verbatim: platform, contents = verbatim_line.value if platform == self._PLATFORM: output.append(str(contents)) return '\n'.join(output) # Source address if self.term.source_address or self.term.source_address_exclude: src_addrs = self._CalculateAddrs(self.term.source_address, self.term.source_address_exclude) if not src_addrs: logging.warn( self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='source', af=self.af)) return '' # TODO(castagno): Add support for ipv6 output.append('ip saddr %s' % self._FormatMatch(src_addrs)) # Destination address if self.term.destination_address or self.term.source_address_exclude: dst_addrs = self._CalculateAddrs( self.term.destination_address, self.term.destination_address_exclude) if not dst_addrs: logging.warn( self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='destination', af=self.af)) return '' # TODO(castagno): Add support for ipv6 output.append('ip daddr %s' % self._FormatMatch(dst_addrs)) # Protocol # # nft intepreter shortcuts protocol specification if there are more specific # matches. At the moment, these are: # * source port # * destination port # * ICMP type if self.term.protocol and not (self.term.source_port or self.term.destination_port or self.term.icmp_type): output.append('ip protocol %s' % self._FormatMatch(self.term.protocol)) # Source port if self.term.source_port: output.append('%s sport %s' % (self._FormatMatch( self.term.protocol), self._FormatMatch(self.term.source_port))) # Destination port if self.term.destination_port: output.append('%s dport %s' % (self._FormatMatch(self.term.protocol), self._FormatMatch(self.term.destination_port))) # Icmp type if self.term.icmp_type: icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, self.term.protocol, self.af) if icmp_types != ['']: # nft intepreter requires ICMP types to be spelled out icmp_name_types = self.ICMP_TYPE[self.AF_MAP[self.af]] icmp_type_names = dict( (v, k) for k, v in six.iteritems(icmp_name_types)) output.append('icmp type %s' % self._FormatMatch( [icmp_type_names[icmp_type] for icmp_type in icmp_types])) # Counter # This does not use the value that was passed in the term. if self.term.counter: output.append('counter') # Log # Setup logic so that only one log statement is printed. if self.term.logging and not self.term.log_name: output.append('log') elif (self.term.logging and self.term.log_name) or self.term.log_name: # Only supports log prefix's of 128 characters truncate to 126 to support # the additional suffix that is being added output.append('log prefix "%s: "' % self.term.log_name[:126]) # Action output.append(self._ACTIONS[self.term.action[0]]) # Owner (implement as comment) if self.term.owner: self.term.comment.append('Owner: %s' % self.term.owner) # Comment if self.term.comment: comment_data = ' '.join(self.term.comment) # Have to truncate MAX_CHARACTERS characters due to NFTables limitation if len(comment_data) > self.MAX_CHARACTERS: # Have to use the first MAX_CHARACTERS characters comment_data = comment_data[:self.MAX_CHARACTERS] logging.warn( 'Term %s in policy is too long (>%d characters) ' 'and will be truncated', self.term.name, self.MAX_CHARACTERS) output.append('comment "%s"' % comment_data) return ' '.join(output)
def _TranslatePolicy(self, policy, expiration): """Translates policy contents to platform specific data structures. Args: policy: policy object to be transalted to platform specific data structures. expiration: integer number of weeks to be warned about term expiration in. Raises: InvalidTargetOption: if supplied target options are invalid. """ self.tables = collections.defaultdict(list) for header, terms in policy.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) if not filter_options: raise InvalidTargetOption('Chain name not specified.') if len(filter_options) > 4: raise InvalidTargetOption('Too many target options.') if len(filter_options) == 1: raise InvalidTargetOption('Must have at least hook name') # Chain name, mandatory chain_name = filter_options[0] # Hook name, mandatory hook_name = filter_options[1].lower() if hook_name not in self._VALID_HOOK_NAMES: raise InvalidTargetOption( 'Specified hook name (%s) is not a valid hook name.' % hook_name) # chain priority, mandatory chain_priority = None if len(filter_options) >= 3: try: chain_priority = str(int(filter_options[2])) except ValueError: raise InvalidTargetOption( 'Specified chain priority is not an integer (%s).' % filter_options[2]) # TODO(castagno): fix this. If you dont have hook name it never prints # anyways, so its not really optional if not hook_name or not chain_priority: logging.info( 'Chain %s is a non-base chain, make sure it is linked.', chain_name) raise InvalidTargetOption('A table name is required') # Address family, optional, defaults to capirca inet af = 'inet' if len(filter_options) == 4: af = filter_options[3] if af not in self._VALID_ADDRESS_FAMILIES: raise InvalidTargetOption( 'Specified address family (%s) is not supported.' % af) # Terms valid_terms = [] for term in terms: term = self.FixHighPorts(term, af) if not term: continue current_date = datetime.datetime.utcnow().date() expiration_date = current_date + datetime.timedelta( weeks=expiration) if term.expiration: if term.expiration < current_date: logging.warn( 'Term %s in policy %s is expired and will not be rendered.', term.name, chain_name) continue if term.expiration <= expiration_date: logging.info( 'Term %s in policy %s expires in less than %d weeks.', term.name, chain_name, expiration) valid_terms.append(self._TERM(term, af)) # Add to appropriate table self.tables[af].append( (chain_name, hook_name, chain_priority, valid_terms))
def main(_): """Runs fine-tuning and inference. There are three categories of images. 1) Images where we have previous and next frame, and that are not filtered out by the heuristic. For them, we will use the fine-tuned predictions. 2) Images where we have previous and next frame, but that were filtered out by our heuristic. For them, we will use the ordinary prediction instead. 3) Images where we have at least one missing adjacent frame. For them, we will use the ordinary prediction as indicated by triplet_list_file_remains (if provided). They will also not be part of the generated inference list in the first place. Raises: ValueError: Invalid parameters have been passed. """ if FLAGS.handle_motion and FLAGS.joint_encoder: raise ValueError( 'Using a joint encoder is currently not supported when ' 'modeling object motion.') if FLAGS.handle_motion and FLAGS.seq_length != 3: raise ValueError( 'The current motion model implementation only supports ' 'using a sequence length of three.') if FLAGS.handle_motion and not FLAGS.compute_minimum_loss: raise ValueError( 'Computing the minimum photometric loss is required when ' 'enabling object motion handling.') if FLAGS.size_constraint_weight > 0 and not FLAGS.handle_motion: raise ValueError('To enforce object size constraints, enable motion ' 'handling.') if FLAGS.icp_weight > 0.0: raise ValueError('ICP is currently not supported.') if FLAGS.compute_minimum_loss and FLAGS.seq_length % 2 != 1: raise ValueError( 'Compute minimum loss requires using an odd number of ' 'images in a sequence.') if FLAGS.compute_minimum_loss and FLAGS.exhaustive_mode: raise ValueError( 'Exhaustive mode has no effect when compute_minimum_loss ' 'is enabled.') if FLAGS.img_width % (2**5) != 0 or FLAGS.img_height % (2**5) != 0: logging.warn( 'Image size is not divisible by 2^5. For the architecture ' 'employed, this could cause artefacts caused by resizing in ' 'lower dimensions.') if FLAGS.output_dir.endswith('/'): FLAGS.output_dir = FLAGS.output_dir[:-1] # Create file lists to prepare fine-tuning, save it to unique_file. unique_file_name = (str(datetime.datetime.now().date()) + '_' + str(datetime.datetime.now().time()).replace(':', '_')) unique_file = os.path.join(FLAGS.data_dir, unique_file_name + '.txt') with gfile.FastGFile(FLAGS.triplet_list_file, 'r') as f: files_to_process = f.readlines() files_to_process = [line.rstrip() for line in files_to_process] files_to_process = [line for line in files_to_process if len(line)] logging.info('Creating unique file list %s with %s entries.', unique_file, len(files_to_process)) with gfile.FastGFile(unique_file, 'w') as f_out: fetches_network = FLAGS.num_steps * FLAGS.batch_size fetches_saves = FLAGS.batch_size * int( np.floor(FLAGS.num_steps / SAVE_EVERY)) repetitions = fetches_network + 3 * fetches_saves for i in range(len(files_to_process)): for _ in range(repetitions): f_out.write(files_to_process[i] + '\n') # Read remaining files. remaining = [] if gfile.Exists(FLAGS.triplet_list_file_remains): with gfile.FastGFile(FLAGS.triplet_list_file_remains, 'r') as f: remaining = f.readlines() remaining = [line.rstrip() for line in remaining] remaining = [line for line in remaining if len(line)] logging.info('Running fine-tuning on %s files, %s files are remaining.', len(files_to_process), len(remaining)) # Run fine-tuning process and save predictions in id-folders. tf.set_random_seed(FIXED_SEED) np.random.seed(FIXED_SEED) random.seed(FIXED_SEED) flipping_mode = reader.FLIP_ALWAYS if FLAGS.flip else reader.FLIP_NONE train_model = model.Model( data_dir=FLAGS.data_dir, file_extension=FLAGS.file_extension, is_training=True, learning_rate=FLAGS.learning_rate, beta1=FLAGS.beta1, reconstr_weight=FLAGS.reconstr_weight, smooth_weight=FLAGS.smooth_weight, ssim_weight=FLAGS.ssim_weight, icp_weight=FLAGS.icp_weight, batch_size=FLAGS.batch_size, img_height=FLAGS.img_height, img_width=FLAGS.img_width, seq_length=FLAGS.seq_length, architecture=FLAGS.architecture, imagenet_norm=FLAGS.imagenet_norm, weight_reg=FLAGS.weight_reg, exhaustive_mode=FLAGS.exhaustive_mode, random_scale_crop=FLAGS.random_scale_crop, flipping_mode=flipping_mode, random_color=False, depth_upsampling=FLAGS.depth_upsampling, depth_normalization=FLAGS.depth_normalization, compute_minimum_loss=FLAGS.compute_minimum_loss, use_skip=FLAGS.use_skip, joint_encoder=FLAGS.joint_encoder, build_sum=False, shuffle=False, input_file=unique_file_name, handle_motion=FLAGS.handle_motion, size_constraint_weight=FLAGS.size_constraint_weight, train_global_scale_var=False) failed_heuristic_ids = finetune_inference(train_model, FLAGS.model_ckpt, FLAGS.output_dir + '_ft') logging.info( 'Fine-tuning completed, %s files were filtered out by ' 'heuristic.', len(failed_heuristic_ids)) for failed_id in failed_heuristic_ids: failed_entry = files_to_process[failed_id] remaining.append(failed_entry) logging.info('In total, %s images were fine-tuned, while %s were not.', len(files_to_process) - len(failed_heuristic_ids), len(remaining)) # Copy all results to have the same structural output as running ordinary # inference. for i in range(len(files_to_process)): if files_to_process[i] not in remaining: # Use fine-tuned result. elements = files_to_process[i].split(' ') source_file = os.path.join( FLAGS.output_dir + '_ft', FLAGS.ft_name + 'id_' + str(i), str(FLAGS.num_steps).zfill(10) + ('_flip' if FLAGS.flip else '')) if len(elements) == 2: # No differing mapping defined. target_dir = os.path.join(FLAGS.output_dir + '_ft', elements[0]) target_file = os.path.join( target_dir, elements[1] + ('_flip' if FLAGS.flip else '')) else: # Other mapping for file defined, copy to this location instead. target_dir = os.path.join(FLAGS.output_dir + '_ft', os.path.dirname(elements[2])) target_file = os.path.join( target_dir, os.path.basename(elements[2]) + ('_flip' if FLAGS.flip else '')) if not gfile.Exists(target_dir): gfile.MakeDirs(target_dir) logging.info('Copy refined result %s to %s.', source_file, target_file) gfile.Copy(source_file + '.npy', target_file + '.npy', overwrite=True) gfile.Copy(source_file + '.txt', target_file + '.txt', overwrite=True) gfile.Copy(source_file + '.%s' % FLAGS.file_extension, target_file + '.%s' % FLAGS.file_extension, overwrite=True) for j in range(len(remaining)): elements = remaining[j].split(' ') if len(elements) == 2: # No differing mapping defined. target_dir = os.path.join(FLAGS.output_dir + '_ft', elements[0]) target_file = os.path.join( target_dir, elements[1] + ('_flip' if FLAGS.flip else '')) else: # Other mapping for file defined, copy to this location instead. target_dir = os.path.join(FLAGS.output_dir + '_ft', os.path.dirname(elements[2])) target_file = os.path.join( target_dir, os.path.basename(elements[2]) + ('_flip' if FLAGS.flip else '')) if not gfile.Exists(target_dir): gfile.MakeDirs(target_dir) source_file = target_file.replace('_ft', '') logging.info('Copy unrefined result %s to %s.', source_file, target_file) gfile.Copy(source_file + '.npy', target_file + '.npy', overwrite=True) gfile.Copy(source_file + '.%s' % FLAGS.file_extension, target_file + '.%s' % FLAGS.file_extension, overwrite=True) logging.info('Done, predictions saved in %s.', FLAGS.output_dir + '_ft')
def _warn(msg): print(f'Warning: {msg}') logging.warn(msg)
def __init__(self, config, task_id, ps_tasks, num_workers, is_chief=True, summary_writer=None, dtype=tf.float32, summary_interval=1, run_number=0, logging_dir='/tmp', model_v=0): self.config = config self.data_manager = data.DataManager( config, run_number=run_number, do_code_simplification=not FLAGS.stop_on_success) self.task_id = task_id self.ps_tasks = ps_tasks self.is_chief = is_chief if ps_tasks == 0: assert task_id == 0, 'No parameter servers specified. Expecting 1 task.' assert num_workers == 1, ( 'No parameter servers specified. Expecting 1 task.') worker_device = '/job:localhost/replica:%d/task:0/cpu:0' % task_id # worker_device = '/cpu:0' # ps_device = '/cpu:0' else: assert num_workers > 0, 'There must be at least 1 training worker.' worker_device = '/job:worker/replica:%d/task:0/cpu:0' % task_id # ps_device = '/job:ps/replica:0/task:0/cpu:0' logging.info('worker_device: %s', worker_device) logging_file = os.path.join( logging_dir, 'solutions_%d.txt' % task_id) experience_replay_file = os.path.join( logging_dir, 'replay_buffer_%d.pickle' % task_id) self.topk_file = os.path.join( logging_dir, 'topk_buffer_%d.pickle' % task_id) tf.get_variable_scope().set_use_resource(True) # global model with tf.device(tf.train.replica_device_setter(ps_tasks, ps_device='/job:ps/replica:0', worker_device=worker_device)): with tf.variable_scope('global'): global_model = agent_lib.LMAgent(config, dtype=dtype, is_local=False) global_params_dict = {p.name: p for p in global_model.sync_variables} self.global_model = global_model self.global_step = make_initialized_variable( 0, 'global_step', dtype=tf.int64) self.global_best_reward = make_initialized_variable( -10.0, 'global_best_reward', dtype=tf.float64) self.is_best_model = make_initialized_variable( False, 'is_best_model', dtype=tf.bool) self.reset_is_best_model = self.is_best_model.assign(False) self.global_best_reward_placeholder = tf.placeholder( tf.float64, [], name='global_best_reward_placeholder') self.assign_global_best_reward_op = tf.group( self.global_best_reward.assign( self.global_best_reward_placeholder), self.is_best_model.assign(True)) def assign_global_best_reward_fn(session, reward): reward = round(reward, 10) best_reward = round(session.run(self.global_best_reward), 10) is_best = reward > best_reward if is_best: session.run(self.assign_global_best_reward_op, {self.global_best_reward_placeholder: reward}) return is_best self.assign_global_best_reward_fn = assign_global_best_reward_fn # Any worker will set to true when it finds a solution. self.found_solution_flag = make_initialized_variable( False, 'found_solution_flag', dtype=tf.bool) self.found_solution_op = self.found_solution_flag.assign(True) self.run_number = make_initialized_variable( run_number, 'run_number', dtype=tf.int32) # Store a solution when found. self.code_solution_variable = tf.get_variable( 'code_solution', [], tf.string, initializer=tf.constant_initializer('')) self.code_solution_ph = tf.placeholder( tf.string, [], name='code_solution_ph') self.code_solution_assign_op = self.code_solution_variable.assign( self.code_solution_ph) def assign_code_solution_fn(session, code_solution_string): session.run(self.code_solution_assign_op, {self.code_solution_ph: code_solution_string}) self.assign_code_solution_fn = assign_code_solution_fn # Count all programs sampled from policy. This does not include # programs sampled from replay buffer. # This equals NPE (number of programs executed). Only programs sampled # from the policy need to be executed. self.program_count = make_initialized_variable( 0, 'program_count', dtype=tf.int64) # local model with tf.device(worker_device): with tf.variable_scope('local'): self.model = model = agent_lib.LMAgent( config, task_id=task_id, logging_file=logging_file, experience_replay_file=experience_replay_file, dtype=dtype, global_best_reward_fn=self.assign_global_best_reward_fn, found_solution_op=self.found_solution_op, assign_code_solution_fn=self.assign_code_solution_fn, program_count=self.program_count, stop_on_success=FLAGS.stop_on_success, verbose_level=model_v) local_params = model.trainable_variables local_params_dict = {p.name: p for p in local_params} # Pull global params to local model. def _global_to_local_scope(name): assert name.startswith('global/') return 'local' + name[6:] sync_dict = { local_params_dict[_global_to_local_scope(p_name)]: p for p_name, p in global_params_dict.items()} self.sync_op = tf.group(*[v_local.assign(v_global) for v_local, v_global in sync_dict.items()]) # Pair local gradients with global params. grad_var_dict = { gradient: sync_dict[local_var] for local_var, gradient in model.gradients_dict.items()} # local model model.make_summary_ops() # Don't put summaries under 'local' scope. with tf.variable_scope('local'): self.train_op = model.optimizer.apply_gradients( grad_var_dict.items(), global_step=self.global_step) self.local_init_op = tf.variables_initializer( tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES, tf.get_variable_scope().name)) self.local_step = 0 self.last_summary_time = time.time() self.summary_interval = summary_interval self.summary_writer = summary_writer self.cached_global_step = -1 self.cached_global_npe = -1 logging.info('summary_interval: %d', self.summary_interval) # Load top-k buffer. if self.model.top_episodes is not None and tf.gfile.Exists(self.topk_file): try: with tf.gfile.FastGFile(self.topk_file, 'r') as f: self.model.top_episodes = cPickle.loads(f.read()) logging.info( 'Loaded top-k buffer from disk with %d items. Location: "%s"', len(self.model.top_episodes), self.topk_file) except (cPickle.UnpicklingError, EOFError) as e: logging.warn( 'Failed to load existing top-k buffer from disk. Removing bad file.' '\nLocation: "%s"\nException: %s', self.topk_file, str(e)) tf.gfile.Remove(self.topk_file)
def trim_and_pack_dataset( dataset: tf.data.Dataset, feature_lengths: Mapping[str, int], use_custom_ops: bool = False ) -> tf.data.Dataset: """Creates a 'packed' version of a dataset on-the-fly. Modified from the tensor2tensor library. This is meant to replace the irritation of having to create a separate "packed" version of a dataset to train efficiently on TPU. Each example in the output dataset represents several examples in the input dataset. For each key in the input dataset that also exists in `feature_lengths`, two additional keys are created: <key>_segment_id: an int32 tensor identifying the parts representing the original example. <key>_position: an int32 tensor identifying the position within the original example. Features that are not in `feature_lengths` will be removed. Example: Two input examples get combined to form an output example. The input examples are: {"inputs": [8, 7, 1, 0], "targets":[4, 1, 0], "idx": 0} {"inputs": [2, 3, 4, 1], "targets":[5, 6, 1], "idx": 1} The output example is: { "inputs": [8, 7, 1, 2, 3, 4, 1, 0, 0, 0] "inputs_segment_id": [1, 1, 1, 2, 2, 2, 2, 0, 0, 0] "inputs_position": [0, 1, 2, 0, 1, 2, 3, 0, 0, 0] "targets": [4, 1, 5, 6, 1, 0, 0, 0, 0, 0] "targets_segment_id": [1, 1, 2, 2, 2, 0, 0, 0, 0, 0] "targets_position": [0, 1, 0, 1, 2, 0, 0, 0, 0, 0] } 0 represents padding in both the inputs and the outputs. Sequences in the incoming examples are truncated to length in `feature_lengths`, and the sequences in the output examples all have this fixed (padded) length. Features not in `features_length` (i.e, "idx") are removed. Args: dataset: a tf.data.Dataset feature_lengths: map from feature key to final length. Other features will be discarded. use_custom_ops: a boolean - custom ops are faster but require a custom-built binary, which is not currently possible on cloud-tpu. Returns: a tf.data.Dataset """ element_spec = dataset.element_spec # Make sure that the dataset contains all keys in `feature_lengths`. for k in feature_lengths: if k not in element_spec: raise ValueError( f"Feature '{k}' not found in dataset. Available keys are " f"{list(element_spec.keys())}") if not element_spec[k].shape.is_compatible_with(tf.TensorShape([None])): raise ValueError( f"Features to be packed must be one-dimensional. '{k}' is not.'") # Warn if there are any additional keys that will be removed. additional_keys = set(element_spec) - set(feature_lengths) if additional_keys: logging.warn( "Features not in `features_length` will be removed during packing: %s", additional_keys) ds = dataset.map( lambda x: {k: x[k][:l] for k, l in feature_lengths.items()}, num_parallel_calls=tf.data.experimental.AUTOTUNE) # Setting batch_size=length ensures that the concatenated sequences (if they # have length >=1) are sufficient to fill at least one packed example. batch_size = max(feature_lengths.values()) ds = ds.padded_batch( batch_size, padded_shapes={k: [-1] for k in feature_lengths}) if use_custom_ops and len(feature_lengths) <= 2: ds = _pack_with_custom_ops(ds, feature_lengths) else: ds = _pack_with_tf_ops(ds, feature_lengths) # Set the Tensor shapes correctly since they get lost in the process. def _set_shape(x): for k, v in x.items(): v.set_shape([feature_lengths[_strip_packed_feature_key(k)]]) return x return ds.map(_set_shape, num_parallel_calls=tf.data.experimental.AUTOTUNE)
def get_paths_to_events(root_dir, restrict_to_architectures, restrict_to_pretrained_source, restrict_to_variants=None): """Returns a dict that maps each variant name to its event file. The name of the variant is the basename of the directory where it's stored. Assumes the following directory organization root_dir contains a sub-directory for every variant where event files can be found. There may be more than one event file for each variant, e.g. a new one will be created upon restarting an experiment that was pre-empted. So later event files contain the summaries for larger values of 'step'. We need all of them for determining the global 'best'. Args: root_dir: A str. The root directory of experiments of all models variants. restrict_to_architectures: A list of names of architectures to restrict to when choosing the best variant. restrict_to_pretrained_source: A string. The pretrained_source to restrict to when choosing the best variant. restrict_to_variants: Optionally, a set of variant names to restrict to. """ params_dir = os.path.join(root_dir, 'params') summary_dir = os.path.join(root_dir, 'summaries') def get_variant_architecture(name): """Get the architecture of the given variant if it's recorded, o/w None.""" variant_params_dir = os.path.join(params_dir, name) variant_params_file = os.path.join(variant_params_dir, 'params.pkl') return get_value_from_pkl(variant_params_file, '_gin.LearnerConfig.embedding_network') def get_variant_pretrained_source(name): variant_params_dir = os.path.join(params_dir, name) variant_params_file = os.path.join(variant_params_dir, 'params.pkl') return get_value_from_pkl(variant_params_file, '_gin.LearnerConfig.pretrained_source') def keep_variant(name): """Determine if the variant in directory name should be considered.""" value_error_msg = ( 'Requested to restrict to an architecture or ' 'pretrained_source but the given experiment does not ' 'have its params recorded. Looked in: {}'.format(params_dir)) if restrict_to_architectures: architecture = get_variant_architecture(name) if architecture is None: raise ValueError(value_error_msg) valid_architecture = (not restrict_to_architectures or architecture in restrict_to_architectures) if restrict_to_pretrained_source: pretrained_source = get_variant_pretrained_source(name) if pretrained_source is None: raise ValueError(value_error_msg) valid_pretrained_source = (not restrict_to_pretrained_source or pretrained_source == restrict_to_pretrained_source) valid_variant_name = True if restrict_to_variants is not None: valid_variant_name = name in restrict_to_variants return (valid_architecture and valid_pretrained_source and valid_variant_name) variant_names = [ fname for fname in tf.io.gfile.listdir(summary_dir) if tf.io.gfile.isdir(os.path.join(summary_dir, fname)) ] # Further filter variant names based on the given restrictions. variant_names = [name for name in variant_names if keep_variant(name)] if not variant_names: raise ValueError('Found no subdirectories in {}. Was expecting a ' 'subdirectory per variant.'.format(summary_dir)) variant_paths = [ os.path.join(summary_dir, variant_dir) for variant_dir in variant_names ] event_paths = {} for variant_path, variant_name in zip(variant_paths, variant_names): event_filenames = [ f_name for f_name in tf.io.gfile.listdir(variant_path) if f_name.startswith('events.out.tfevents') ] if len(event_filenames) < 1: logging.warn('Skipping empty variant %s.', variant_path) logging.info( 'Was expecting at least one event file ' 'in directory %s. Instead, found %d.', variant_path, len(event_filenames)) continue event_paths[variant_name] = [ os.path.join(variant_path, event_filename) for event_filename in event_filenames ] logging.info('Found event files for variants: %s', list(event_paths.keys())) return event_paths
def build_ensemble_spec(self, name, candidate, ensembler, subnetwork_specs, summary, features, mode, iteration_number, labels=None, previous_ensemble_spec=None): """Builds an `_EnsembleSpec` with the given `adanet.ensemble.Candidate`. Args: name: The string name of the ensemble. Typically the name of the builder that returned the given `Subnetwork`. candidate: The `adanet.ensemble.Candidate` for this spec. ensembler: The :class:`adanet.ensemble.Ensembler` to use to ensemble a group of subnetworks. subnetwork_specs: Iterable of `_SubnetworkSpecs` for this iteration. summary: A `_ScopedSummary` instance for recording ensemble summaries. features: Input `dict` of `Tensor` objects. mode: Estimator `ModeKeys` indicating training, evaluation, or inference. iteration_number: Integer current iteration number. labels: Labels `Tensor` or a dictionary of string label name to `Tensor` (for multi-head). previous_ensemble_spec: Link the rest of the `_EnsembleSpec` from iteration t-1. Used for creating the subnetwork train_op. Returns: An `_EnsembleSpec` instance. """ with tf_compat.v1.variable_scope("ensemble_{}".format(name)): step = tf_compat.v1.get_variable( "step", shape=[], initializer=tf_compat.v1.zeros_initializer(), trainable=False, dtype=tf.int64) # Convert to tensor so that users cannot mutate it. step_tensor = tf.convert_to_tensor(value=step) with summary.current_scope(): summary.scalar("iteration_step/adanet/iteration_step", step_tensor) architecture = _Architecture(candidate.name, ensembler.name) previous_subnetworks = [] subnetwork_builders = [] previous_ensemble = None if previous_ensemble_spec: previous_ensemble = previous_ensemble_spec.ensemble previous_architecture = previous_ensemble_spec.architecture keep_indices = range(len(previous_ensemble.subnetworks)) if len(candidate.subnetwork_builders) == 1 and previous_ensemble: # Prune previous ensemble according to the subnetwork.Builder for # backwards compatibility. subnetwork_builder = candidate.subnetwork_builders[0] prune_previous_ensemble = getattr(subnetwork_builder, "prune_previous_ensemble", None) if callable(prune_previous_ensemble): logging.warn( "Using an `adanet.subnetwork.Builder#prune_previous_ensemble` " "is deprecated. Please use a custom `adanet.ensemble.Strategy` " "instead.") keep_indices = prune_previous_ensemble(previous_ensemble) for i, builder in enumerate(previous_ensemble_spec.subnetwork_builders): if i not in keep_indices: continue if builder not in candidate.previous_ensemble_subnetwork_builders: continue previous_subnetworks.append(previous_ensemble.subnetworks[i]) subnetwork_builders.append(builder) architecture.add_subnetwork(*previous_architecture.subnetworks[i]) for builder in candidate.subnetwork_builders: architecture.add_subnetwork(iteration_number, builder.name) subnetwork_builders.append(builder) subnetwork_map = {s.builder.name: s.subnetwork for s in subnetwork_specs} subnetworks = [ subnetwork_map[s.name] for s in candidate.subnetwork_builders ] ensemble_scope = tf_compat.v1.get_variable_scope() before_var_list = tf_compat.v1.trainable_variables() with summary.current_scope(), _monkey_patch_context( iteration_step_scope=ensemble_scope, scoped_summary=summary, trainable_vars=[]): ensemble = ensembler.build_ensemble( subnetworks, previous_ensemble_subnetworks=previous_subnetworks, features=features, labels=labels, logits_dimension=self._head.logits_dimension, training=mode == tf.estimator.ModeKeys.TRAIN, iteration_step=step_tensor, summary=summary, previous_ensemble=previous_ensemble) ensemble_var_list = _new_trainable_variables(before_var_list) estimator_spec = _create_estimator_spec(self._head, features, labels, mode, ensemble.logits, self._use_tpu) ensemble_loss = estimator_spec.loss adanet_loss = None if mode != tf.estimator.ModeKeys.PREDICT: # TODO: Support any kind of Ensemble. Use a moving average of # their train loss for the 'adanet_loss'. if not isinstance(ensemble, ComplexityRegularized): raise ValueError( "Only ComplexityRegularized ensembles are supported.") adanet_loss = estimator_spec.loss + ensemble.complexity_regularization ensemble_metrics = _EnsembleMetrics(use_tpu=self._use_tpu) if mode == tf.estimator.ModeKeys.EVAL: ensemble_metrics.create_eval_metrics( features=features, labels=labels, estimator_spec=estimator_spec, metric_fn=self._metric_fn, architecture=architecture) if mode == tf.estimator.ModeKeys.TRAIN: with summary.current_scope(): summary.scalar("loss", estimator_spec.loss) # Create train ops for training subnetworks and ensembles. train_op = None if mode == tf.estimator.ModeKeys.TRAIN: # Note that these mixture weights are on top of the last_layer of the # subnetwork constructed in TRAIN mode, which means that dropout is # still applied when the mixture weights are being trained. ensemble_scope = tf_compat.v1.get_variable_scope() with tf_compat.v1.variable_scope("train_mixture_weights"): with summary.current_scope(), _monkey_patch_context( iteration_step_scope=ensemble_scope, scoped_summary=summary, trainable_vars=ensemble_var_list): # For backwards compatibility. subnetwork_builder = candidate.subnetwork_builders[0] old_train_op_fn = getattr(subnetwork_builder, "build_mixture_weights_train_op", None) if callable(old_train_op_fn): logging.warn( "The `build_mixture_weights_train_op` method is deprecated. " "Please use the `Ensembler#build_train_op` instead.") train_op = _to_train_op_spec( subnetwork_builder.build_mixture_weights_train_op( loss=adanet_loss, var_list=ensemble_var_list, logits=ensemble.logits, labels=labels, iteration_step=step_tensor, summary=summary)) else: train_op = _to_train_op_spec( ensembler.build_train_op( ensemble=ensemble, loss=adanet_loss, var_list=ensemble_var_list, labels=labels, iteration_step=step_tensor, summary=summary, previous_ensemble=previous_ensemble)) return _EnsembleSpec( name=name, architecture=architecture, subnetwork_builders=subnetwork_builders, ensemble=ensemble, predictions=estimator_spec.predictions, step=step, loss=ensemble_loss, adanet_loss=adanet_loss, train_op=train_op, eval_metrics=ensemble_metrics.eval_metrics_tuple(), export_outputs=estimator_spec.export_outputs)
def _TranslatePolicy(self, pol, exp_info): self.cisco_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) # a mixed filter outputs both ipv4 and ipv6 acls in the same output file good_filters = ['extended', 'standard', 'object-group', 'inet6', 'mixed', 'enable_dsmo'] for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue obj_target = ObjectGroup() filter_options = header.FilterOptions(self._PLATFORM) filter_name = header.FilterName(self._PLATFORM) # extended is the most common filter type. filter_type = 'extended' if len(filter_options) > 1: filter_type = filter_options[1] # check if filter type is renderable if filter_type not in good_filters: raise UnsupportedCiscoAccessListError( 'access list type %s not supported by %s (good types: %s)' % ( filter_type, self._PLATFORM, str(good_filters))) filter_list = [filter_type] if filter_type == 'mixed': # Loop through filter and generate output for inet and inet6 in sequence filter_list = ['extended', 'inet6'] for next_filter in filter_list: # Numeric access lists can be extended or standard, but have specific # known ranges. if next_filter == 'extended' and filter_name.isdigit(): if int(filter_name) in range(1, 100) + range(1300, 2000): raise UnsupportedCiscoAccessListError( 'Access lists between 1-99 and 1300-1999 are reserved for ' 'standard ACLs') if next_filter == 'standard' and filter_name.isdigit(): if int(filter_name) not in range(1, 100) + range(1300, 2000): raise UnsupportedCiscoAccessListError( 'Standard access lists must be numeric in the range of 1-99' ' or 1300-1999.') term_dup_check = set() new_terms = [] for term in terms: if term.name in term_dup_check: raise CiscoDuplicateTermError('You have a duplicate term: %s' % term.name) term_dup_check.add(term.name) term.name = self.FixTermLength(term.name) af = 'inet' if next_filter == 'inet6': af = 'inet6' term = self.FixHighPorts(term, af=af) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue # render terms based on filter type if next_filter == 'standard': # keep track of sequence numbers across terms new_terms.append(TermStandard(term, filter_name, self._PLATFORM)) elif next_filter == 'extended': enable_dsmo = (len(filter_options) > 2 and filter_options[2] == 'enable_dsmo') new_terms.append( Term(term, proto_int=self._PROTO_INT, enable_dsmo=enable_dsmo, term_remark=self._TERM_REMARK, platform=self._PLATFORM)) elif next_filter == 'object-group': obj_target.AddTerm(term) self._SetObjectGroupProtos(ObjectGroupTerm) obj_group_term = ObjectGroupTerm(term, filter_name) new_terms.append(obj_group_term) elif next_filter == 'inet6': new_terms.append( Term( term, 6, proto_int=self._PROTO_INT, platform=self._PLATFORM)) # cisco requires different name for the v4 and v6 acls if filter_type == 'mixed' and next_filter == 'inet6': filter_name = 'ipv6-%s' % filter_name self.cisco_policies.append((header, filter_name, [next_filter], new_terms, obj_target))
def _TranslatePolicy(self, pol, exp_info): # pylint: disable=attribute-defined-outside-init """Transform a policy object into a JuniperSRX object. Args: pol: policy.Policy object exp_info: print a info message when a term is set to expire in that many weeks Raises: UnsupportedFilterError: An unsupported filter was specified UnsupportedHeader: A header option exists that is not understood/usable SRXDuplicateTermError: Two terms were found with same name in same filter ConflictingTargetOptions: Two target options are conflicting in the header MixedAddrBookTypes: Global and Zone address books in the same policy ConflictingApplicationSets: When two duplicate named terms have conflicting application entries """ self.srx_policies = [] self.addressbook = collections.OrderedDict() self.applications = [] self.ports = [] self.from_zone = '' self.to_zone = '' self.addr_book_type = set() current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) if (len(filter_options) < 4 or filter_options[0] != 'from-zone' or filter_options[2] != 'to-zone'): raise UnsupportedFilterError( 'SRX filter arguments must specify ' 'from-zone and to-zone.') # check if to-zone is not a supported target option if filter_options[1] in self._SUPPORTED_TARGET_OPTIONS: raise UnsupportedFilterError( 'to-zone %s cannot be the same as any ' 'valid SRX target-options' % (filter_options[1])) else: self.from_zone = filter_options[1] # check if from-zone is not a supported target option if filter_options[3] in self._SUPPORTED_TARGET_OPTIONS: raise UnsupportedFilterError( 'from-zone %s cannot be the same as any ' 'valid SRX target-options' % (filter_options[3])) else: self.to_zone = filter_options[3] # variables used to collect target-options and set defaults target_options = [] filter_type = '' # parse srx target options extra_options = filter_options[4:] if self._SUPPORTED_TARGET_OPTIONS.issubset(extra_options): raise ConflictingTargetOptions( 'only one address-book-type can ' 'be specified per header "%s"' % ' '.join(filter_options)) else: address_book_type = self._SUPPORTED_TARGET_OPTIONS.intersection( extra_options) if len(address_book_type) is 0: address_book_type = {self._GLOBAL_ADDR_BOOK} self.addr_book_type.update(address_book_type) if len(self.addr_book_type) > 1: raise MixedAddrBookTypes( 'Global and Zone address-book-types cannot ' 'be used in the same policy') if self.from_zone == 'all' and self.to_zone == 'all': if self._ZONE_ADDR_BOOK in self.addr_book_type: raise UnsupportedFilterError( 'Zone address books cannot be used ' 'with a global policy.') elif self.from_zone == 'all' or self.to_zone == 'all': raise UnsupportedFilterError( 'The zone name all is reserved for ' 'global policies.') for filter_opt in filter_options[4:]: # validate address families if filter_opt in self._SUPPORTED_AF: if not filter_type: filter_type = filter_opt else: raise ConflictingTargetOptions( 'only one address family can be ' 'specified per header "%s"' % ' '.join(filter_options)) elif filter_opt in self._SUPPORTED_TARGET_OPTIONS: target_options.append(filter_opt) else: raise UnsupportedHeader( 'SRX Generator currently does not support ' '%s as a header option "%s"' % (filter_opt, ' '.join(filter_options))) # if address-family and address-book-type have not been set then default if not filter_type: filter_type = 'mixed' term_dup_check = set() new_terms = [] self._FixLargePolices(terms, filter_type) for term in terms: term.name = self.FixTermLength(term.name) if term.name in term_dup_check: raise SRXDuplicateTermError( 'You have a duplicate term: %s' % term.name) term_dup_check.add(term.name) if term.expiration: if term.expiration <= exp_info_date: logging.info( 'INFO: Term %s in policy %s>%s expires ' 'in less than two weeks.', term.name, self.from_zone, self.to_zone) if term.expiration <= current_date: logging.warn( 'WARNING: Term %s in policy %s>%s is expired.', term.name, self.from_zone, self.to_zone) continue # SRX address books leverage network token names for IPs. # When excluding addresses, we lose those distinct names so we need # to create a new unique name based off the term name before excluding. if term.source_address_exclude: # If we have a naked source_exclude, we need something to exclude from if not term.source_address: term.source_address = [ nacaddr.IP('0.0.0.0/0', term.name.upper(), term.name.upper()) ] # Use the term name as the token & parent_token new_src_parent_token = term.name.upper() + '_SRC_EXCLUDE' new_src_token = new_src_parent_token for i in term.source_address_exclude: term.source_address = nacaddr.RemoveAddressFromList( term.source_address, i) for i in term.source_address: i.token = new_src_token i.parent_token = new_src_parent_token if term.destination_address_exclude: if not term.destination_address: term.destination_address = [ nacaddr.IP('0.0.0.0/0', term.name.upper(), term.name.upper()) ] new_dst_parent_token = term.name.upper() + '_DST_EXCLUDE' new_dst_token = new_dst_parent_token for i in term.destination_address_exclude: term.destination_address = nacaddr.RemoveAddressFromList( term.destination_address, i) for i in term.destination_address: i.token = new_dst_token i.parent_token = new_dst_parent_token # SRX policies are controlled by addresses that are used within, so # policy can be at the same time inet and inet6. if self._GLOBAL_ADDR_BOOK in self.addr_book_type: for zone in self.addressbook: for unused_name, ips in sorted( self.addressbook[zone].iteritems()): ips = [i for i in ips] if term.source_address == ips: term.source_address = ips if term.destination_address == ips: term.destination_address = ips for addr in term.source_address: if addr.version in self._AF_MAP[filter_type]: self._BuildAddressBook(self.from_zone, addr) for addr in term.destination_address: if addr.version in self._AF_MAP[filter_type]: self._BuildAddressBook(self.to_zone, addr) new_term = Term(term, filter_options) new_terms.append(new_term) # Because SRX terms can contain inet and inet6 addresses. We have to # have ability to recover proper AF for ICMP type we need. # If protocol is empty or we cannot map to inet or inet6 we insert bogus # af_type name which will cause new_term.NormalizeIcmpTypes to fail. if not term.protocol: icmp_af_type = 'unknown_af_icmp' else: icmp_af_type = self._AF_ICMP_MAP.get( term.protocol[0], 'unknown_af_icmp') tmp_icmptype = new_term.NormalizeIcmpTypes( term.icmp_type, term.protocol, icmp_af_type) # NormalizeIcmpTypes returns [''] for empty, convert to [] for eval normalized_icmptype = tmp_icmptype if tmp_icmptype != [ '' ] else [] # rewrites the protocol icmpv6 to icmp6 if 'icmpv6' in term.protocol: protocol = list(term.protocol) protocol[protocol.index('icmpv6')] = 'icmp6' else: protocol = term.protocol new_application_set = { 'sport': self._BuildPort(term.source_port), 'dport': self._BuildPort(term.destination_port), 'protocol': protocol, 'icmp-type': normalized_icmptype, 'timeout': term.timeout } for application_set in self.applications: if all(item in application_set.items() for item in new_application_set.items()): new_application_set = '' term.replacement_application_name = application_set[ 'name'] break if (term.name == application_set['name'] and new_application_set != application_set): raise ConflictingApplicationSets( 'Application set %s has a conflicting entry' % term.name) if new_application_set: new_application_set['name'] = term.name self.applications.append(new_application_set) self.srx_policies.append((header, new_terms, filter_options))
def _FixLargePolices(self, terms, address_family): """Loops over all terms finding terms exceeding SRXs policy limit. Args: terms: List of terms from a policy. address_family: Tuple containing address family versions. See the following URL for more information http://www.juniper.net/techpubs/en_US/junos12.1x44/topics/reference/ general/address-address-sets-limitations.html """ def Chunks(l): """Splits a list of IP addresses into smaller lists based on byte size.""" return_list = [[]] counter = 0 index = 0 for i in l: # Size is split in half due to the max size being a sum of src and dst. if counter > (self._ADDRESS_LENGTH_LIMIT/2): counter = 0 index += 1 return_list.append([]) if i.version == 6: counter += self._IPV6_SIZE else: counter += 1 return_list[index].append(i) return return_list expanded_terms = [] for term in terms: if (term.AddressesByteLength( self._AF_MAP[address_family]) > self._ADDRESS_LENGTH_LIMIT): logging.warn('LARGE TERM ENCOUNTERED') src_chunks = Chunks(term.source_address) counter = 0 for chunk in src_chunks: for ip in chunk: ip.parent_token = 'src_' + term.name + str(counter) counter += 1 dst_chunks = Chunks(term.destination_address) counter = 0 for chunk in dst_chunks: for ip in chunk: ip.parent_token = 'dst_' + term.name + str(counter) counter += 1 src_dst_products = itertools.product(src_chunks, dst_chunks) counter = 0 for src_dst_list in src_dst_products: new_term = copy.copy(term) new_term.source_address = src_dst_list[0] new_term.destination_address = src_dst_list[1] new_term.name = new_term.name + '_' + str(counter) expanded_terms.append(new_term) counter += 1 else: expanded_terms.append(term) if expanded_terms: del terms[:] terms.extend(expanded_terms)
def _FixLargePolices(self, terms, address_family): """Loops over all terms finding terms exceeding SRXs policy limit. Args: terms: List of terms from a policy. address_family: Tuple containing address family versions. See the following URL for more information http://www.juniper.net/techpubs/en_US/junos12.1x44/topics/reference/ general/address-address-sets-limitations.html """ def Chunks(l): """Splits a list of IP addresses into smaller lists based on byte size.""" return_list = [[]] counter = 0 index = 0 for i in l: # Size is split in half due to the max size being a sum of src and dst. if counter > (self._ADDRESS_LENGTH_LIMIT / 2): counter = 0 index += 1 return_list.append([]) if i.version == 6: counter += self._IPV6_SIZE else: counter += 1 return_list[index].append(i) return return_list expanded_terms = [] for term in terms: if (term.AddressesByteLength(self._AF_MAP[address_family]) > self._ADDRESS_LENGTH_LIMIT): logging.warn('LARGE TERM ENCOUNTERED') src_chunks = Chunks(term.source_address) counter = 0 for chunk in src_chunks: for ip in chunk: ip.parent_token = 'src_' + term.name + str(counter) counter += 1 dst_chunks = Chunks(term.destination_address) counter = 0 for chunk in dst_chunks: for ip in chunk: ip.parent_token = 'dst_' + term.name + str(counter) counter += 1 src_dst_products = itertools.product(src_chunks, dst_chunks) counter = 0 for src_dst_list in src_dst_products: new_term = copy.copy(term) new_term.source_address = src_dst_list[0] new_term.destination_address = src_dst_list[1] new_term.name = new_term.name + '_' + str(counter) expanded_terms.append(new_term) counter += 1 else: expanded_terms.append(term) if expanded_terms: del terms[:] terms.extend(expanded_terms)
def __str__(self): output = [] # Don't render term if not in platforms or in excluded platforms if self.term.platform and self._PLATFORM not in self.term.platform: return '' if (self.term.platform_exclude and self._PLATFORM in self.term.platform_exclude): return '' # Don't render icmpv6 protocol terms under inet, or icmp under inet6 # Does not currently support mixed family. if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or (self.af == 'inet' and 'icmpv6' in self.term.protocol)): logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name, proto=self.term.protocol, af=self.af)) return '' # Term verbatim output - this will skip over most normal term # creation code by returning early. Warnings provided in policy.py. if self.term.verbatim: for verbatim_line in self.term.verbatim: platform, contents = verbatim_line.value if platform == self._PLATFORM: output.append(str(contents)) return '\n'.join(output) # Source address if self.term.source_address or self.term.source_address_exclude: src_addrs = self._CalculateAddrs(self.term.source_address, self.term.source_address_exclude) if not src_addrs: logging.warn(self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='source', af=self.af)) return '' # TODO(castagno): Add support for ipv6 output.append('ip saddr %s' % self._FormatMatch(src_addrs)) # Destination address if self.term.destination_address or self.term.source_address_exclude: dst_addrs = self._CalculateAddrs(self.term.destination_address, self.term.destination_address_exclude) if not dst_addrs: logging.warn(self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='destination', af=self.af)) return '' # TODO(castagno): Add support for ipv6 output.append('ip daddr %s' % self._FormatMatch(dst_addrs)) # Protocol # # nft intepreter shortcuts protocol specification if there are more specific # matches. At the moment, these are: # * source port # * destination port # * ICMP type if self.term.protocol and not (self.term.source_port or self.term.destination_port or self.term.icmp_type): output.append('ip protocol %s' % self._FormatMatch(self.term.protocol)) # Source port if self.term.source_port: output.append('%s sport %s' % (self._FormatMatch(self.term.protocol), self._FormatMatch(self.term.source_port))) # Destination port if self.term.destination_port: output.append('%s dport %s' % (self._FormatMatch(self.term.protocol), self._FormatMatch(self.term.destination_port))) # Icmp type if self.term.icmp_type: icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, self.term.protocol, self.af) if icmp_types != ['']: # nft intepreter requires ICMP types to be spelled out icmp_name_types = self.ICMP_TYPE[self.AF_MAP[self.af]] icmp_type_names = dict((v, k) for k, v in icmp_name_types.iteritems()) output.append('icmp type %s' % self._FormatMatch([icmp_type_names[icmp_type] for icmp_type in icmp_types])) # Counter # This does not use the value that was passed in the term. if self.term.counter: output.append('counter') # Log # Setup logic so that only one log statement is printed. if self.term.logging and not self.term.log_name: output.append('log') elif (self.term.logging and self.term.log_name) or self.term.log_name: # Only supports log prefix's of 128 characters truncate to 126 to support # the additional suffix that is being added output.append('log prefix "%s: "' % self.term.log_name[:126]) # Action output.append(self._ACTIONS[self.term.action[0]]) # Owner (implement as comment) if self.term.owner: self.term.comment.append('Owner: %s' % self.term.owner) # Comment if self.term.comment: comment_data = ' '.join(self.term.comment) # Have to truncate MAX_CHARACTERS characters due to NFTables limitation if len(comment_data) > self.MAX_CHARACTERS: # Have to use the first MAX_CHARACTERS characters comment_data = comment_data[:self.MAX_CHARACTERS] logging.warn( 'Term %s in policy is too long (>%d characters) ' 'and will be truncated', self.term.name, self.MAX_CHARACTERS) output.append('comment "%s"' % comment_data) return ' '.join(output)
def start_aip_training(input_dict: Dict[Text, List[types.Artifact]], output_dict: Dict[Text, List[types.Artifact]], exec_properties: Dict[Text, Any], executor_class_path: Text, training_inputs: Dict[Text, Any], job_id: Optional[Text]): """Start a trainer job on AI Platform (AIP). This is done by forwarding the inputs/outputs/exec_properties to the tfx.scripts.run_executor module on a AI Platform training job interpreter. Args: input_dict: Passthrough input dict for tfx.components.Trainer.executor. output_dict: Passthrough input dict for tfx.components.Trainer.executor. exec_properties: Passthrough input dict for tfx.components.Trainer.executor. executor_class_path: class path for TFX core default trainer. training_inputs: Training input argument for AI Platform training job. 'pythonModule', 'pythonVersion' and 'runtimeVersion' will be inferred. For the full set of parameters, refer to https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs#TrainingInput job_id: Job ID for AI Platform Training job. If not supplied, system-determined unique ID is given. Refer to https://cloud.google.com/ml-engine/reference/rest/v1/projects.jobs#resource-job Returns: None """ training_inputs = training_inputs.copy() json_inputs = artifact_utils.jsonify_artifact_dict(input_dict) logging.info('json_inputs=\'%s\'.', json_inputs) json_outputs = artifact_utils.jsonify_artifact_dict(output_dict) logging.info('json_outputs=\'%s\'.', json_outputs) json_exec_properties = json.dumps(exec_properties, sort_keys=True) logging.info('json_exec_properties=\'%s\'.', json_exec_properties) # We use custom containers to launch training on AI Platform, which invokes # the specified image using the container's entrypoint. The default # entrypoint for TFX containers is to call scripts/run_executor.py. The # arguments below are passed to this run_executor entry to run the executor # specified in `executor_class_path`. container_command = _CONTAINER_COMMAND + [ '--executor_class_path', executor_class_path, '--inputs', json_inputs, '--outputs', json_outputs, '--exec-properties', json_exec_properties, ] if not training_inputs.get('masterConfig'): training_inputs['masterConfig'] = { 'imageUri': _TFX_IMAGE, } # Always use our own entrypoint instead of relying on container default. if 'containerCommand' in training_inputs['masterConfig']: logging.warn('Overriding custom value of containerCommand') training_inputs['masterConfig']['containerCommand'] = container_command # Pop project_id so AIP doesn't complain about an unexpected parameter. # It's been a stowaway in aip_args and has finally reached its destination. project = training_inputs.pop('project') with telemetry_utils.scoped_labels( {telemetry_utils.LABEL_TFX_EXECUTOR: executor_class_path}): job_labels = telemetry_utils.get_labels_dict() # 'tfx_YYYYmmddHHMMSS' is the default job ID if not explicitly specified. job_id = job_id or 'tfx_{}'.format( datetime.datetime.now().strftime('%Y%m%d%H%M%S')) _launch_aip_training( job_id=job_id, project=project, training_input=training_inputs, job_labels=job_labels)
def _TranslatePolicy(self, pol, exp_info): """Translate a policy from objects into strings.""" self.iptables_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) default_action = None good_default_actions = ['ACCEPT', 'DROP'] good_afs = ['inet', 'inet6'] good_options = ['nostate', 'abbreviateterms', 'truncateterms', 'noverbose'] all_protocols_stateful = True self.verbose = True for header, terms in pol.filters: filter_type = None if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM)[1:] filter_name = header.FilterName(self._PLATFORM) self._WarnIfCustomTarget(filter_name) # ensure all options after the filter name are expected for opt in filter_options: if opt not in good_default_actions + good_afs + good_options: raise UnsupportedTargetOption('%s %s %s %s' % ( '\nUnsupported option found in', self._PLATFORM, 'target definition:', opt)) # disable stateful? if 'nostate' in filter_options: all_protocols_stateful = False if 'noverbose' in filter_options: self.verbose = False # Check for matching af for address_family in good_afs: if address_family in filter_options: # should not specify more than one AF in options if filter_type is not None: raise UnsupportedFilterError('%s %s %s %s' % ( '\nMay only specify one of', good_afs, 'in filter options:', filter_options)) filter_type = address_family if filter_type is None: filter_type = 'inet' if self._PLATFORM == 'iptables' and filter_name == 'FORWARD': default_action = 'DROP' # does this policy override the default filter actions? for next_target in header.target: if next_target.platform == self._PLATFORM: if len(next_target.options) > 1: for arg in next_target.options: if arg in good_default_actions: default_action = arg if default_action and default_action not in good_default_actions: raise UnsupportedDefaultAction('%s %s %s %s %s' % ( '\nOnly', ', '.join(good_default_actions), 'default filter action allowed;', default_action, 'used.')) # add the terms new_terms = [] term_names = set() for term in terms: term.name = self.FixTermLength(term.name, 'abbreviateterms' in filter_options, 'truncateterms' in filter_options) if term.name in term_names: raise aclgenerator.DuplicateTermError( 'You have a duplicate term: %s' % term.name) term_names.add(term.name) term = self.FixHighPorts(term, af=filter_type, all_protocols_stateful=all_protocols_stateful) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue new_terms.append(self._TERM(term, filter_name, all_protocols_stateful, default_action, filter_type, self.verbose)) self.iptables_policies.append((header, filter_name, filter_type, default_action, new_terms))
def main(_): """Runs fine-tuning and inference. There are three categories of images. 1) Images where we have previous and next frame, and that are not filtered out by the heuristic. For them, we will use the fine-tuned predictions. 2) Images where we have previous and next frame, but that were filtered out by our heuristic. For them, we will use the ordinary prediction instead. 3) Images where we have at least one missing adjacent frame. For them, we will use the ordinary prediction as indicated by triplet_list_file_remains (if provided). They will also not be part of the generated inference list in the first place. Raises: ValueError: Invalid parameters have been passed. """ if FLAGS.handle_motion and FLAGS.joint_encoder: raise ValueError('Using a joint encoder is currently not supported when ' 'modeling object motion.') if FLAGS.handle_motion and FLAGS.seq_length != 3: raise ValueError('The current motion model implementation only supports ' 'using a sequence length of three.') if FLAGS.handle_motion and not FLAGS.compute_minimum_loss: raise ValueError('Computing the minimum photometric loss is required when ' 'enabling object motion handling.') if FLAGS.size_constraint_weight > 0 and not FLAGS.handle_motion: raise ValueError('To enforce object size constraints, enable motion ' 'handling.') if FLAGS.icp_weight > 0.0: raise ValueError('ICP is currently not supported.') if FLAGS.compute_minimum_loss and FLAGS.seq_length % 2 != 1: raise ValueError('Compute minimum loss requires using an odd number of ' 'images in a sequence.') if FLAGS.compute_minimum_loss and FLAGS.exhaustive_mode: raise ValueError('Exhaustive mode has no effect when compute_minimum_loss ' 'is enabled.') if FLAGS.img_width % (2 ** 5) != 0 or FLAGS.img_height % (2 ** 5) != 0: logging.warn('Image size is not divisible by 2^5. For the architecture ' 'employed, this could cause artefacts caused by resizing in ' 'lower dimensions.') if FLAGS.output_dir.endswith('/'): FLAGS.output_dir = FLAGS.output_dir[:-1] # Create file lists to prepare fine-tuning, save it to unique_file. unique_file_name = (str(datetime.datetime.now().date()) + '_' + str(datetime.datetime.now().time()).replace(':', '_')) unique_file = os.path.join(FLAGS.data_dir, unique_file_name + '.txt') with gfile.FastGFile(FLAGS.triplet_list_file, 'r') as f: files_to_process = f.readlines() files_to_process = [line.rstrip() for line in files_to_process] files_to_process = [line for line in files_to_process if len(line)] logging.info('Creating unique file list %s with %s entries.', unique_file, len(files_to_process)) with gfile.FastGFile(unique_file, 'w') as f_out: fetches_network = FLAGS.num_steps * FLAGS.batch_size fetches_saves = FLAGS.batch_size * int(np.floor(FLAGS.num_steps/SAVE_EVERY)) repetitions = fetches_network + 3 * fetches_saves for i in range(len(files_to_process)): for _ in range(repetitions): f_out.write(files_to_process[i] + '\n') # Read remaining files. remaining = [] if gfile.Exists(FLAGS.triplet_list_file_remains): with gfile.FastGFile(FLAGS.triplet_list_file_remains, 'r') as f: remaining = f.readlines() remaining = [line.rstrip() for line in remaining] remaining = [line for line in remaining if len(line)] logging.info('Running fine-tuning on %s files, %s files are remaining.', len(files_to_process), len(remaining)) # Run fine-tuning process and save predictions in id-folders. tf.set_random_seed(FIXED_SEED) np.random.seed(FIXED_SEED) random.seed(FIXED_SEED) flipping_mode = reader.FLIP_ALWAYS if FLAGS.flip else reader.FLIP_NONE train_model = model.Model(data_dir=FLAGS.data_dir, file_extension=FLAGS.file_extension, is_training=True, learning_rate=FLAGS.learning_rate, beta1=FLAGS.beta1, reconstr_weight=FLAGS.reconstr_weight, smooth_weight=FLAGS.smooth_weight, ssim_weight=FLAGS.ssim_weight, icp_weight=FLAGS.icp_weight, batch_size=FLAGS.batch_size, img_height=FLAGS.img_height, img_width=FLAGS.img_width, seq_length=FLAGS.seq_length, architecture=FLAGS.architecture, imagenet_norm=FLAGS.imagenet_norm, weight_reg=FLAGS.weight_reg, exhaustive_mode=FLAGS.exhaustive_mode, random_scale_crop=FLAGS.random_scale_crop, flipping_mode=flipping_mode, random_color=False, depth_upsampling=FLAGS.depth_upsampling, depth_normalization=FLAGS.depth_normalization, compute_minimum_loss=FLAGS.compute_minimum_loss, use_skip=FLAGS.use_skip, joint_encoder=FLAGS.joint_encoder, build_sum=False, shuffle=False, input_file=unique_file_name, handle_motion=FLAGS.handle_motion, size_constraint_weight=FLAGS.size_constraint_weight, train_global_scale_var=False) failed_heuristic_ids = finetune_inference(train_model, FLAGS.model_ckpt, FLAGS.output_dir + '_ft') logging.info('Fine-tuning completed, %s files were filtered out by ' 'heuristic.', len(failed_heuristic_ids)) for failed_id in failed_heuristic_ids: failed_entry = files_to_process[failed_id] remaining.append(failed_entry) logging.info('In total, %s images were fine-tuned, while %s were not.', len(files_to_process)-len(failed_heuristic_ids), len(remaining)) # Copy all results to have the same structural output as running ordinary # inference. for i in range(len(files_to_process)): if files_to_process[i] not in remaining: # Use fine-tuned result. elements = files_to_process[i].split(' ') source_file = os.path.join(FLAGS.output_dir + '_ft', FLAGS.ft_name + 'id_' + str(i), str(FLAGS.num_steps).zfill(10) + ('_flip' if FLAGS.flip else '')) if len(elements) == 2: # No differing mapping defined. target_dir = os.path.join(FLAGS.output_dir + '_ft', elements[0]) target_file = os.path.join( target_dir, elements[1] + ('_flip' if FLAGS.flip else '')) else: # Other mapping for file defined, copy to this location instead. target_dir = os.path.join( FLAGS.output_dir + '_ft', os.path.dirname(elements[2])) target_file = os.path.join( target_dir, os.path.basename(elements[2]) + ('_flip' if FLAGS.flip else '')) if not gfile.Exists(target_dir): gfile.MakeDirs(target_dir) logging.info('Copy refined result %s to %s.', source_file, target_file) gfile.Copy(source_file + '.npy', target_file + '.npy', overwrite=True) gfile.Copy(source_file + '.txt', target_file + '.txt', overwrite=True) gfile.Copy(source_file + '.%s' % FLAGS.file_extension, target_file + '.%s' % FLAGS.file_extension, overwrite=True) for j in range(len(remaining)): elements = remaining[j].split(' ') if len(elements) == 2: # No differing mapping defined. target_dir = os.path.join(FLAGS.output_dir + '_ft', elements[0]) target_file = os.path.join( target_dir, elements[1] + ('_flip' if FLAGS.flip else '')) else: # Other mapping for file defined, copy to this location instead. target_dir = os.path.join( FLAGS.output_dir + '_ft', os.path.dirname(elements[2])) target_file = os.path.join( target_dir, os.path.basename(elements[2]) + ('_flip' if FLAGS.flip else '')) if not gfile.Exists(target_dir): gfile.MakeDirs(target_dir) source_file = target_file.replace('_ft', '') logging.info('Copy unrefined result %s to %s.', source_file, target_file) gfile.Copy(source_file + '.npy', target_file + '.npy', overwrite=True) gfile.Copy(source_file + '.%s' % FLAGS.file_extension, target_file + '.%s' % FLAGS.file_extension, overwrite=True) logging.info('Done, predictions saved in %s.', FLAGS.output_dir + '_ft')
def _TranslatePolicy(self, pol, exp_info): self.nsxv_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM) if len(filter_options) >= 2: filter_name = filter_options[1] # get filter type, section id and applied To self._ParseFilterOptions(filter_options) filter_type = self._FILTER_OPTIONS_DICT['filter_type'] applied_to = self._FILTER_OPTIONS_DICT['applied_to'] term_names = set() new_terms = [] for term in terms: # Check for duplicate terms if term.name in term_names: raise NsxvDuplicateTermError('There are multiple terms named: %s' % term.name) term_names.add(term.name) if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue # Get the mapped action value # If there is no mapped action value term is not rendered mapped_action = _ACTION_TABLE.get(str(term.action[0])) if not mapped_action: logging.warn('WARNING: Action %s in Term %s is not valid and ' 'will not be rendered.', term.action, term.name) continue term.name = self.FixTermLength(term.name) if filter_type == 'inet': af = 'inet' term = self.FixHighPorts(term, af=af) if not term: continue new_terms.append(Term(term, filter_type, applied_to, 4)) if filter_type == 'inet6': af = 'inet6' term = self.FixHighPorts(term, af=af) if not term: continue new_terms.append(Term(term, filter_type, applied_to, 6)) if filter_type == 'mixed': if 'icmpv6' not in term.protocol: inet_term = self.FixHighPorts(term, 'inet') if not inet_term: continue new_terms.append(Term(inet_term, filter_type, applied_to, 4)) else: inet6_term = self.FixHighPorts(term, 'inet6') if not inet6_term: continue new_terms.append(Term(inet6_term, filter_type, applied_to, 6)) self.nsxv_policies.append((header, filter_name, [filter_type], new_terms))
def _TranslatePolicy(self, pol, exp_info): self.cisco_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) # a mixed filter outputs both ipv4 and ipv6 acls in the same output file good_filters = [ 'extended', 'standard', 'object-group', 'inet6', 'mixed', 'enable_dsmo' ] for header, terms in pol.filters: if self._PLATFORM not in header.platforms: continue obj_target = ObjectGroup() filter_options = header.FilterOptions(self._PLATFORM) filter_name = header.FilterName(self._PLATFORM) self.verbose = True if 'noverbose' in filter_options: filter_options.remove('noverbose') self.verbose = False # extended is the most common filter type. filter_type = 'extended' if len(filter_options) > 1: filter_type = filter_options[1] # check if filter type is renderable if filter_type not in good_filters: raise UnsupportedCiscoAccessListError( 'access list type %s not supported by %s (good types: %s)' % (filter_type, self._PLATFORM, str(good_filters))) filter_list = [filter_type] if filter_type == 'mixed': # Loop through filter and generate output for inet and inet6 in sequence filter_list = ['extended', 'inet6'] for next_filter in filter_list: # Numeric access lists can be extended or standard, but have specific # known ranges. if next_filter == 'extended' and filter_name.isdigit(): if int(filter_name) in list(range(1, 100)) + list( range(1300, 2000)): raise UnsupportedCiscoAccessListError( 'Access lists between 1-99 and 1300-1999 are reserved for ' 'standard ACLs') if next_filter == 'standard' and filter_name.isdigit(): if (int(filter_name) not in list(range(1, 100)) + list(range(1300, 2000))): raise UnsupportedCiscoAccessListError( 'Standard access lists must be numeric in the range of 1-99' ' or 1300-1999.') term_dup_check = set() new_terms = [] for term in terms: if term.name in term_dup_check: raise CiscoDuplicateTermError( 'You have a duplicate term: %s' % term.name) term_dup_check.add(term.name) term.name = self.FixTermLength(term.name) af = 'inet' if next_filter == 'inet6': af = 'inet6' term = self.FixHighPorts(term, af=af) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info( 'INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn( 'WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue # render terms based on filter type if next_filter == 'standard': # keep track of sequence numbers across terms new_terms.append( TermStandard(term, filter_name, self._PLATFORM, self.verbose)) elif next_filter == 'extended': enable_dsmo = (len(filter_options) > 2 and filter_options[2] == 'enable_dsmo') new_terms.append( Term(term, proto_int=self._PROTO_INT, enable_dsmo=enable_dsmo, term_remark=self._TERM_REMARK, platform=self._PLATFORM, verbose=self.verbose)) elif next_filter == 'object-group': obj_target.AddTerm(term) new_terms.append( self._GetObjectGroupTerm(term, filter_name, verbose=self.verbose)) elif next_filter == 'inet6': new_terms.append( Term(term, 6, proto_int=self._PROTO_INT, platform=self._PLATFORM, verbose=self.verbose)) # cisco requires different name for the v4 and v6 acls if filter_type == 'mixed' and next_filter == 'inet6': filter_name = 'ipv6-%s' % filter_name self.cisco_policies.append((header, filter_name, [next_filter], new_terms, obj_target))
def deploy_model_for_aip_prediction(api: discovery.Resource, serving_path: Text, model_version: Text, ai_platform_serving_args: Dict[Text, Any], job_labels: Dict[Text, Text], skip_model_creation: bool = False, set_default_version: bool = True) -> None: """Deploys a model for serving with AI Platform. Args: api: Google API client resource. serving_path: The path to the model. Must be a GCS URI. model_version: Version of the model being deployed. Must be different from what is currently being served. ai_platform_serving_args: Dictionary containing arguments for pushing to AI Platform. The full set of parameters supported can be found at https://cloud.google.com/ml-engine/reference/rest/v1/projects.models.versions#Version. Most keys are forwarded as-is, but following keys are handled specially: - name: this must be empty (and will be filled by pusher). - deployment_uri: this must be empty (and will be filled by pusher). - python_version: when left empty, this will be filled by python version of the environment being used. - runtime_version: when left empty, this will be filled by TensorFlow version from the environment. - labels: a list of job labels will be merged with user's input. job_labels: The dict of labels that will be attached to this job. They are merged with optional labels from `ai_platform_serving_args`. skip_model_creation: If true, the method assuem model already exist in AI platform, therefore skipping model creation. set_default_version: Whether set the newly deployed model version as the default version. Raises: RuntimeError: if an error is encountered when trying to push. """ logging.info( 'Deploying to model with version %s to AI Platform for serving: %s', model_version, ai_platform_serving_args) model_name = ai_platform_serving_args['model_name'] project_id = ai_platform_serving_args['project_id'] default_runtime_version = _get_tf_runtime_version(tf.__version__) runtime_version = ai_platform_serving_args.get('runtime_version', default_runtime_version) python_version = _get_caip_python_version(runtime_version) if not skip_model_creation: create_model_for_aip_prediction_if_not_exist(api, job_labels, ai_platform_serving_args) version_body = dict(ai_platform_serving_args) for model_only_key in ['model_name', 'project_id', 'regions']: version_body.pop(model_only_key, None) version_body['name'] = model_version version_body['deployment_uri'] = serving_path version_body['runtime_version'] = version_body.get('runtime_version', runtime_version) version_body['python_version'] = version_body.get('python_version', python_version) version_body['labels'] = {**version_body.get('labels', {}), **job_labels} logging.info( 'Creating new version of model_name %s in project %s, request body: %s', model_name, project_id, version_body) # Push to AIP, and record the operation name so we can poll for its state. model_name = 'projects/{}/models/{}'.format(project_id, model_name) try: operation = api.projects().models().versions().create( body=version_body, parent=model_name).execute() _wait_for_operation(api, operation, 'projects.models.versions.create') except errors.HttpError as e: # If the error is to create an already existing model version, it's ok to # ignore. if e.resp.status == 409: logging.warn('Model version %s already exists', model_version) else: raise RuntimeError('Creating model verseion to AI Platform failed: {}' .format(e)) if set_default_version: # Set the new version as default. # By API specification, if Long-Running-Operation is done and there is # no error, 'response' is guaranteed to exist. api.projects().models().versions().setDefault(name='{}/versions/{}'.format( model_name, model_version)).execute() logging.info( 'Successfully deployed model %s with version %s, serving from %s', model_name, model_version, serving_path)
def utt_to_samples(self, args): ''' Utt to samples of (feat_chunk, spk_id, sample_key). Will be run in a process pool so restrictions apply. ''' result_queue, utt_info = args logging.debug(utt_info) # TODO: wrap into a function or something utt_key, utt_meta = utt_info # Load features and select voiced frames. feat_scp = utt_meta['feat'] feat_mat = kaldiio.load_mat(feat_scp) num_frames_feat, feat_dim = feat_mat.shape vad_scp = utt_meta['vad'] if vad_scp: vad_mat = kaldiio.load_mat(vad_scp) num_frames_vad = vad_mat.shape[0] logging.debug('feat_mat: %s, vad_mat: %s' % (str(feat_mat.shape), str(vad_mat.shape))) if num_frames_feat != num_frames_vad: logging.debug('num_frames_feat != num_frames_vad: %d vs %d' % (num_frames_feat, num_frames_vad)) return None voiced_frames_index = np.where(vad_mat == 1)[0] logging.debug('voiced_frames_index: %s' % (str(voiced_frames_index.shape))) feat_mat_voiced = feat_mat[voiced_frames_index, :] else: # If no VAD info was found, the entire utt will be used. feat_mat_voiced = feat_mat num_frames_voiced = feat_mat_voiced.shape[0] logging.debug('feat_mat_voiced: %s' % (str(feat_mat_voiced.shape))) spk_id = utt_meta['spkid'] logging.debug('Chunk size: %d' % (self.chunk_size)) results = [] chunk_idx = 0 if self.add_random_offset: random_offset = np.random.randint(0, self.chunk_size) else: random_offset = 0 for offset in range(random_offset, num_frames_voiced, self.chunk_size): if self.single_chunk: available = num_frames_voiced - self.chunk_size if available < 0: # No padding. logging.warn('Single chunk mode: available < 0.') break offset = random.randint(0, available) logging.debug('offset = %d' % (offset)) feat_chunk = feat_mat_voiced[offset:offset + self.chunk_size, :] unpadded_frames = feat_chunk.shape[0] if self.pad_chunks and unpadded_frames < self.chunk_size: rel_chunk_len = float(unpadded_frames) / self.chunk_size if rel_chunk_len < self.drop_short_chunks: continue logging.debug('Padding chunk of frames %d ...' % (unpadded_frames)) padded = np.zeros((self.chunk_size, feat_dim), dtype=feat_chunk.dtype) padded[:unpadded_frames, :] = feat_chunk feat_chunk = padded feat_chunk = np.expand_dims(feat_chunk, axis=2) # TODO: not here sample_key = '%s_chunk%02d' % (utt_key, chunk_idx) sample = (feat_chunk, spk_id, sample_key) chunk_idx += 1 results.append(sample) if self.single_chunk: break if result_queue: # queue mode result_queue.put(results) return None # imap mode return results
def main(_): # Fixed seed for repeatability seed = 8964 tf.set_random_seed(seed) np.random.seed(seed) random.seed(seed) if FLAGS.handle_motion and FLAGS.joint_encoder: raise ValueError('Using a joint encoder is currently not supported when ' 'modeling object motion.') if FLAGS.handle_motion and FLAGS.seq_length != 3: raise ValueError('The current motion model implementation only supports ' 'using a sequence length of three.') if FLAGS.handle_motion and not FLAGS.compute_minimum_loss: raise ValueError('Computing the minimum photometric loss is required when ' 'enabling object motion handling.') if FLAGS.size_constraint_weight > 0 and not FLAGS.handle_motion: raise ValueError('To enforce object size constraints, enable motion ' 'handling.') if FLAGS.imagenet_ckpt and not FLAGS.imagenet_norm: logging.warn('When initializing with an ImageNet-pretrained model, it is ' 'recommended to normalize the image inputs accordingly using ' 'imagenet_norm.') if FLAGS.compute_minimum_loss and FLAGS.seq_length % 2 != 1: raise ValueError('Compute minimum loss requires using an odd number of ' 'images in a sequence.') if FLAGS.architecture != nets.RESNET and FLAGS.imagenet_ckpt: raise ValueError('Can only load weights from pre-trained ImageNet model ' 'when using ResNet-architecture.') if FLAGS.compute_minimum_loss and FLAGS.exhaustive_mode: raise ValueError('Exhaustive mode has no effect when compute_minimum_loss ' 'is enabled.') if FLAGS.img_width % (2 ** 5) != 0 or FLAGS.img_height % (2 ** 5) != 0: logging.warn('Image size is not divisible by 2^5. For the architecture ' 'employed, this could cause artefacts caused by resizing in ' 'lower dimensions.') if FLAGS.icp_weight > 0.0: # TODO(casser): Change ICP interface to take matrix instead of vector. raise ValueError('ICP is currently not supported.') if not gfile.Exists(FLAGS.checkpoint_dir): gfile.MakeDirs(FLAGS.checkpoint_dir) train_model = model.Model(data_dir=FLAGS.data_dir, file_extension=FLAGS.file_extension, is_training=True, learning_rate=FLAGS.learning_rate, beta1=FLAGS.beta1, reconstr_weight=FLAGS.reconstr_weight, smooth_weight=FLAGS.smooth_weight, ssim_weight=FLAGS.ssim_weight, icp_weight=FLAGS.icp_weight, batch_size=FLAGS.batch_size, img_height=FLAGS.img_height, img_width=FLAGS.img_width, seq_length=FLAGS.seq_length, architecture=FLAGS.architecture, imagenet_norm=FLAGS.imagenet_norm, weight_reg=FLAGS.weight_reg, exhaustive_mode=FLAGS.exhaustive_mode, random_scale_crop=FLAGS.random_scale_crop, flipping_mode=FLAGS.flipping_mode, depth_upsampling=FLAGS.depth_upsampling, depth_normalization=FLAGS.depth_normalization, compute_minimum_loss=FLAGS.compute_minimum_loss, use_skip=FLAGS.use_skip, joint_encoder=FLAGS.joint_encoder, handle_motion=FLAGS.handle_motion, equal_weighting=FLAGS.equal_weighting, size_constraint_weight=FLAGS.size_constraint_weight) train(train_model, FLAGS.pretrained_ckpt, FLAGS.imagenet_ckpt, FLAGS.checkpoint_dir, FLAGS.train_steps, FLAGS.summary_freq)
def RenderFile(input_file, output_directory, definitions, exp_info, write_files): """Render a single file. Args: input_file: the name of the input policy file. output_directory: the directory in which we place the rendered file. definitions: the definitions from naming.Naming(). exp_info: print a info message when a term is set to expire in that many weeks. write_files: a list of file tuples, (output_file, acl_text), to write """ logging.debug('rendering file: %s into %s', input_file, output_directory) pol = None jcl = False acl = False asacl = False aacl = False bacl = False eacl = False gcefw = False ips = False ipt = False spd = False nsx = False pcap_accept = False pcap_deny = False pf = False srx = False jsl = False nft = False win_afw = False xacl = False paloalto = False try: conf = open(input_file).read() logging.debug('opened and read %s', input_file) except IOError as e: logging.warn('bad file: \n%s', e) raise try: pol = policy.ParsePolicy( conf, definitions, optimize=FLAGS.optimize, base_dir=FLAGS.base_directory, shade_check=FLAGS.shade_check) except policy.ShadingError as e: logging.warn('shading errors for %s:\n%s', input_file, e) return except (policy.Error, naming.Error): raise ACLParserError('Error parsing policy file %s:\n%s%s' % ( input_file, sys.exc_info()[0], sys.exc_info()[1])) platforms = set() for header in pol.headers: platforms.update(header.platforms) if 'juniper' in platforms: jcl = copy.deepcopy(pol) if 'cisco' in platforms: acl = copy.deepcopy(pol) if 'ciscoasa' in platforms: asacl = copy.deepcopy(pol) if 'brocade' in platforms: bacl = copy.deepcopy(pol) if 'arista' in platforms: eacl = copy.deepcopy(pol) if 'aruba' in platforms: aacl = copy.deepcopy(pol) if 'ipset' in platforms: ips = copy.deepcopy(pol) if 'iptables' in platforms: ipt = copy.deepcopy(pol) if 'nsxv' in platforms: nsx = copy.deepcopy(pol) if 'packetfilter' in platforms: pf = copy.deepcopy(pol) if 'pcap' in platforms: pcap_accept = copy.deepcopy(pol) pcap_deny = copy.deepcopy(pol) if 'speedway' in platforms: spd = copy.deepcopy(pol) if 'srx' in platforms: srx = copy.deepcopy(pol) if 'srxlo' in platforms: jsl = copy.deepcopy(pol) if 'windows_advfirewall' in platforms: win_afw = copy.deepcopy(pol) if 'ciscoxr' in platforms: xacl = copy.deepcopy(pol) if 'nftables' in platforms: nft = copy.deepcopy(pol) if 'gce' in platforms: gcefw = copy.deepcopy(pol) if 'paloalto' in platforms: paloalto = copy.deepcopy(pol) if not output_directory.endswith('/'): output_directory += '/' try: if jcl: acl_obj = juniper.Juniper(jcl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if srx: acl_obj = junipersrx.JuniperSRX(srx, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if acl: acl_obj = cisco.Cisco(acl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if asacl: acl_obj = ciscoasa.CiscoASA(asacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if aacl: acl_obj = aruba.Aruba(aacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if bacl: acl_obj = brocade.Brocade(bacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if eacl: acl_obj = arista.Arista(eacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if ips: acl_obj = ipset.Ipset(ips, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if ipt: acl_obj = iptables.Iptables(ipt, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if nsx: acl_obj = nsxv.Nsxv(nsx, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if spd: acl_obj = speedway.Speedway(spd, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if pcap_accept: acl_obj = pcap.PcapFilter(pcap_accept, exp_info) RenderACL(str(acl_obj), '-accept' + acl_obj.SUFFIX, output_directory, input_file, write_files) if pcap_deny: acl_obj = pcap.PcapFilter(pcap_deny, exp_info, invert=True) RenderACL(str(acl_obj), '-deny' + acl_obj.SUFFIX, output_directory, input_file, write_files) if pf: acl_obj = packetfilter.PacketFilter(pf, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if win_afw: acl_obj = windows_advfirewall.WindowsAdvFirewall(win_afw, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if jsl: acl_obj = srxlo.SRXlo(jsl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if xacl: acl_obj = ciscoxr.CiscoXR(xacl, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if nft: acl_obj = nftables.Nftables(nft, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if gcefw: acl_obj = gce.GCE(gcefw, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) if paloalto: acl_obj = paloaltofw.PaloAltoFW(paloalto, exp_info) RenderACL(str(acl_obj), acl_obj.SUFFIX, output_directory, input_file, write_files) # TODO(robankeny) add additional errors. except (juniper.Error, junipersrx.Error, cisco.Error, ipset.Error, iptables.Error, speedway.Error, pcap.Error, aclgenerator.Error, aruba.Error, nftables.Error, gce.Error) as e: raise ACLGeneratorError( 'Error generating target ACL for %s:\n%s' % (input_file, e))
def _TranslatePolicy(self, pol, exp_info): self.pf_policies = [] self.address_book = {} self.def_short_to_long = {} current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) good_afs = ['inet', 'inet6', 'mixed'] good_options = ['in', 'out', 'nostate'] all_protocols_stateful = True for header, terms in pol.filters: filter_type = None if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM)[1:] filter_name = header.FilterName(self._PLATFORM) direction = '' # ensure all options after the filter name are expected for opt in filter_options: if opt not in good_afs + good_options: raise aclgenerator.UnsupportedTargetOption('%s %s %s %s' % ( '\nUnsupported option found in', self._PLATFORM, 'target definition:', opt)) # pf will automatically add 'keep state flags S/SA' to all TCP connections # by default. if 'nostate' in filter_options: all_protocols_stateful = False if 'in' in filter_options: direction = 'in' elif 'out' in filter_options: direction = 'out' # Check for matching af for address_family in good_afs: if address_family in filter_options: # should not specify more than one AF in options if filter_type is not None: raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % ( '\nMay only specify one of', good_afs, 'in filter options:', filter_options)) filter_type = address_family if filter_type is None: filter_type = 'inet' # add the terms new_terms = [] term_names = set() for term in terms: term.name = self.FixTermLength(term.name) if term.name in term_names: raise DuplicateTermError( 'You have a duplicate term: %s' % term.name) term_names.add(term.name) for source_addr in term.source_address: src_token = source_addr.parent_token[:self._DEF_MAX_LENGTH] if (src_token in self.def_short_to_long and self.def_short_to_long[src_token] != source_addr.parent_token): raise DuplicateShortenedTableName( 'There is a shortened name conflict between names %s and %s ' '(different named objects would conflict when shortened to %s)' % (self.def_short_to_long[src_token], source_addr.parent_token, src_token)) else: self.def_short_to_long[src_token] = source_addr.parent_token if src_token not in self.address_book: self.address_book[src_token] = set([source_addr]) else: self.address_book[src_token].add(source_addr) for dest_addr in term.destination_address: dst_token = dest_addr.parent_token[:self._DEF_MAX_LENGTH] if (dst_token in self.def_short_to_long and self.def_short_to_long[dst_token] != dest_addr.parent_token): raise DuplicateShortenedTableName( 'There is a shortened name conflict between names %s and %s ' '(different named objects would conflict when shortened to %s)' %(self.def_short_to_long[dst_token], dest_addr.parent_token, dst_token)) else: self.def_short_to_long[dst_token] = dest_addr.parent_token if dst_token not in self.address_book: self.address_book[dst_token] = set([dest_addr]) else: self.address_book[dst_token].add(dest_addr) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue new_terms.append(self._TERM(term, filter_name, all_protocols_stateful, filter_type, direction)) self.pf_policies.append((header, filter_name, filter_type, new_terms))
def __str__(self): # Verify platform specific terms. Skip whole term if platform does not # match. if self.term.platform: if self._PLATFORM not in self.term.platform: return '' if self.term.platform_exclude: if self._PLATFORM in self.term.platform_exclude: return '' config = Config(indent=self._DEFAULT_INDENT) from_str = [] # Don't render icmpv6 protocol terms under inet, or icmp under inet6 if ((self.term_type == 'inet6' and 'icmp' in self.term.protocol) or (self.term_type == 'inet' and 'icmpv6' in self.term.protocol)): logging.debug(self.NO_AF_LOG_PROTO.substitute(term=self.term.name, proto=self.term.protocol, af=self.term_type)) return '' # comment # this deals just fine with multi line comments, but we could probably # output them a little cleaner; do things like make sure the # len(output) < 80, etc. Note, if 'noverbose' is set for the filter, skip # all comment processing. if self.term.owner and not self.noverbose: self.term.comment.append('Owner: %s' % self.term.owner) if self.term.comment and not self.noverbose: config.Append('/*') for comment in self.term.comment: for line in comment.split('\n'): config.Append('** ' + line) config.Append('*/') # Term verbatim output - this will skip over normal term creation # code. Warning generated from policy.py if appropriate. if self.term.verbatim: for next_term in self.term.verbatim: if next_term.value[0] == self._PLATFORM: config.Append(str(next_term.value[1]), verbatim=True) return str(config) # Helper for per-address-family keywords. family_keywords = self._TERM_TYPE.get(self.term_type) # option # this is going to be a little ugly b/c there are a few little messed # up options we can deal with. if self.term.option: for opt in [str(x) for x in self.term.option]: # there should be a better way to search the array of protocols if opt.startswith('sample'): self.extra_actions.append('sample') # only append tcp-established for option established when # tcp is the only protocol, otherwise other protos break on juniper elif opt.startswith('established'): if self.term.protocol == ['tcp']: if 'tcp-established;' not in from_str: from_str.append(family_keywords['tcp-est'] + ';') # if tcp-established specified, but more than just tcp is included # in the protocols, raise an error elif opt.startswith('tcp-established'): flag = family_keywords['tcp-est'] + ';' if self.term.protocol == ['tcp']: if flag not in from_str: from_str.append(flag) else: raise TcpEstablishedWithNonTcp( 'tcp-established can only be used with tcp protocol in term %s' % self.term.name) elif opt.startswith('rst'): from_str.append('tcp-flags "rst";') elif opt.startswith('initial') and 'tcp' in self.term.protocol: from_str.append('tcp-initial;') elif opt.startswith('first-fragment'): from_str.append('first-fragment;') # we don't have a special way of dealing with this, so we output it and # hope the user knows what they're doing. else: from_str.append('%s;' % opt) # term name config.Append('term %s {' % self.term.name) # a default action term doesn't have any from { clause has_match_criteria = (self.term.address or self.term.dscp_except or self.term.dscp_match or self.term.destination_address or self.term.destination_port or self.term.destination_prefix or self.term.destination_prefix_except or self.term.ether_type or self.term.flexible_match_range or self.term.forwarding_class or self.term.forwarding_class_except or self.term.fragment_offset or self.term.hop_limit or self.term.next_ip or self.term.port or self.term.precedence or self.term.protocol or self.term.protocol_except or self.term.source_address or self.term.source_port or self.term.source_prefix or self.term.source_prefix_except or self.term.traffic_type or self.term.ttl) if has_match_criteria: config.Append('from {') term_af = self.AF_MAP.get(self.term_type) # address address = self.term.GetAddressOfVersion('address', term_af) if self.enable_dsmo: address = summarizer.Summarize(address) if address: config.Append('%s {' % family_keywords['addr']) for addr in address: if self.enable_dsmo: config.Append('%s/%s;' % summarizer.ToDottedQuad(addr, nondsm=True)) else: for comment in self._Comment(addr): config.Append('%s' % comment) config.Append('%s;' % addr) config.Append('}') elif self.term.address: logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name, af=self.term_type)) return '' # source address src_addr = self.term.GetAddressOfVersion('source_address', term_af) src_addr_ex = self.term.GetAddressOfVersion('source_address_exclude', term_af) if self.enable_dsmo: src_addr = summarizer.Summarize(src_addr) src_addr_ex = summarizer.Summarize(src_addr_ex) else: src_addr, src_addr_ex = self._MinimizePrefixes(src_addr, src_addr_ex) if src_addr: config.Append('%s {' % family_keywords['saddr']) for addr in src_addr: if self.enable_dsmo: config.Append('%s/%s;' % summarizer.ToDottedQuad(addr, nondsm=True)) else: for comment in self._Comment(addr): config.Append('%s' % comment) config.Append('%s;' % addr) for addr in src_addr_ex: if self.enable_dsmo: config.Append('%s/%s except;' % summarizer.ToDottedQuad(addr, nondsm=True)) else: for comment in self._Comment(addr, exclude=True): config.Append('%s' % comment) config.Append('%s except;' % addr) config.Append('}') elif self.term.source_address: logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='source', af=self.term_type)) return '' # destination address dst_addr = self.term.GetAddressOfVersion('destination_address', term_af) dst_addr_ex = self.term.GetAddressOfVersion('destination_address_exclude', term_af) if self.enable_dsmo: dst_addr = summarizer.Summarize(dst_addr) dst_addr_ex = summarizer.Summarize(dst_addr_ex) else: dst_addr, dst_addr_ex = self._MinimizePrefixes(dst_addr, dst_addr_ex) if dst_addr: config.Append('%s {' % family_keywords['daddr']) for addr in dst_addr: if self.enable_dsmo: config.Append('%s/%s;' % summarizer.ToDottedQuad(addr, nondsm=True)) else: for comment in self._Comment(addr): config.Append('%s' % comment) config.Append('%s;' % addr) for addr in dst_addr_ex: if self.enable_dsmo: config.Append('%s/%s except;' % summarizer.ToDottedQuad(addr, nondsm=True)) else: for comment in self._Comment(addr, exclude=True): config.Append('%s' % comment) config.Append('%s except;' % addr) config.Append('}') elif self.term.destination_address: logging.debug(self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='destination', af=self.term_type)) return '' # forwarding-class if self.term.forwarding_class: config.Append('forwarding-class %s' % self._Group( self.term.forwarding_class, lc=False)) # forwarding-class-except if self.term.forwarding_class_except: config.Append('forwarding-class-except %s' % self._Group( self.term.forwarding_class_except, lc=False)) # source prefix <except> list if self.term.source_prefix or self.term.source_prefix_except: config.Append('source-prefix-list {') for pfx in self.term.source_prefix: config.Append(pfx + ';') for epfx in self.term.source_prefix_except: config.Append(epfx + ' except;') config.Append('}') # destination prefix <except> list if self.term.destination_prefix or self.term.destination_prefix_except: config.Append('destination-prefix-list {') for pfx in self.term.destination_prefix: config.Append(pfx + ';') for epfx in self.term.destination_prefix_except: config.Append(epfx + ' except;') config.Append('}') if self.term.ttl: config.Append('ttl %s;' % self.term.ttl) # protocol if self.term.protocol: # both are supported on JunOS, but only icmp6 is supported # on SRX loopback stateless filter config.Append(family_keywords['protocol'] + ' ' + self._Group(self.term.protocol)) # protocol if self.term.protocol_except: # same as above config.Append(family_keywords['protocol-except'] + ' ' + self._Group(self.term.protocol_except)) # port if self.term.port: config.Append('port %s' % self._Group(self.term.port)) # source port if self.term.source_port: config.Append('source-port %s' % self._Group(self.term.source_port)) # destination port if self.term.destination_port: config.Append('destination-port %s' % self._Group(self.term.destination_port)) # append any options beloging in the from {} section for next_str in from_str: config.Append(next_str) # packet length if self.term.packet_length: config.Append('packet-length %s;' % self.term.packet_length) # fragment offset if self.term.fragment_offset: config.Append('fragment-offset %s;' % self.term.fragment_offset) # icmp-types icmp_types = [''] if self.term.icmp_type: icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, self.term.protocol, self.term_type) if icmp_types != ['']: config.Append('icmp-type %s' % self._Group(icmp_types)) if self.term.icmp_code: config.Append('icmp-code %s' % self._Group(self.term.icmp_code)) if self.term.ether_type: config.Append('ether-type %s' % self._Group(self.term.ether_type)) if self.term.traffic_type: config.Append('traffic-type %s' % self._Group(self.term.traffic_type)) if self.term.precedence: # precedence may be a single integer, or a space separated list policy_precedences = set() # precedence values may only be 0 through 7 for precedence in self.term.precedence: if int(precedence) in range(0, 8): policy_precedences.add(precedence) else: raise PrecedenceError('Precedence value %s is out of bounds in %s' % (precedence, self.term.name)) config.Append('precedence %s' % self._Group(sorted(policy_precedences))) # DSCP Match if self.term.dscp_match: if self.term_type == 'inet6': config.Append('traffic-class [ %s ];' % ( ' '.join(self.term.dscp_match))) else: config.Append('dscp [ %s ];' % ' '.join(self.term.dscp_match)) # DSCP Except if self.term.dscp_except: if self.term_type == 'inet6': config.Append('traffic-class-except [ %s ];' % ( ' '.join(self.term.dscp_except))) else: config.Append('dscp-except [ %s ];' % ' '.join(self.term.dscp_except)) if self.term.hop_limit: # Only generate a hop-limit if inet6, inet4 has not hop-limit. if self.term_type == 'inet6': config.Append('hop-limit %s;' % (self.term.hop_limit)) # flexible-match if self.term.flexible_match_range: config.Append('flexible-match-range {') for fm_opt in self.term.flexible_match_range: config.Append('%s %s;' % (fm_opt[0], fm_opt[1])) config.Append('}') config.Append('}') # end from { ... } #### # ACTIONS go below here #### # If the action is only one line, include it in the same line as "then " # statement. # For example, if the action is only accept, it should be: # "then accept;" rather than: # "then { # accept; # }" # unique_actions = set(self.extra_actions) if not self.term.routing_instance: unique_actions.update(self.term.action) if len(unique_actions) <= 1: for action in [self.term.logging, self.term.routing_instance, self.term.counter, self.term.policer, self.term.qos, self.term.loss_priority, self.term.dscp_set, self.term.next_ip, self.term.traffic_class_count]: if action: try: unique_actions.update(action) except TypeError: unique_actions.add(action) if len(unique_actions) > 1: break if len(unique_actions) == 1: # b/21795531: Juniper device treats a set of IPv4 actions differently # than any other actions. # For example, if the term is in IPv4 and the action is only discard, # it should be: # "then { # discard; # }" rather than: # "then discard;" current_action = self.ACTIONS.get(unique_actions.pop(), 'next_ip') if (self.term_type == 'inet' and current_action in ['discard', 'reject', 'reject tcp-reset'] ) or (self.term_type == 'inet6' and current_action in ['reject', 'reject tcp-reset']): config.Append('then {') config.Append('%s;' % current_action) config.Append('}') elif current_action == 'next_ip': self.NextIpCheck(self.term.next_ip, self.term.name) config.Append('then {') if self.term.next_ip[0].version == 4: config.Append('next-ip %s;' % str(self.term.next_ip[0])) else: config.Append('next-ip6 %s;' % str(self.term.next_ip[0])) config.Append('}') else: config.Append('then %s;' % current_action) elif len(unique_actions) > 1: config.Append('then {') # logging if self.term.logging: for log_target in self.term.logging: if str(log_target) == 'local': config.Append('log;') else: config.Append('syslog;') if self.term.routing_instance: config.Append('routing-instance %s;' % self.term.routing_instance) if self.term.counter: config.Append('count %s;' % self.term.counter) if self.term.traffic_class_count: config.Append('traffic-class-count %s;' % self.term.traffic_class_count) oid_length = 128 if self.term.policer: config.Append('policer %s;' % self.term.policer) if len(self.term.policer) > oid_length: logging.warn('WARNING: %s is longer than %d bytes. Due to limitation' ' in JUNOS, OIDs longer than %dB can cause SNMP ' 'timeout issues.', self.term.policer, oid_length, oid_length) if self.term.qos: config.Append('forwarding-class %s;' % self.term.qos) if self.term.loss_priority: config.Append('loss-priority %s;' % self.term.loss_priority) if self.term.next_ip: self.NextIpCheck(self.term.next_ip, self.term.name) if self.term.next_ip[0].version == 4: config.Append('next-ip %s;' % str(self.term.next_ip[0])) else: config.Append('next-ip6 %s;' % str(self.term.next_ip[0])) for action in self.extra_actions: config.Append(action + ';') # If there is a routing-instance defined, skip reject/accept/etc actions. if not self.term.routing_instance: for action in self.term.action: config.Append(self.ACTIONS.get(action) + ';') # DSCP SET if self.term.dscp_set: if self.term_type == 'inet6': config.Append('traffic-class %s;' % self.term.dscp_set) else: config.Append('dscp %s;' % self.term.dscp_set) config.Append('}') # end then{...} config.Append('}') # end term accept-foo-to-bar { ... } return str(config)
def _generate_signature(func, reverse_index): """Given a function, returns a list of strings representing its args. This function produces a list of strings representing the arguments to a python function. It uses tf_inspect.getfullargspec, which does not generalize well to Python 3.x, which is more flexible in how *args and **kwargs are handled. This is not a problem in TF, since we have to remain compatible to Python 2.7 anyway. This function uses `__name__` for callables if it is available. This can lead to poor results for functools.partial and other callable objects. The returned string is Python code, so if it is included in a Markdown document, it should be typeset as code (using backticks), or escaped. Args: func: A function, method, or functools.partial to extract the signature for. reverse_index: A map from object ids to canonical full names to use. Returns: A list of strings representing the argument signature of `func` as python code. """ args_list = [] argspec = tf_inspect.getfullargspec(func) first_arg_with_default = ( len(argspec.args or []) - len(argspec.defaults or [])) # Python documentation skips `self` when printing method signatures. # Note we cannot test for ismethod here since unbound methods do not register # as methods (in Python 3). first_arg = 1 if 'self' in argspec.args[:1] else 0 # Add all args without defaults. for arg in argspec.args[first_arg:first_arg_with_default]: args_list.append(arg) # Add all args with defaults. if argspec.defaults: try: source = _remove_first_line_indent(tf_inspect.getsource(func)) func_ast = ast.parse(source) ast_defaults = func_ast.body[0].args.defaults except IOError: # If this is a builtin, getsource fails with IOError # If we cannot get the source, assume the AST would be equal to the repr # of the defaults. ast_defaults = [None] * len(argspec.defaults) for arg, default, ast_default in zip( argspec.args[first_arg_with_default:], argspec.defaults, ast_defaults): if id(default) in reverse_index: default_text = reverse_index[id(default)] elif ast_default is not None: default_text = ( astor.to_source(ast_default).rstrip('\n').replace('\t', '\\t') .replace('\n', '\\n').replace('"""', "'")) default_text = PAREN_NUMBER_RE.sub('\\1', default_text) if default_text != repr(default): # This may be an internal name. If so, handle the ones we know about. # TODO(wicke): This should be replaced with a lookup in the index. # TODO(wicke): (replace first ident with tf., check if in index) internal_names = { 'ops.GraphKeys': 'tf.GraphKeys', '_ops.GraphKeys': 'tf.GraphKeys', 'init_ops.zeros_initializer': 'tf.zeros_initializer', 'init_ops.ones_initializer': 'tf.ones_initializer', 'saver_pb2.SaverDef': 'tf.train.SaverDef', } full_name_re = '^%s(.%s)+' % (IDENTIFIER_RE, IDENTIFIER_RE) match = re.match(full_name_re, default_text) if match: lookup_text = default_text for internal_name, public_name in six.iteritems(internal_names): if match.group(0).startswith(internal_name): lookup_text = public_name + default_text[len(internal_name):] break if default_text is lookup_text: logging.warn( 'WARNING: Using default arg, failed lookup: %s, repr: %r', default_text, default) else: default_text = lookup_text else: default_text = repr(default) args_list.append('%s=%s' % (arg, default_text)) # Add *args and *kwargs. if argspec.varargs: args_list.append('*' + argspec.varargs) if argspec.varkw: args_list.append('**' + argspec.varkw) return args_list
def _TranslatePolicy(self, pol, exp_info): """Translate a policy from objects into strings.""" current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) default_action = None good_default_actions = ['ACCEPT', 'DROP'] good_afs = ['inet', 'inet6'] all_protocols_stateful = True self.verbose = True for header, terms in pol.filters: filter_type = None if self._PLATFORM not in header.platforms: continue self.filter_options = header.FilterOptions(self._PLATFORM)[1:] filter_name = header.FilterName(self._PLATFORM) self._WarnIfCustomTarget(filter_name) # ensure all options after the filter name are expected for opt in self.filter_options: if opt not in good_default_actions + good_afs + self._GOOD_OPTIONS: raise UnsupportedTargetOptionError( '%s %s %s %s' % ('\nUnsupported option found in', self._PLATFORM, 'target definition:', opt)) # disable stateful? if 'nostate' in self.filter_options: all_protocols_stateful = False if 'noverbose' in self.filter_options: self.verbose = False # Check for matching af for address_family in good_afs: if address_family in self.filter_options: # should not specify more than one AF in options if filter_type is not None: raise UnsupportedFilterError( '%s %s %s %s' % ('\nMay only specify one of', good_afs, 'in filter options:', self.filter_options)) filter_type = address_family if filter_type is None: filter_type = 'inet' if self._PLATFORM == 'iptables' and filter_name == 'FORWARD': default_action = 'DROP' # does this policy override the default filter actions? for next_target in header.target: if next_target.platform == self._PLATFORM: if len(next_target.options) > 1: for arg in next_target.options: if arg in good_default_actions: default_action = arg if default_action and default_action not in good_default_actions: raise UnsupportedDefaultActionError( '%s %s %s %s %s' % ('\nOnly', ', '.join(good_default_actions), 'default filter action allowed;', default_action, 'used.')) # add the terms new_terms = [] term_names = set() for term in terms: term.name = self.FixTermLength( term.name, 'abbreviateterms' in self.filter_options, 'truncateterms' in self.filter_options) if term.name in term_names: raise aclgenerator.DuplicateTermError( 'You have a duplicate term: %s' % term.name) term_names.add(term.name) if not term.logging and term.log_limit: raise LimitButNoLogError( 'Term %s: Cannoy use log-limit without logging' % term.name) term = self.FixHighPorts( term, af=filter_type, all_protocols_stateful=all_protocols_stateful) if not term: continue if term.expiration: if term.expiration <= exp_info_date: logging.info( 'INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn( 'WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue new_terms.append( self._TERM(term, filter_name, all_protocols_stateful, default_action, filter_type, self.verbose)) self.iptables_policies.append( (header, filter_name, filter_type, default_action, new_terms))
def _TranslatePolicy(self, pol, exp_info): """Translate a policy from objects into strings.""" self.windows_policies = [] current_date = datetime.datetime.utcnow().date() exp_info_date = current_date + datetime.timedelta(weeks=exp_info) default_action = None good_default_actions = ['permit', 'block'] good_options = [] for header, terms in pol.filters: filter_type = None if self._PLATFORM not in header.platforms: continue filter_options = header.FilterOptions(self._PLATFORM)[1:] filter_name = header.FilterName(self._PLATFORM) # ensure all options after the filter name are expected for opt in filter_options: if opt not in good_default_actions + self._GOOD_AFS + good_options: raise aclgenerator.UnsupportedTargetOption('%s %s %s %s' % ( '\nUnsupported option found in', self._PLATFORM, 'target definition:', opt)) # Check for matching af for address_family in self._GOOD_AFS: if address_family in filter_options: # should not specify more than one AF in options if filter_type is not None: raise aclgenerator.UnsupportedFilterError('%s %s %s %s' % ( '\nMay only specify one of', self._GOOD_AFS, 'in filter options:', filter_options)) filter_type = address_family if filter_type is None: filter_type = 'inet' # does this policy override the default filter actions? for next_target in header.target: if next_target.platform == self._PLATFORM: if len(next_target.options) > 1: for arg in next_target.options: if arg in good_default_actions: default_action = arg if default_action and default_action not in good_default_actions: raise aclgenerator.UnsupportedDefaultAction('%s %s %s %s %s' % ( '\nOnly', ', '.join(good_default_actions), 'default filter action allowed;', default_action, 'used.')) # add the terms new_terms = [] term_names = set() for term in terms: if term.name in term_names: raise aclgenerator.DuplicateTermError( 'You have a duplicate term: %s' % term.name) term_names.add(term.name) if term.expiration: if term.expiration <= exp_info_date: logging.info('INFO: Term %s in policy %s expires ' 'in less than two weeks.', term.name, filter_name) if term.expiration <= current_date: logging.warn('WARNING: Term %s in policy %s is expired and ' 'will not be rendered.', term.name, filter_name) continue new_terms.append(self._TERM(term, filter_name, default_action, filter_type)) self.windows_policies.append((header, filter_name, filter_type, default_action, new_terms))
def make_rhn_column(): init_params = collections.OrderedDict([ ('B_c', { 'initializer': tf.constant_initializer(forget_bias) }), ]) if overlay_rank > 0: assert sparsity_ratio < 0 # TODO(melisgl): Specify initializers for the shared matrices. tiled_linear_class = tiled_linear.OverlayedTiledLinear init_params.update( collections.OrderedDict([ ('W_x_h', { 'overlay_sharing_key': 'W_x_any', 'overlay_rank': overlay_rank }), ('W_x_c', { 'overlay_sharing_key': 'W_x_any', 'overlay_rank': overlay_rank }), ('W_x_t', { 'overlay_sharing_key': 'W_x_any', 'overlay_rank': overlay_rank }), ('W_s_h', { 'overlay_sharing_key': 'W_s_any', 'overlay_rank': overlay_rank }), ('W_s_c', { 'overlay_sharing_key': 'W_s_any', 'overlay_rank': overlay_rank }), ('W_s_t', { 'overlay_sharing_key': 'W_s_any', 'overlay_rank': overlay_rank }), ])) elif sparsity_ratio >= 0.0: assert overlay_rank == -1 tiled_linear_class = tiled_linear.SparseTiledLinear sparse_initializer = tf.truncated_normal_initializer( stddev=math.sqrt( cell_init_factor / sparsity_ratio / # TODO(melisgl): This is off if the input # embedding size is different from the hidden # size. hidden_size)) init_params.update( collections.OrderedDict([ ('W_x_.*', { 'sparse_indices_sharing_key': 'W_x' }), ('W_s_.*', { 'sparse_indices_sharing_key': 'W_s' }), ('W_x', { 'sparsity_ratio': sparsity_ratio, 'initializer': sparse_initializer }), ('W_s', { 'sparsity_ratio': sparsity_ratio, 'initializer': sparse_initializer }), ])) else: tiled_linear_class = tiled_linear.TiledLinear init_params.update( collections.OrderedDict([ ('W_.*', { 'initializer': cell_initializer }), ])) logging.info('Creating RHN of depth %s', num_layers) if layer_norm: logging.warn('RHN does not support layer normalization.') cell = TiledRHNCell(hidden_size, depth=num_layers, tie_gates=tie_forget_and_input_gates, input_transform=input_transforms[layer_index], state_transform=state_transforms[layer_index], update_transform=update_transforms[layer_index], tiled_linear_class=tiled_linear_class, tiled_linear_var_init_params=init_params, cell_clip=cell_clip if cell_clip > 0 else None, activation=eval(activation_fn)) # pylint: disable=eval-used return cell
def main(args): FLAGS(args) if FLAGS.verbose: logging.basicConfig(level=logging.INFO) if FLAGS.debug: logging.basicConfig(level=logging.DEBUG) logging.debug('binary: %s\noptimize: %d\nbase_directory: %s\n' 'policy_file: %s\nrendered_acl_directory: %s', str(sys.argv[0]), int(FLAGS.optimize), str(FLAGS.base_directory), str(FLAGS.policy_file), str(FLAGS.output_directory)) definitions = None try: definitions = naming.Naming(FLAGS.definitions_directory) except naming.NoDefinitionsError: err_msg = 'bad definitions directory: %s', FLAGS.definitions_directory logging.fatal(err_msg) sys.exit(1) # thead-safe list for storing files to write manager = multiprocessing.Manager() write_files = manager.list() with_errors = False if FLAGS.policy_file: # render just one file logging.info('rendering one file') RenderFile(FLAGS.policy_file, FLAGS.output_directory, definitions, FLAGS.exp_info, write_files) else: # render all files in parallel logging.info('finding policies...') pols = [] pols.extend(DescendRecursively(FLAGS.base_directory, FLAGS.output_directory, definitions)) pool = multiprocessing.Pool(processes=FLAGS.max_renderers) results = [] for x in pols: results.append(pool.apply_async(RenderFile, args=(x.get('in_file'), x.get('out_dir'), definitions, FLAGS.exp_info, write_files))) pool.close() pool.join() for result in results: try: result.get() except (ACLParserError, ACLGeneratorError) as e: with_errors = True logging.warn('\n\nerror encountered in rendering process:\n%s\n\n', e) # actually write files to disk WriteFiles(write_files) if with_errors: logging.warn('done, with errors.') sys.exit(1) else: logging.info('done.')
def __str__(self): # Verify platform specific terms. Skip whole term if platform does not # match. if self.term.platform: if self._PLATFORM not in self.term.platform: return '' if self.term.platform_exclude: if self._PLATFORM in self.term.platform_exclude: return '' ret_str = [] # Don't render icmpv6 protocol terms under inet, or icmp under inet6 if ((self.af == 'inet6' and 'icmp' in self.term.protocol) or (self.af == 'inet' and 'icmpv6' in self.term.protocol)): logging.debug( self.NO_AF_LOG_PROTO.substitute(term=self.term.name, proto=', '.join( self.term.protocol), af=self.af)) return '' # Term verbatim output - this will skip over most normal term # creation code by returning early. Warnings provided in policy.py if self.term.verbatim: for next_verbatim in self.term.verbatim: if next_verbatim[0] == self._PLATFORM: ret_str.append(str(next_verbatim[1])) return '\n'.join(ret_str) # Create a new term self._SetDefaultAction() if self._TERM_FORMAT: ret_str.append(self._TERM_FORMAT.substitute(term=self.term_name)) if self._PREJUMP_FORMAT: ret_str.append( self._PREJUMP_FORMAT.substitute(filter=self.filter, term=self.term_name)) if self.verbose: if self.term.owner: self.term.comment.append('Owner: %s' % self.term.owner) # reformat long comments, if needed # # iptables allows individual comments up to 256 chars. # But our generator will limit a single comment line to < 120, using: # max = 119 - 27 (static chars in comment command) - [length of term name] comment_max_width = 92 - len(self.term_name) if comment_max_width < 40: comment_max_width = 40 comments = aclgenerator.WrapWords(self.term.comment, comment_max_width) # append comments to output if comments and comments[0]: for line in comments: if not line: continue # iptables-restore does not like 0-length comments. # term comments # Strip out quotes as iptables cant have nested quotes ret_str.append( self._COMMENT_FORMAT.substitute( filter=self.filter, term=self.term_name, comment=str(line).replace('\"', ''))) # Unsupported configuration; in the case of 'accept' or 'next', we # skip the rule. In other cases, we blow up (raise an exception) # to ensure that this is not considered valid configuration. if self.term.source_prefix or self.term.destination_prefix: if str(self.term.action[0]) not in set(['accept', 'next']): raise UnsupportedFilterError( '%s %s %s %s %s %s %s %s' % ('\nTerm', self.term.name, 'has action', str(self.term.action[0]), 'with source_prefix or destination_prefix,', ' which is unsupported in', self._PLATFORM, 'iptables output.')) return ('# skipped %s due to source or destination prefix rule' % self.term.name) # protocol if self.term.protocol: protocol = self.term.protocol else: protocol = ['all'] if 'hopopt' in protocol and self.af == 'inet': logging.warn('Term %s is using hopopt in IPv4 context.', self.term_name) return '' (term_saddr, exclude_saddr, term_daddr, exclude_daddr) = self._CalculateAddresses( self.term.source_address, self.term.source_address_exclude, self.term.destination_address, self.term.destination_address_exclude) if not term_saddr: logging.debug( self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='source', af=self.af)) return '' if not term_daddr: logging.debug( self.NO_AF_LOG_ADDR.substitute(term=self.term.name, direction='destination', af=self.af)) return '' # ports source_port = [] destination_port = [] if self.term.source_port: source_port = self.term.source_port if self.term.destination_port: destination_port = self.term.destination_port # icmp-types icmp_types = [''] if self.term.icmp_type: icmp_types = self.NormalizeIcmpTypes(self.term.icmp_type, protocol, self.af) source_interface = '' if self.term.source_interface: source_interface = self.term.source_interface destination_interface = '' if self.term.destination_interface: destination_interface = self.term.destination_interface log_hits = False if self.term.logging: # Iptables sends logs to hosts configured syslog log_hits = True # options tcp_flags = [] tcp_track_options = [] for next_opt in [str(x) for x in self.term.option]: # # Sanity checking and high-ports are added as appropriate in # pre-processing that is done in __str__ within class Iptables. # Option established will add destination port high-ports if protocol # contains only tcp, udp or both. This is done earlier in class Iptables. # if ((next_opt.find('established') == 0 or next_opt.find('tcp-established') == 0) and 'ESTABLISHED' not in [x.strip() for x in self.options]): if next_opt.find( 'tcp-established') == 0 and protocol != ['tcp']: raise TcpEstablishedError('%s %s %s' % ( '\noption tcp-established can only be applied for proto tcp.', '\nError in term:', self.term.name)) if self.trackstate: # Use nf_conntrack to track state -- works with any proto self.options.append('-m state --state ESTABLISHED,RELATED') elif protocol == ['tcp']: # Simple established-only rule for TCP: Must have ACK field # (SYN/ACK or subsequent ACK), or RST and no other flags. tcp_track_options = [(['ACK'], ['ACK']), (['SYN', 'FIN', 'ACK', 'RST'], ['RST'])] # Iterate through flags table, and create list of tcp-flags to append for next_flag in self._TCP_FLAGS_TABLE: if next_opt.find(next_flag) == 0: tcp_flags.append(self._TCP_FLAGS_TABLE.get(next_flag)) if next_opt in self._KNOWN_OPTIONS_MATCHERS: self.options.append(self._KNOWN_OPTIONS_MATCHERS[next_opt]) if self.term.packet_length: # Policy format is "#-#", but iptables format is "#:#" self.options.append('-m length --length %s' % self.term.packet_length.replace('-', ':')) if self.term.fragment_offset: self.options.append('-m u32 --u32 4&0x1FFF=%s' % self.term.fragment_offset.replace('-', ':')) icmp_code = [''] if self.term.icmp_code: icmp_code = self.term.icmp_code for saddr in exclude_saddr: ret_str.extend( self._FormatPart('', saddr, '', '', '', '', '', '', '', '', '', '', '', self._action_table.get('next'))) for daddr in exclude_daddr: ret_str.extend( self._FormatPart('', '', '', daddr, '', '', '', '', '', '', '', '', '', self._action_table.get('next'))) for saddr in term_saddr: for daddr in term_daddr: for icmp in icmp_types: for code in icmp_code: for proto in protocol: for tcp_matcher in tcp_track_options or (([], []), ): ret_str.extend( self._FormatPart( str(proto), saddr, source_port, daddr, destination_port, self.options, tcp_flags, icmp, code, tcp_matcher, source_interface, destination_interface, log_hits, self._action_table.get( str(self.term.action[0])))) if self._POSTJUMP_FORMAT: ret_str.append( self._POSTJUMP_FORMAT.substitute(filter=self.filter, term=self.term_name)) return '\n'.join(str(v) for v in ret_str if v is not '')