def get_info(self): """ Get all items which belong to this class :return: None """ pt_intx_map = common.get_leaf_tag_map(self.scenario_info, "pt_intx") # translation table to normalize the paired phys_gsi and virt_gsi string table = {ord('[') : ord('('), ord(']') : ord(')'), ord('{') : ord('('), ord('}') : ord(')'), ord(';') : ord(','), ord('\n') : None, ord('\r') : None, ord(' ') : None} for vm_i, s in pt_intx_map.items(): #normalize the phys_gsi and virt_gsi pair string s = s.translate(table) #extract the phys_gsi and virt_gsi pairs between parenthesis to a list s = re.findall(r'\(([^)]+)', s) self.phys_gsi[vm_i] = []; self.virt_gsi[vm_i] = []; for part in s: if not part: continue assert ',' in part, "you need to use ',' to separate phys_gsi and virt_gsi!" a, b = part.split(',') if not a and not b: continue assert a and b, "you need to specify both phys_gsi and virt_gsi!" a, b = common.str2int(a), common.str2int(b) self.phys_gsi[vm_i].append(a) self.virt_gsi[vm_i].append(b)
def _parse_raw_strings(self, data): ''' Process strings that aren't NULL byte terminated, but for which we know the string length. This should be called prior to any other smart parsing functions. @data - String to parse. Returns a parsed string. ''' if self.pre_filter_signatures and self._is_valid(data): # Get the raw string keyword arg raw_string = self._get_keyword_arg(data, 'raw-string') # Was a raw string keyword specified? if raw_string: # Get the raw string length arg raw_size = self._get_keyword_arg(data, 'raw-size') # Is the raw string length arg is a numeric value? if re.match('^-?[0-9]+$', raw_size): # Replace all instances of raw-replace in data with raw_string[:raw_size] # Also strip out everything after the raw-string keyword, including the keyword itself. # Failure to do so may (will) result in non-printable characters and this string will be # marked as invalid when it shouldn't be. data = data[:data. find(self.KEYWORDS['raw-string'])].replace( self.KEYWORDS['raw-replace'], '"' + raw_string[:str2int(raw_size)] + '"') return data
def parse(self, data): ''' Parse a given data string for smart signature keywords. If any are found, interpret them and strip them. @data - String to parse, as returned by libmagic. Returns a dictionary of parsed values. ''' results = { 'offset': '', # Offset where the match was found, filled in by Binwalk.single_scan. 'description': '', # The libmagic data string, stripped of all keywords 'name': '', # The original name of the file, if known 'delay': '', # Extract delay description 'extract': '', # Name of the extracted file, filled in by Binwalk.single_scan. 'jump': 0, # The relative offset to resume the scan from 'size': 0, # The size of the file, if known 'adjust': 0, # The relative offset to add to the reported offset } # If smart signatures are disabled, or the result data is not valid (i.e., potentially malicious), # don't parse anything, just return the raw data as the description. if self.ignore_smart_signatures or not self._is_valid(data): results['description'] = data else: # Parse the offset-adjust value. This is used to adjust the reported offset at which # a signature was located due to the fact that MagicParser.match expects all signatures # to be located at offset 0, which some wil not be. results['adjust'] = self._get_math_arg(data, 'adjust') # Parse the file-size value. This is used to determine how many bytes should be extracted # when extraction is enabled. If not specified, everything to the end of the file will be # extracted (see Binwalk.scan). try: results['size'] = str2int( self._get_keyword_arg(data, 'filesize')) except: pass results['delay'] = self._get_keyword_arg(data, 'delay') # Parse the string for the jump-to-offset keyword. # This keyword is honored, even if this string result is one of many. results['jump'] = self._get_math_arg(data, 'jump') # If this is one of many, don't do anything and leave description as a blank string. # Else, strip all keyword tags from the string and process additional keywords as necessary. if not self._one_of_many(data): results['name'] = self._get_keyword_arg(data, 'filename').strip('"') results['description'] = self._strip_tags(data) return results
def parse(self, data): ''' Parse a given data string for smart signature keywords. If any are found, interpret them and strip them. @data - String to parse, as returned by libmagic. Returns a dictionary of parsed values. ''' results = { 'offset' : '', # Offset where the match was found, filled in by Binwalk.single_scan. 'description' : '', # The libmagic data string, stripped of all keywords 'name' : '', # The original name of the file, if known 'delay' : '', # Extract delay description 'extract' : '', # Name of the extracted file, filled in by Binwalk.single_scan. 'jump' : 0, # The relative offset to resume the scan from 'size' : 0, # The size of the file, if known 'adjust' : 0, # The relative offset to add to the reported offset } # If smart signatures are disabled, or the result data is not valid (i.e., potentially malicious), # don't parse anything, just return the raw data as the description. if self.ignore_smart_signatures or not self._is_valid(data): results['description'] = data else: # Parse the offset-adjust value. This is used to adjust the reported offset at which # a signature was located due to the fact that MagicParser.match expects all signatures # to be located at offset 0, which some wil not be. results['adjust'] = self._get_math_arg(data, 'adjust') # Parse the file-size value. This is used to determine how many bytes should be extracted # when extraction is enabled. If not specified, everything to the end of the file will be # extracted (see Binwalk.scan). try: results['size'] = str2int(self._get_keyword_arg(data, 'filesize')) except: pass results['delay'] = self._get_keyword_arg(data, 'delay') # Parse the string for the jump-to-offset keyword. # This keyword is honored, even if this string result is one of many. results['jump'] = self._get_math_arg(data, 'jump') # If this is one of many, don't do anything and leave description as a blank string. # Else, strip all keyword tags from the string and process additional keywords as necessary. if not self._one_of_many(data): results['name'] = self._get_keyword_arg(data, 'filename').strip('"') results['description'] = self._strip_tags(data) return results
def _jump(self, data): ''' Obtains the jump-to-offset value of a signature, if any. @data - String result data. Returns the offset to jump to. ''' offset = 0 offset_str = self._get_keyword_arg(data, 'jump') if offset_str: try: offset = str2int(offset_str) except: pass return offset
def __deal_with_rank(self, ranks: list): splits = self.site_config.product_rank_split replace = self.site_config.product_rank_replace if type(splits) == str: splits = [splits] rs = dict() for rank in ranks: for split in splits: item = rank.split(split) if len(item) == 2: rank_num = str2int(replace_multi(item[0], replace, '')) if rank_num: rs[item[1].replace('(', '').replace(')', '').strip()] = rank_num break return rs
def _get_math_arg(self, data, keyword): ''' Retrieves the argument for keywords that specifiy mathematical expressions as arguments. @data - String result data, as returned by libmagic. @keyword - Keyword index in KEYWORDS. Returns the resulting calculated value. ''' value = 0 arg = self._get_keyword_arg(data, keyword) if arg: for string_int in arg.split('+'): try: value += str2int(string_int) except: self.invalid = True return value
def _get_math_arg(self, data, keyword): ''' Retrieves the argument for keywords that specifiy mathematical expressions as arguments. @data - String result data, as returned by libmagic. @keyword - Keyword index in KEYWORDS. Returns the resulting calculated value. ''' value = 0 arg = self._get_keyword_arg(data, keyword) if arg: for string_int in arg.split('+'): try: value += str2int(string_int) except: pass return value
def __get_git_status(self, repo, base_commit, new_commit): '''Get and parse the commits between base_commit and new_commit, return the operation list to sync cc view. Returns: A list for the operations on files for the total info of all commits between base_commit (excluded) and new_commit (included). ''' if gitutil.compare_commit(base_commit, new_commit): return None # actually no C (copy) with '-M', and no 'S' in iOS branch. types = {'M':Modify, 'R':Rename, 'D':Delete, 'A':Add, 'C':Add, 'S':Add} # actually no C (copy) with '-M'. diff_args = ['--name-status', '-M'] # result list cc_ops = [] # the diff result is like: # 'A\ta/test5.1.txt\nD\ttest.txt\nA\ttest3.1.txt\nA\ttest3.2.txt\nA\ttest5.txt\nA\ttest6.txt' # 'A\ttest3.1.txt\nR080\ttest3.txt\ttest3.2.txt' args = [base_commit.hexsha, new_commit.hexsha] args.extend(diff_args) diff_re = repo.git.diff(args) for line in diff_re.splitlines(): split = line.split("\t") op_code = split[0] char = op_code[0] file_name = split[1] op_type = types[char] if char == 'R': new_file = split[2] similarity = common.str2int(op_code[1:]) op = op_type(self.ct, file_name, new_file, new_commit, similarity, self.user_config) else: op = op_type(self.ct, file_name, new_commit, self.user_config) cc_ops.append(op) return cc_ops
# Split the line into white-space separated parts. # For this to work properly, replace escaped spaces ('\ ') with '\x20'. # This means the same thing, but doesn't confuse split(). line_parts = line.replace('\\ ', '\\x20').split() entry['offset'] = line_parts[0] entry['type'] = line_parts[1] # The condition line may contain escaped sequences, so be sure to decode it properly. entry['condition'] = line_parts[2].decode('string_escape') entry['description'] = ' '.join(line_parts[3:]) except Exception, e: raise Exception("%s :: %s", (str(e), line)) # We've already verified that the first character in this line is a number, so this *shouldn't* # throw an exception, but let's catch it just in case... try: entry['offset'] = str2int(entry['offset']) except Exception, e: raise Exception("%s :: %s", (str(e), line)) # If this is a string, get the length of the string if 'string' in entry['type'] or entry['condition'] == self.WILDCARD: entry['length'] = len(entry['condition']) # Else, we need to jump through a few more hoops... else: # Default to little endian, unless the type field starts with 'be'. # This assumes that we're running on a little endian system... if entry['type'].startswith('be'): endianess = self.BIG_ENDIAN else: endianess = self.LITTLE_ENDIAN
# Split the line into white-space separated parts. # For this to work properly, replace escaped spaces ('\ ') with '\x20'. # This means the same thing, but doesn't confuse split(). line_parts = line.replace('\\ ', '\\x20').split() entry['offset'] = line_parts[0] entry['type'] = line_parts[1] # The condition line may contain escaped sequences, so be sure to decode it properly. entry['condition'] = line_parts[2].decode('string_escape') entry['description'] = ' '.join(line_parts[3:]) except Exception, e: raise Exception("%s :: %s", (str(e), line)) # We've already verified that the first character in this line is a number, so this *shouldn't* # throw an exception, but let's catch it just in case... try: entry['offset'] = str2int(entry['offset']) except Exception, e: raise Exception("%s :: %s", (str(e), line)) # If this is a string, get the length of the string if 'string' in entry['type']: entry['length'] = len(entry['condition']) # Else, we need to jump through a few more hoops... else: # Default to little endian, unless the type field starts with 'be'. # This assumes that we're running on a little endian system... if entry['type'].startswith('be'): endianess = self.BIG_ENDIAN else: endianess = self.LITTLE_ENDIAN
def __deal_with_helpers(elements: list): if not elements: return 0 helper_str = replace_multi(''.join(elements).strip(), [",", "."], '').split(' ')[0] return str2int(helper_str, 1)
def _parse_raw_strings(self, data): ''' Process strings that aren't NULL byte terminated, but for which we know the string length. This should be called prior to any other smart parsing functions. @data - String to parse. Returns a parsed string. ''' if not self.ignore_smart_signatures and self._is_valid(data): # Get the raw string keyword arg raw_string = self._get_keyword_arg(data, 'raw-string') # Was a raw string keyword specified? if raw_string: # Get the raw string length arg raw_size = self._get_keyword_arg(data, 'raw-size') # Is the raw string length arg is a numeric value? if re.match('^-?[0-9]+$', raw_size): # Replace all instances of raw-replace in data with raw_string[:raw_size] # Also strip out everything after the raw-string keyword, including the keyword itself. # Failure to do so may (will) result in non-printable characters and this string will be # marked as invalid when it shouldn't be. data = data[:data.find(self.KEYWORDS['raw-string'])].replace(self.KEYWORDS['raw-replace'], '"' + raw_string[:str2int(raw_size)] + '"') return data