def parse(f, context, metadata_dict, suppress_handler, skip_handler, steps, processed_path_hashes): """ Prints the results in the given file to the standard output in a human- readable format. Returns the report statistics collected by the result handler. """ if not f.endswith(".plist"): LOG.debug("Skipping input file '" + f + "' as it is not a plist.") return {}, set() LOG.debug("Parsing input file '" + f + "'") rh = plist_parser.PlistToPlaintextFormatter(suppress_handler, skip_handler, context.severity_map, processed_path_hashes) rh.print_steps = steps # Set some variables of the result handler to use the saved file. analyzed_source_file = "UNKNOWN" if 'result_source_files' in metadata_dict and \ f in metadata_dict['result_source_files']: analyzed_source_file = \ metadata_dict['result_source_files'][f] files, reports = rh.parse(f) plist_mtime = util.get_last_mod_time(f) changed_files = set() for source_file in files: if plist_mtime is None: # Failed to get the modification time for # a file mark it as changed. changed_files.add(source_file) LOG.warning(source_file + ' is missing since the last analysis.') continue file_mtime = util.get_last_mod_time(source_file) if file_mtime > plist_mtime: changed_files.add(source_file) LOG.warning(source_file + ' did change since the last analysis.') if changed_files: return {}, changed_files else: report_stats = rh.write(files, reports, analyzed_source_file) return report_stats, set()
def parse(plist_file, metadata_dict, rh, file_report_map): """ Prints the results in the given file to the standard output in a human- readable format. Returns the report statistics collected by the result handler. """ if not plist_file.endswith(".plist"): LOG.debug("Skipping input file '%s' as it is not a plist.", plist_file) return set() LOG.debug("Parsing input file '%s'", plist_file) if 'result_source_files' in metadata_dict and \ plist_file in metadata_dict['result_source_files']: analyzed_source_file = \ metadata_dict['result_source_files'][plist_file] if analyzed_source_file not in file_report_map: file_report_map[analyzed_source_file] = [] files, reports = rh.parse(plist_file) plist_mtime = util.get_last_mod_time(plist_file) changed_files = set() for source_file in files: if plist_mtime is None: # Failed to get the modification time for # a file mark it as changed. changed_files.add(source_file) LOG.warning('%s is missing since the last analysis.', source_file) continue file_mtime = util.get_last_mod_time(source_file) if file_mtime > plist_mtime: changed_files.add(source_file) LOG.warning('%s did change since the last analysis.', source_file) if not changed_files: for report in reports: file_path = report.file_path if file_path not in file_report_map: file_report_map[file_path] = [] file_report_map[file_path].append(report) return changed_files
def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times of files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk. """ missing_files = [] try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return missing_files except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex))
def collect_file_hashes_from_plist(plist_file): try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): return False content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return True except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex))
def assemble_zip(inputs, zip_file, client): hash_to_file = {} # There can be files with same hash, # but different path. file_to_hash = {} file_to_mtime = {} missing_source_files = set() def collect_file_hashes_from_plist(plist_file): """ Collects file content hashes and last modification times of files which can be found in the given plist file. :returns List of file paths which are in the processed plist file but missing from the user's disk. """ missing_files = [] try: files, _ = plist_parser.parse_plist(plist_file) for f in files: if not os.path.isfile(f): missing_files.append(f) missing_source_files.add(f) continue content_hash = util.get_file_content_hash(f) hash_to_file[content_hash] = f file_to_hash[f] = content_hash file_to_mtime[f] = util.get_last_mod_time(f) return missing_files except Exception as ex: LOG.error('Parsing the plist failed: ' + str(ex)) plist_report_files = [] changed_files = set() for input_path in inputs: input_path = os.path.abspath(input_path) if not os.path.exists(input_path): raise OSError(errno.ENOENT, "Input path does not exist", input_path) if os.path.isfile(input_path): files = [input_path] else: _, _, files = next(os.walk(input_path), ([], [], [])) for f in files: plist_file = os.path.join(input_path, f) if f.endswith(".plist"): missing_files = collect_file_hashes_from_plist(plist_file) if not missing_files: LOG.debug( "Copying file '{0}' to ZIP assembly dir...".format( plist_file)) plist_report_files.append(os.path.join(input_path, f)) else: LOG.warning( "Skipping '%s' because it refers " "the following missing source files: %s", plist_file, missing_files) elif f == 'metadata.json': plist_report_files.append(os.path.join(input_path, f)) elif f == 'skip_file': plist_report_files.append(os.path.join(input_path, f)) plist_mtime = util.get_last_mod_time(plist_file) for k, v in file_to_mtime.items(): if v > plist_mtime: changed_files.add(k) if changed_files: changed_files = '\n'.join([' - ' + f for f in changed_files]) LOG.warning("The following source file contents changed since the " "latest analysis:\n{0}\nPlease analyze your project " "again to update the reports!".format(changed_files)) sys.exit(1) with zipfile.ZipFile(zip_file, 'a', allowZip64=True) as zipf: for pl in plist_report_files: _, plist_filename = os.path.split(pl) zip_target = os.path.join('reports', plist_filename) zipf.write(pl, zip_target) if len(hash_to_file) == 0: LOG.warning("There is no report to store. After uploading these " "results the previous reports become resolved.") necessary_hashes = client.getMissingContentHashes(hash_to_file.keys()) for f, h in file_to_hash.items(): if h in necessary_hashes: LOG.debug( "File contents for '{0}' needed by the server".format(f)) zipf.write(f, os.path.join('root', f.lstrip('/'))) zipf.writestr('content_hashes.json', json.dumps(file_to_hash)) # Compressing .zip file with open(zip_file, 'rb') as source: compressed = zlib.compress(source.read(), zlib.Z_BEST_COMPRESSION) with open(zip_file, 'wb') as target: target.write(compressed) LOG.debug("[ZIP] Mass store zip written at '{0}'".format(zip_file)) if missing_source_files: LOG.warning( "Missing source files: \n%s", '\n'.join(map(lambda f_: " - " + f_, missing_source_files)))