def main(): ft = FileType() setup_dir(TEMP_DIR) ftype = ft.get_type(fname) ftff = FeedToFlatFiles(TEMP_DIR) if ftype == "xml": ftff.process_feed(fname) elif ft.is_compression(ftype) or ft.is_archived(ftype): unpack = Unpack(fname, TEMP_DIR) unpack.flatten_folder() xml_file = unpack.find_file_by_extension(".xml") if xml_file: ftff.process_feed(xml_file) fc = FormatCheck(urlopen(SCHEMA_URL), TEMP_DIR) fc.validate_and_clean() # need to get error report here write_and_archive(fc.get_valid_files(), fc.get_vip_id())
def process_file_darwin(pArgs): ft = FileType.getFileType(pArgs.inputFile) logging.debug('Detected file type is {0}'.format(FileType.revMap[ft])) pArgs.arCmd = ['ar', '-x', '-v'] if pArgs.verboseFlag else ['ar', '-x'] pArgs.extractor = extract_section_darwin pArgs.fileType = FileType.MACH_OBJECT if ft == FileType.MACH_EXECUTABLE or ft == FileType.MACH_SHARED or ft == FileType.MACH_OBJECT: logging.info('Generating LLVM Bitcode module') return handleExecutable(pArgs) elif ft == FileType.ARCHIVE: if pArgs.bitcodeModuleFlag: logging.info('Generating LLVM Bitcode module from an archive') else: logging.info('Generating LLVM Bitcode archive from an archive') return handleArchive(pArgs) else: logging.error('File "{0}" of type {1} cannot be used'.format(pArgs.inputFile, FileType.revMap[ft])) return 1
def __init__(self, file_name, extract_path=None): self.ft = FileType() self.extract_path = extract_path if os.path.exists(file_name) and self.extract_path and not os.path.exists(self.extract_path): os.makedirs(self.extract_path) if os.path.isdir(file_name) and self.extract_path: shutil.copytree(file_name, self.extract_path) file_name = self.extract_path elif os.path.isfile(file_name) and self.extract_path: shutil.copy(file_name, self.extract_path) file_name = self.extract_path elif os.path.isdir(file_name): self.extract_path = file_name else: self.extract_path = os.getcwd() if os.path.isdir(file_name): self.unpack_dir(file_name) else: self.unpack_file(file_name)
def handleArchive(pArgs): originalDir = os.getcwd() # This will be the destination pArgs.arCmd.append(pArgs.inputFile) # Make temporary directory to extract objects to tempDir = '' bitCodeFiles = [ ] retCode=0 try: tempDir = tempfile.mkdtemp(suffix='wllvm') os.chdir(tempDir) # Extract objects from archive try: arP = Popen(pArgs.arCmd) except OSError as e: if e.errno == 2: errorMsg = 'Your ar does not seem to be easy to find.\n' else: errorMsg = 'OS error({0}): {1}'.format(e.errno, e.strerror) logging.error(errorMsg) raise Exception(errorMsg) arPE = arP.wait() if arPE != 0: errorMsg = 'Failed to execute archiver with command {0}'.format(pArgs.arCmd) logging.error(errorMsg) raise Exception(errorMsg) # Iterate over objects and examine their bitcode inserts for (root, dirs, files) in os.walk(tempDir): logging.debug('Exploring "{0}"'.format(root)) for f in files: fPath = os.path.join(root, f) if FileType.getFileType(fPath) == pArgs.fileType: # Extract bitcode locations from object contents = pArgs.extractor(fPath) for bcFile in contents: if bcFile != '': if not os.path.exists(bcFile): logging.warning('{0} lists bitcode library "{1}" but it could not be found'.format(f, bcFile)) else: bitCodeFiles.append(bcFile) else: logging.info('Ignoring file "{0}" in archive'.format(f)) logging.info('Found the following bitcode file names to build bitcode archive:\n{0}'.format( pprint.pformat(bitCodeFiles))) finally: # Delete the temporary folder logging.debug('Deleting temporary folder "{0}"'.format(tempDir)) shutil.rmtree(tempDir) #write the manifest file if asked for if pArgs.manifestFlag: manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile) with open(manifestFile, 'w') as output: for f in bitCodeFiles: output.write('{0}\n'.format(f)) # Build bitcode archive os.chdir(originalDir) return buildArchive(pArgs, bitCodeFiles)
class Unpack: def __init__(self, file_name, extract_path=None): self.ft = FileType() self.extract_path = extract_path if os.path.exists(file_name) and self.extract_path and not os.path.exists(self.extract_path): os.makedirs(self.extract_path) if os.path.isdir(file_name) and self.extract_path: shutil.copytree(file_name, self.extract_path) file_name = self.extract_path elif os.path.isfile(file_name) and self.extract_path: shutil.copy(file_name, self.extract_path) file_name = self.extract_path elif os.path.isdir(file_name): self.extract_path = file_name else: self.extract_path = os.getcwd() if os.path.isdir(file_name): self.unpack_dir(file_name) else: self.unpack_file(file_name) def uncompress(self, fname): ftype = self.ft.get_type(fname) if ftype == "gz": ext = GzipFile(fname, 'rb') elif ftype == "bz2": ext = BZ2File(fname, 'rb') filedata = ext.read() new_name = fname[:fname.rfind(".")] w = open(new_name, "w") w.write(filedata) new_type = self.ft.get_type(new_name) if new_type: os.rename(new_name, new_name + "." + new_type) return new_name + "." + new_type return new_name def unarchive(self, fname): ftype = self.ft.get_type(fname) if ftype == "rar": ext = RarFile(fname) elif ftype == "tar": ext = tarfile.open(fname) elif ftype == "zip": ext = ZipFile(fname) new_path = fname[:fname.rfind(".")] + "_extracted" if not os.path.exists(new_path): os.makedirs(new_path) ext.extractall(path=new_path) return new_path def unpack_dir(self, directory): for root, dirs, dirfiles in os.walk(directory): for name in dirfiles: full_name = root + "/" + name self.unpack_file(full_name) def unpack_file(self, fname): if self.ft.is_compression(fname) or self.ft.is_archived(fname): if self.ft.is_compression(fname): new_file = self.uncompress(fname) elif self.ft.is_archived(fname): new_file = self.unarchive(fname) if fname != new_file: os.remove(fname) if os.path.isdir(new_file): self.unpack_dir(new_file) else: self.unpack_file(new_file) def find_file_by_name(self, file_name): return self.find_files_by_name(file_name)[0] def find_files_by_name(self, file_name): return self.find_files(re.compile(file_name)) def find_file_by_extension(self, file_extension): return self.find_files_by_extension(file_extension)[0] def find_files_by_extension(self, file_extension): if not file_extension.startswith("."): file_extension = "." + file_extension return self.find_files(re.compile(".*\\" + file_extension)) def find_file_by_partial(self, file_partial): return self.find_files_by_partial(file_partial)[0] def find_files_by_partial(self, file_partial): return self.find_files(re.compile(".*" + file_partial + ".*")) def find_files(self, regex): file_list = [] for root, dirs, dirfiles in os.walk(self.extract_path): for name in dirfiles: if regex.match(name) and name.find("/.") < 0: file_list.append(root + "/" + name) if len(file_list) > 0: return file_list def get_file_list(self): return self.find_files(re.compile(".*")) def find_folder_by_name(self, folder_name): return self.find_folder(re.compile(folder_name)) def find_folder_by_partial(self, folder_partial): return self.find_folder(re.compile(".*" + folder_partial + ".*")) def find_folder(self, regex): for root, dirs, dirfiles in os.walk(self.extract_path): for dir_name in dirs: if regex.match(dir_name) and dir_name.find("/.") < 0: return root + "/" + dir_name def get_folder_list(self): folder_list= [] for root, dirs, dirfiles in os.walk(self.extract_path): for dir_name in dirs: return folder_list.append(root + "/" + dir_name) if len(folder_list) > 0: return folder_list def flatten_folder(self): try: if self.extract_path == os.getcwd(): raise SameDirError except SameDirError: raise for root, dirs, files in os.walk(self.extract_path): if root != self.extract_path: for name in files: shutil.move(root + "/" + name, self.extract_path + "/" + name) for root, dirs, files in os.walk(self.extract_path): for d in dirs: shutil.rmtree(os.path.join(root, d))