def main():
ft = FileType()
setup_dir(TEMP_DIR)
ftype = ft.get_type(fname)
ftff = FeedToFlatFiles(TEMP_DIR)
if ftype == "xml":
ftff.process_feed(fname)
elif ft.is_compression(ftype) or ft.is_archived(ftype):
unpack = Unpack(fname, TEMP_DIR)
unpack.flatten_folder()
xml_file = unpack.find_file_by_extension(".xml")
if xml_file:
ftff.process_feed(xml_file)
fc = FormatCheck(urlopen(SCHEMA_URL), TEMP_DIR)
fc.validate_and_clean()
# need to get error report here
write_and_archive(fc.get_valid_files(), fc.get_vip_id())
def process_file_darwin(pArgs):
ft = FileType.getFileType(pArgs.inputFile)
logging.debug('Detected file type is {0}'.format(FileType.revMap[ft]))
pArgs.arCmd = ['ar', '-x', '-v'] if pArgs.verboseFlag else ['ar', '-x']
pArgs.extractor = extract_section_darwin
pArgs.fileType = FileType.MACH_OBJECT
if ft == FileType.MACH_EXECUTABLE or ft == FileType.MACH_SHARED or ft == FileType.MACH_OBJECT:
logging.info('Generating LLVM Bitcode module')
return handleExecutable(pArgs)
elif ft == FileType.ARCHIVE:
if pArgs.bitcodeModuleFlag:
logging.info('Generating LLVM Bitcode module from an archive')
else:
logging.info('Generating LLVM Bitcode archive from an archive')
return handleArchive(pArgs)
else:
logging.error('File "{0}" of type {1} cannot be used'.format(pArgs.inputFile, FileType.revMap[ft]))
return 1
def __init__(self, file_name, extract_path=None): self.ft = FileType() self.extract_path = extract_path if os.path.exists(file_name) and self.extract_path and not os.path.exists(self.extract_path): os.makedirs(self.extract_path) if os.path.isdir(file_name) and self.extract_path: shutil.copytree(file_name, self.extract_path) file_name = self.extract_path elif os.path.isfile(file_name) and self.extract_path: shutil.copy(file_name, self.extract_path) file_name = self.extract_path elif os.path.isdir(file_name): self.extract_path = file_name else: self.extract_path = os.getcwd() if os.path.isdir(file_name): self.unpack_dir(file_name) else: self.unpack_file(file_name)
def handleArchive(pArgs):
originalDir = os.getcwd() # This will be the destination
pArgs.arCmd.append(pArgs.inputFile)
# Make temporary directory to extract objects to
tempDir = ''
bitCodeFiles = [ ]
retCode=0
try:
tempDir = tempfile.mkdtemp(suffix='wllvm')
os.chdir(tempDir)
# Extract objects from archive
try:
arP = Popen(pArgs.arCmd)
except OSError as e:
if e.errno == 2:
errorMsg = 'Your ar does not seem to be easy to find.\n'
else:
errorMsg = 'OS error({0}): {1}'.format(e.errno, e.strerror)
logging.error(errorMsg)
raise Exception(errorMsg)
arPE = arP.wait()
if arPE != 0:
errorMsg = 'Failed to execute archiver with command {0}'.format(pArgs.arCmd)
logging.error(errorMsg)
raise Exception(errorMsg)
# Iterate over objects and examine their bitcode inserts
for (root, dirs, files) in os.walk(tempDir):
logging.debug('Exploring "{0}"'.format(root))
for f in files:
fPath = os.path.join(root, f)
if FileType.getFileType(fPath) == pArgs.fileType:
# Extract bitcode locations from object
contents = pArgs.extractor(fPath)
for bcFile in contents:
if bcFile != '':
if not os.path.exists(bcFile):
logging.warning('{0} lists bitcode library "{1}" but it could not be found'.format(f, bcFile))
else:
bitCodeFiles.append(bcFile)
else:
logging.info('Ignoring file "{0}" in archive'.format(f))
logging.info('Found the following bitcode file names to build bitcode archive:\n{0}'.format(
pprint.pformat(bitCodeFiles)))
finally:
# Delete the temporary folder
logging.debug('Deleting temporary folder "{0}"'.format(tempDir))
shutil.rmtree(tempDir)
#write the manifest file if asked for
if pArgs.manifestFlag:
manifestFile = '{0}.llvm.manifest'.format(pArgs.inputFile)
with open(manifestFile, 'w') as output:
for f in bitCodeFiles:
output.write('{0}\n'.format(f))
# Build bitcode archive
os.chdir(originalDir)
return buildArchive(pArgs, bitCodeFiles)
class Unpack:
def __init__(self, file_name, extract_path=None):
self.ft = FileType()
self.extract_path = extract_path
if os.path.exists(file_name) and self.extract_path and not os.path.exists(self.extract_path):
os.makedirs(self.extract_path)
if os.path.isdir(file_name) and self.extract_path:
shutil.copytree(file_name, self.extract_path)
file_name = self.extract_path
elif os.path.isfile(file_name) and self.extract_path:
shutil.copy(file_name, self.extract_path)
file_name = self.extract_path
elif os.path.isdir(file_name):
self.extract_path = file_name
else:
self.extract_path = os.getcwd()
if os.path.isdir(file_name):
self.unpack_dir(file_name)
else:
self.unpack_file(file_name)
def uncompress(self, fname):
ftype = self.ft.get_type(fname)
if ftype == "gz":
ext = GzipFile(fname, 'rb')
elif ftype == "bz2":
ext = BZ2File(fname, 'rb')
filedata = ext.read()
new_name = fname[:fname.rfind(".")]
w = open(new_name, "w")
w.write(filedata)
new_type = self.ft.get_type(new_name)
if new_type:
os.rename(new_name, new_name + "." + new_type)
return new_name + "." + new_type
return new_name
def unarchive(self, fname):
ftype = self.ft.get_type(fname)
if ftype == "rar":
ext = RarFile(fname)
elif ftype == "tar":
ext = tarfile.open(fname)
elif ftype == "zip":
ext = ZipFile(fname)
new_path = fname[:fname.rfind(".")] + "_extracted"
if not os.path.exists(new_path):
os.makedirs(new_path)
ext.extractall(path=new_path)
return new_path
def unpack_dir(self, directory):
for root, dirs, dirfiles in os.walk(directory):
for name in dirfiles:
full_name = root + "/" + name
self.unpack_file(full_name)
def unpack_file(self, fname):
if self.ft.is_compression(fname) or self.ft.is_archived(fname):
if self.ft.is_compression(fname):
new_file = self.uncompress(fname)
elif self.ft.is_archived(fname):
new_file = self.unarchive(fname)
if fname != new_file:
os.remove(fname)
if os.path.isdir(new_file):
self.unpack_dir(new_file)
else:
self.unpack_file(new_file)
def find_file_by_name(self, file_name):
return self.find_files_by_name(file_name)[0]
def find_files_by_name(self, file_name):
return self.find_files(re.compile(file_name))
def find_file_by_extension(self, file_extension):
return self.find_files_by_extension(file_extension)[0]
def find_files_by_extension(self, file_extension):
if not file_extension.startswith("."):
file_extension = "." + file_extension
return self.find_files(re.compile(".*\\" + file_extension))
def find_file_by_partial(self, file_partial):
return self.find_files_by_partial(file_partial)[0]
def find_files_by_partial(self, file_partial):
return self.find_files(re.compile(".*" + file_partial + ".*"))
def find_files(self, regex):
file_list = []
for root, dirs, dirfiles in os.walk(self.extract_path):
for name in dirfiles:
if regex.match(name) and name.find("/.") < 0:
file_list.append(root + "/" + name)
if len(file_list) > 0:
return file_list
def get_file_list(self):
return self.find_files(re.compile(".*"))
def find_folder_by_name(self, folder_name):
return self.find_folder(re.compile(folder_name))
def find_folder_by_partial(self, folder_partial):
return self.find_folder(re.compile(".*" + folder_partial + ".*"))
def find_folder(self, regex):
for root, dirs, dirfiles in os.walk(self.extract_path):
for dir_name in dirs:
if regex.match(dir_name) and dir_name.find("/.") < 0:
return root + "/" + dir_name
def get_folder_list(self):
folder_list= []
for root, dirs, dirfiles in os.walk(self.extract_path):
for dir_name in dirs:
return folder_list.append(root + "/" + dir_name)
if len(folder_list) > 0:
return folder_list
def flatten_folder(self):
try:
if self.extract_path == os.getcwd():
raise SameDirError
except SameDirError:
raise
for root, dirs, files in os.walk(self.extract_path):
if root != self.extract_path:
for name in files:
shutil.move(root + "/" + name, self.extract_path + "/" + name)
for root, dirs, files in os.walk(self.extract_path):
for d in dirs:
shutil.rmtree(os.path.join(root, d))
