def pack_begin(self, in_path):
# remove trailing slash
if in_path[-1] == '/':
in_path = in_path[:-1]
meta_path = in_path + ".xdfmeta"
if os.path.exists(meta_path):
self.meta_db = MetaDB()
self.meta_db.load(meta_path)
def unpack(self, volume, out_path):
# check for volume path
vol_name = volume.name.get_unicode()
if not os.path.exists(out_path):
vol_path = out_path
else:
path = os.path.abspath(out_path)
vol_path = os.path.join(path, self.to_path_str(vol_name))
if os.path.exists(vol_path):
raise IOError("Unpack directory already exists: "+vol_path)
# check for meta file
meta_path = vol_path + ".xdfmeta"
if os.path.exists(meta_path):
raise IOError("Unpack meta file already exists:"+meta_path)
# check for block dev file
blkdev_path = vol_path + ".blkdev"
if os.path.exists(blkdev_path):
raise IOError("Unpack blkdev file aready exists:"+blkdev_path)
# create volume path
self.meta_db = MetaDB()
self.unpack_root(volume, vol_path)
# save meta db
self.meta_db.set_volume_name(volume.name.get_unicode())
self.meta_db.set_root_meta_info(volume.get_meta_info())
self.meta_db.set_dos_type(volume.boot.dos_type)
self.meta_db.save(meta_path)
# save boot code
if volume.boot.boot_code != None:
boot_code_path = vol_path + ".bootcode"
f = open(boot_code_path,"wb")
f.write(volume.boot.boot_code)
f.close()
# save blkdev
f = open(blkdev_path,"wb")
f.write("%s\n" % volume.blkdev.get_chs_str())
f.close()
class Imager:
def __init__(self, path_encoding=None):
self.meta_db = None
self.total_bytes = 0
self.path_encoding = path_encoding
# get path name encoding for host file system
if self.path_encoding == None:
self.path_encoding = sys.getfilesystemencoding()
def get_total_bytes(self):
return self.total_bytes
def to_path_str(self, u):
"""convert a unicode string to OS path name encoding"""
if type(u) != unicode:
raise ValueError("to_path_str: must pass a unicode string")
return u.encode(self.path_encoding)
def from_path_str(self, s):
"""convert a OS path name encoded string to unicode"""
if type(s) != str:
raise ValueError("from_path_str: must pass a string")
u = s.decode(self.path_encoding)
# on Mac OS X normalize from decomposed form
if sys.platform.startswith('darwin'):
return unicodedata.normalize('NFC',u)
else:
return u
# ----- unpack -----
def unpack(self, volume, out_path):
# check for volume path
vol_name = volume.name.get_unicode()
if not os.path.exists(out_path):
vol_path = out_path
else:
path = os.path.abspath(out_path)
vol_path = os.path.join(path, self.to_path_str(vol_name))
if os.path.exists(vol_path):
raise IOError("Unpack directory already exists: "+vol_path)
# check for meta file
meta_path = vol_path + ".xdfmeta"
if os.path.exists(meta_path):
raise IOError("Unpack meta file already exists:"+meta_path)
# check for block dev file
blkdev_path = vol_path + ".blkdev"
if os.path.exists(blkdev_path):
raise IOError("Unpack blkdev file aready exists:"+blkdev_path)
# create volume path
self.meta_db = MetaDB()
self.unpack_root(volume, vol_path)
# save meta db
self.meta_db.set_volume_name(volume.name.get_unicode())
self.meta_db.set_root_meta_info(volume.get_meta_info())
self.meta_db.set_dos_type(volume.boot.dos_type)
self.meta_db.save(meta_path)
# save boot code
if volume.boot.boot_code != None:
boot_code_path = vol_path + ".bootcode"
f = open(boot_code_path,"wb")
f.write(volume.boot.boot_code)
f.close()
# save blkdev
f = open(blkdev_path,"wb")
f.write("%s\n" % volume.blkdev.get_chs_str())
f.close()
def unpack_root(self, volume, vol_path):
self.unpack_dir(volume.get_root_dir(), vol_path)
def unpack_dir(self, dir, path):
if not os.path.exists(path):
os.mkdir(path)
for e in dir.get_entries():
self.unpack_node(e, path)
def unpack_node(self, node, path):
name = node.name.get_unicode_name()
# store meta info
if self.meta_db != None:
# get path as FSString
node_path = node.get_node_path_name()
self.meta_db.set_meta_info(node_path.get_unicode(), node.meta_info)
# sub dir
if node.is_dir():
sub_dir = os.path.join(path, self.to_path_str(name))
os.mkdir(sub_dir)
for sub_node in node.get_entries():
self.unpack_node(sub_node, sub_dir)
node.flush()
# file
elif node.is_file():
data = node.get_file_data()
node.flush()
file_path = os.path.join(path, self.to_path_str(name))
fh = open(file_path, "wb")
fh.write(data)
fh.close()
self.total_bytes += len(data)
# ----- pack -----
def pack(self, in_path, image_file, force=True, options=None):
self.pack_begin(in_path)
blkdev = self.pack_create_blkdev(in_path, image_file, force, options)
if blkdev == None:
raise IOError("Can't create block device for image: "+in_path)
volume = self.pack_create_volume(in_path, blkdev)
if not volume.valid:
raise IOError("Can't create volume for image: "+in_path)
self.pack_root(in_path, volume)
self.pack_end(in_path, volume)
def pack_begin(self, in_path):
# remove trailing slash
if in_path[-1] == '/':
in_path = in_path[:-1]
meta_path = in_path + ".xdfmeta"
if os.path.exists(meta_path):
self.meta_db = MetaDB()
self.meta_db.load(meta_path)
def pack_end(self, in_path, volume):
boot_code_path = in_path + ".bootcode"
if os.path.exists(boot_code_path):
# read boot code
f = open(boot_code_path, "rb")
data = f.read()
f.close()
# write boot code
bb = volume.boot
ok = bb.set_boot_code(data)
if ok:
bb.write()
else:
raise IOError("Invalid Boot Code")
def pack_create_blkdev(self, in_path, image_file, force=True, options=None):
# try to read options from blkdev file
if options == None or len(options) == 0:
blkdev_path = in_path + ".blkdev"
if os.path.exists(blkdev_path):
f = open(blkdev_path, "rb")
options = {}
for line in f:
KeyValue.parse_key_value_string(line, options)
f.close()
f = BlkDevFactory()
return f.create(image_file, force=force, options=options)
def pack_create_volume(self, in_path, blkdev):
if self.meta_db != None:
name = self.meta_db.get_volume_name()
meta_info = self.meta_db.get_root_meta_info()
dos_type = self.meta_db.get_dos_type()
else:
# try to derive volume name from image name
if in_path == None or in_path == "":
raise IOError("Invalid pack input path!")
# remove trailing slash
if in_path[-1] == '/':
in_path = in_path[:-1]
name = self.from_path_str(os.path.basename(in_path))
meta_info = None
dos_type = DosType.DOS0
volume = ADFSVolume(blkdev)
volume.create(FSString(name), meta_info, dos_type=dos_type)
return volume
def pack_root(self, in_path, volume):
self.pack_dir(in_path, volume.get_root_dir())
def pack_dir(self, in_path, parent_node):
path = os.path.abspath(in_path)
if not os.path.exists(path):
raise IOError("Pack directory does not exist: "+path)
for name in os.listdir(in_path):
sub_path = os.path.join(in_path, name)
self.pack_entry(sub_path, parent_node)
def pack_entry(self, in_path, parent_node):
ami_name = self.from_path_str(os.path.basename(in_path))
# retrieve meta info for path from DB
if self.meta_db != None:
ami_path = parent_node.get_node_path_name().get_unicode()
if ami_path != u"":
ami_path += u"/" + ami_name
else:
ami_path = ami_name
meta_info = self.meta_db.get_meta_info(ami_path)
else:
meta_info = None
# pack directory
if os.path.isdir(in_path):
node = parent_node.create_dir(FSString(ami_name), meta_info, False)
for name in os.listdir(in_path):
sub_path = os.path.join(in_path, name)
self.pack_entry(sub_path, node)
node.flush()
# pack file
elif os.path.isfile(in_path):
# read file
fh = open(in_path, "rb")
data = fh.read()
fh.close()
node = parent_node.create_file(FSString(ami_name), data, meta_info, False)
node.flush()
self.total_bytes += len(data)
def createCustomer(self):
self.metaDB = MetaDB()
def __init__(self, appConfig, fileMgr, regexMgr, lmLgr):
self.lmLogger = lmLgr
self.metaDB = MetaDB()
self.fileManager = fileMgr
self.regexManager = regexMgr
self.keywordDict = {}
class DataManager(object):
"""
Contructor which is responsible for the data operations
"""
def __init__(self, appConfig, fileMgr, regexMgr, lmLgr):
self.lmLogger = lmLgr
self.metaDB = MetaDB()
self.fileManager = fileMgr
self.regexManager = regexMgr
self.keywordDict = {}
def createCustomer(self):
self.metaDB = MetaDB()
def getMetaDB(self):
return self.metaDB
def addAnalysisToCustomer(self, customer):
self.lmLogger.debug("Adding customer to db.")
"""
findFileNameString function gets the filename of the log file that is currently parsed
file - the file that is parsed to build the metaDB
"""
def findFileNameString(self, file):
filename = os.path.splitext(
os.path.basename(file))[0].__str__().split('.')
fileString = filename[1] + " " + filename[0]
print(fileString)
return fileString
"""
ImportLogFiles function process the ilog files to build hte metaDB
analysisId - the analysisId that is uniquely created
logFilenames - the logfiles passed in the zip file
"""
def importLogFiles(self, analysisId, logFilenames):
if (os.name == 'posix'):
import resource
self.lmLogger.info(
"Building associations from the given log files... Check the ViPR Log Manager log (ViPRLogManager.log), "
"to see which input log file is being analysed. ")
start_time = time.time()
for logfile in logFilenames:
logfilename = os.path.splitext(logfile)
lfname = None
suffix = None
if (len(logfilename) > 0):
lfname = os.path.basename(os.path.normpath(logfilename[0]))
if (len(logfilename) >= 1):
suffix = logfilename[1]
# if(os.name == 'posix'):
# self.lmLogger.info("Memory usage (in MB): " + str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
self.lmLogger.debug("Working on the log file: " + lfname + suffix)
# initialization
endLineNumber = startLineNumber = 0
date = ""
keywordsToWrite = []
negateAssociationFlagMRow = False
#fileString = self.findFileNameString(logfile)
for line in fileinput.input(logfile):
#get the keywords for the current line that is read
keywords, negateAssociationFlag = self.regexManager.getKeywordsForLine(
line)
dateFromLine = self.regexManager.getDateFromLine(line)
# if date is present then it is a new line
if (not dateFromLine == ""):
#if this is not a first line then this is a new line where the fileString is found. So go ahead and store the start and end line numbers
if (not fileinput.isfirstline()
and (len(keywordsToWrite) > 0) and date != ""):
"""
The -if- condition ensures that we have looked the earlier lines for new line with file string. If first line then we should continue until we find the next fileString
write start line and end line.
"""
metaRow = MetaRow(date, fileinput.filename(),
startLineNumber, endLineNumber,
negateAssociationFlagMRow,
list(set(keywordsToWrite)))
self.metaDB.InsertData(metaRow)
lastInd = self.metaDB.getLastIndexFromMetaDB()
for eachKeyword in keywordsToWrite:
self.keywordDict.setdefault(eachKeyword,
list()).append(lastInd)
#reset the keywordsToWrite for next block
keywordsToWrite = []
negateAssociationFlagMRow = False
#This is the line number of the new line
endLineNumber = startLineNumber = fileinput.filelineno()
#get the date here, not earlier since this is the new line
date = self.regexManager.getDateFromLine(line)
#reset the list of keywords for the new block
keywordsToWrite += keywords
# logical or on the negate flag
negateAssociationFlagMRow = negateAssociationFlagMRow or negateAssociationFlag
else:
#the line is continuation of previous line so keep reading file
endLineNumber = fileinput.filelineno()
#save the read keywords
keywordsToWrite += keywords
negateAssociationFlagMRow = negateAssociationFlagMRow or negateAssociationFlag
else:
#this is the last line
if (len(keywordsToWrite) > 0 and date != ""):
metaRow = MetaRow(date, fileinput.filename(),
startLineNumber, endLineNumber,
negateAssociationFlagMRow,
list(set(keywordsToWrite)))
self.metaDB.InsertData(metaRow)
lastInd = self.metaDB.getLastIndexFromMetaDB()
for eachKeyword in keywordsToWrite:
self.keywordDict.setdefault(eachKeyword,
list()).append(lastInd)
#reset the keywordsToWrite for next block
keywordsToWrite = []
negateAssociationFlagMRow = False
fileinput.close()
self.lmLogger.debug(
"--- time to parse and build associations is %s seconds ---" %
round(time.time() - start_time))
self.lmLogger.debug(
"Total number of metarow (input log file - lines containing keywords) "
+ str(len(self.metaDB.tagsList)))
self.lmLogger.debug("Total number of unique keywords " +
str(len(self.keywordDict)))
self.lmLogger.debug("keyword dict size (in KB): " +
str(sys.getsizeof(self.keywordDict) / 1024))
self.lmLogger.debug("self.metaDB.tagsList size (in KB): " +
str(sys.getsizeof(self.metaDB.tagsList) / 1024))
if (os.name == 'posix'):
self.lmLogger.debug(
"Memory usage after making the associations (in MB): " +
str(resource.getrusage(resource.RUSAGE_SELF).ru_maxrss / 1000))
class Imager:
def __init__(self):
self.meta_db = None
self.total_bytes = 0
def get_total_bytes(self):
return self.total_bytes
# ----- unpack -----
def unpack(self, volume, out_path):
# check for volume path
vol_name = volume.name
if not os.path.exists(out_path):
vol_path = out_path
else:
path = os.path.abspath(out_path)
vol_path = os.path.join(path, vol_name)
if os.path.exists(vol_path):
raise IOError("Unpack directory already exists: "+vol_path)
# check for meta file
meta_path = vol_path + ".xdfmeta"
if os.path.exists(meta_path):
raise IOError("Unpack meta file already exists:"+meta_path)
# check for block dev file
blkdev_path = vol_path + ".blkdev"
if os.path.exists(blkdev_path):
raise IOErrro("Unpack blkdev file aready exists:"+blkdev_path)
# create volume path
self.meta_db = MetaDB()
self.unpack_root(volume, vol_path)
# save meta db
self.meta_db.set_volume_name(volume.name)
self.meta_db.set_root_meta_info(volume.get_meta_info())
self.meta_db.set_dos_type(volume.boot.dos_type)
self.meta_db.save(meta_path)
# save boot code
if volume.boot.boot_code != None:
boot_code_path = vol_path + ".bootcode"
f = open(boot_code_path,"wb")
f.write(volume.boot.boot_code)
f.close()
# save blkdev
f = open(blkdev_path,"wb")
f.write("%s\n" % volume.blkdev.get_chs_str())
f.close()
def unpack_root(self, volume, vol_path):
self.unpack_dir(volume.get_root_dir(), vol_path)
def unpack_dir(self, dir, path):
if not os.path.exists(path):
os.mkdir(path)
for e in dir.get_entries():
self.unpack_node(e, path)
def unpack_node(self, node, path):
name = node.name.name
# store meta info
if self.meta_db != None:
node_path = node.get_node_path_name()
self.meta_db.set_meta_info(node_path, node.meta_info)
# sub dir
if node.is_dir():
sub_dir = os.path.join(path, name)
os.mkdir(sub_dir)
for sub_node in node.get_entries():
self.unpack_node(sub_node, sub_dir)
node.flush()
# file
elif node.is_file():
data = node.get_file_data()
node.flush()
file_path = os.path.join(path, name)
fh = open(file_path, "wb")
fh.write(data)
fh.close()
self.total_bytes += len(data)
# ----- pack -----
def pack(self, in_path, image_file, force=True, options=None):
self.pack_begin(in_path)
blkdev = self.pack_create_blkdev(in_path, image_file, force, options)
if blkdev == None:
raise IOError("Can't create block device for image: "+in_path)
volume = self.pack_create_volume(in_path, blkdev)
if not volume.valid:
raise IOError("Can't create volume for image: "+in_path)
self.pack_root(in_path, volume)
self.pack_end(in_path, volume)
def pack_begin(self, in_path):
# remove trailing slash
if in_path[-1] == '/':
in_path = in_path[:-1]
meta_path = in_path + ".xdfmeta"
if os.path.exists(meta_path):
self.meta_db = MetaDB()
self.meta_db.load(meta_path)
def pack_end(self, in_path, volume):
boot_code_path = in_path + ".bootcode"
if os.path.exists(boot_code_path):
# read boot code
f = open(boot_code_path, "rb")
data = f.read()
f.close()
# write boot code
bb = volume.boot
ok = bb.set_boot_code(data)
if ok:
bb.write()
else:
raise IOError("Invalid Boot Code")
def pack_create_blkdev(self, in_path, image_file, force=True, options=None):
# try to read options from blkdev file
if options == None or len(options) == 0:
blkdev_path = in_path + ".blkdev"
if os.path.exists(blkdev_path):
f = open(blkdev_path, "rb")
options = {}
for line in f:
KeyValue.parse_key_value_string(line, options)
f.close()
f = BlkDevFactory()
return f.create(image_file, force=force, options=options)
def pack_create_volume(self, in_path, blkdev):
if self.meta_db != None:
name = self.meta_db.get_volume_name()
meta_info = self.meta_db.get_root_meta_info()
dos_type = self.meta_db.get_dos_type()
else:
# try to derive volume name from image name
if in_path == None or in_path == "":
raise IOError("Invalid pack input path!")
# remove trailing slash
if in_path[-1] == '/':
in_path = in_path[:-1]
name = os.path.basename(in_path)
meta_info = None
dos_type = DosType.DOS0
volume = ADFSVolume(blkdev)
volume.create(name, meta_info, dos_type=dos_type)
return volume
def pack_root(self, in_path, volume):
self.pack_dir(in_path, volume.get_root_dir())
def pack_dir(self, in_path, parent_node):
path = os.path.abspath(in_path)
if not os.path.exists(path):
raise IOError("Pack directory does not exist: "+path)
for name in os.listdir(in_path):
sub_path = os.path.join(in_path, name)
self.pack_entry(sub_path, parent_node)
def pack_entry(self, in_path, parent_node):
ami_name = os.path.basename(in_path)
# retrieve meta info for path from DB
if self.meta_db != None:
ami_path = parent_node.get_node_path_name()
if ami_path != "":
ami_path += "/" + ami_name
else:
ami_path = ami_name
meta_info = self.meta_db.get_meta_info(ami_path)
else:
meta_info = None
# pack directory
if os.path.isdir(in_path):
node = parent_node.create_dir(ami_name, meta_info, False)
for name in os.listdir(in_path):
sub_path = os.path.join(in_path, name)
self.pack_entry(sub_path, node)
node.flush()
# pack file
elif os.path.isfile(in_path):
# read file
fh = open(in_path, "rb")
data = fh.read()
fh.close()
node = parent_node.create_file(ami_name, data, meta_info, False)
node.flush()
self.total_bytes += len(data)
