def test_root_joliet(self):
for filename, content in TEST_DATA:
# Rebuild content with utf-16be data (for Joliet schemes)
def reencode(content):
ret = {}
for k, v in content.items():
# When characters aren't allowed, they are replaced with _
if b'?' in k:
k = k.replace(b"?", b"_")
if b';' in k:
k = k.replace(b";", b"_")
if b'\\' in k:
k = k.replace(b"\\", b"_")
new_name = k.decode('utf-8').encode('utf-16be')
if len(new_name) > 128:
# It will also truncate whitespace
new_name = new_name[0:128]
new_name = new_name.decode('utf-16be').rstrip().encode(
'utf-16be')
if isinstance(v, dict):
v = reencode(v)
ret[new_name] = v
return ret
ucs2_content = reencode(content)
iso = isoparser.parse(filename)
self.assertEqual(len(iso.root.children), len(ucs2_content))
self.recursive_test_record(iso.root, ucs2_content)
iso.close()
def parsecontent(filename):
print '[+] Parsing ISO file for suspicious content: ' + filename
parsediso = isoparser.parse(filename)
print '[+] Looking for artifacts....'
if parsediso.root.children:
parsefilecontent(parsediso.root)
def use_iso(self, iso):
iso = isoparser.parse(iso)
content = self._find_iso_content(iso)
content = io.BytesIO(content)
context = etree.iterparse(content)
for action, elem in context:
if elem.text:
text = elem.text
if elem.tag == 'version':
self._values['version'] = text
elif elem.tag == 'buildNumber':
self._values['build'] = text
def touch_iso_file(file_name):
"""Gets the latest date/time found in the ISO file and then touches the .iso
with that date/time
"""
# Try and grab the date/time from the file contents.
try:
iso = isoparser.parse(file_name)
early_datetime = datetime.datetime(1960,1,1,0,0)
isofile_mod_time = iso_parse_path_rec(iso.record().children, early_datetime)
if isofile_mod_time > early_datetime:
touch_file(file_name, time.mktime(isofile_mod_time.timetuple()))
except:
pass
def iso_info(self, iso):
result = dict(product=None, version=None, build=None)
iso = isoparser.parse(iso)
content = iso.record('/METADATA.XML').content
content = io.BytesIO(content)
context = etree.iterparse(content)
for action, elem in context:
if elem.text:
text = elem.text
if elem.tag == 'productName':
result['product'] = text
elif elem.tag == 'version':
result['version'] = text
elif elem.tag == 'buildNumber':
result['build'] = text
return result
def iso_info(self, iso):
result = dict(product=None, version=None, build=None)
iso = isoparser.parse(iso)
content = iso.record("/METADATA.XML").content
content = io.BytesIO(content)
context = etree.iterparse(content)
for action, elem in context:
if elem.text:
text = elem.text
if elem.tag == "productName":
result["product"] = text
elif elem.tag == "version":
result["version"] = text
elif elem.tag == "buildNumber":
result["build"] = text
return result
def _run(self, scanObject, result, depth, args):
moduleResult = []
try:
# Create a temp file so isoparser has a file to analyze
with tempfile.NamedTemporaryFile(dir=self.TEMP_DIR) as temp_file_input:
temp_file_input_name = temp_file_input.name
temp_file_input.write(scanObject.buffer)
temp_file_input.flush()
# Create an iso object
iso = isoparser.parse(temp_file_input_name)
# Loop through iso and identify child object. Write each child object to output directory
for child in iso.root.children:
child_md5 = hashlib.md5(child.content).hexdigest()
moduleResult.append(ModuleObject(buffer=child.content, externalVars=ExternalVars(filename='e_iso_%s' % child_md5)))
except ScanError:
raise
return moduleResult
def iso_info(self, iso):
result = dict(
product=None,
version=None,
build=None
)
iso = isoparser.parse(iso)
content = self._find_iso_content(iso)
content = io.BytesIO(content)
context = etree.iterparse(content)
for action, elem in context:
if elem.text:
text = elem.text
if elem.tag == 'productName':
result['product'] = text
elif elem.tag == 'version':
result['version'] = text
elif elem.tag == 'buildNumber':
result['build'] = text
return result
def ingest_from_iso(self, path):
"""
Ingests NSRL CSV directly from NSRL ISO image.
file.
:param str path: path to NSRL ISO image
"""
with isoparser.parse(path) as iso:
# Check that required filenames are there.
filenames = [c.name for c in iso.root.children]
fmap = self._get_iso_ingest_filenames(filenames)
# Ingest mfg, os and prod (in that order).
for label, key, meth in [("mfg", "NSRLMFG.TXT",
"put_manufacturers"),
("os", "NSRLOS.TXT", "put_oss"),
("prod", "NSRLPROD.TXT", "put_products")]:
self.print("Inserting %s info..." % label, end=" ")
s = time.time()
record = \
[r for r in iso.root.children if r.name == fmap[key]][0]
reader = csv.reader(iso_utf8_readlines(record))
res = getattr(self.client, meth)(reader)
e = time.time()
self.print("done! Put %d in %fs" % (len(res), e - s))
# Copy NSRLFILE.ZIP to tmp (FileStream from isoparser has no seek)
self.print("Creating temporary copy of file info...")
key = "NSRLFILE.ZIP"
file_record = \
[r for r in iso.root.children if r.name == fmap[key]][0]
file_stream = file_record.get_stream()
temp_fd, temp_fp = tempfile.mkstemp(suffix="NSRLFILE.ZIP")
os.close(temp_fd)
try:
with open(temp_fp, "wb") as temp_nsrlfile:
while file_stream.cur_offset < file_record.length:
temp_nsrlfile.write(file_stream.read(1048576))
mb_count = file_stream.cur_offset / 1048576
self.print(" copied %dMb" % mb_count)
self.print("File copy done!")
# Finally, ingest file information.
self.print("Inserting file info...")
s = time.time()
with zipfile.ZipFile(temp_fp) as zf:
reader = \
csv.reader(_zipped_file_readlines(zf, "NSRLFile.txt",
skip_first=True))
count = \
self.client.put_files(reader, verbose=self._verbose)
e = time.time()
self.print("File ingest done! Put %d in %fs" % (count, e - s))
finally:
# Alyways clean up temporary copy.
try:
os.unlink(temp_fp)
except OSError:
pass
#!/usr/bin/env python
import sys
sys.path.append('isoparser')
import errno
import isoparser
import os
import subprocess
import shutil
ISO_FILE = sys.argv[1]
OUTPUT_DIR = sys.argv[2]
VERSION = sys.argv[3]
iso = isoparser.parse(ISO_FILE)
def make_sure_path_exists(path):
try:
os.makedirs(path)
except OSError as exception:
if exception.errno != errno.EEXIST:
raise
def extract(node, path):
make_sure_path_exists(path)
for entry in node.children:
full_path = os.path.join(path, entry.name)
def __init__(self, iso_path):
self.iso_path = iso_path
self._iso = isoparser.parse(iso_path)
def com_iso_load(self, url_file):
"""
Open the iso file for parsing (url or file)
"""
common_global.es_inst.com_elastic_index('info', {"iso url/file": url_file})
self.iso_inst = isoparser.parse(url_file)
def __enter__(self):
self._iso = isoparser.parse(self._filename)
return self
def test_root(self):
for filename, content in TEST_DATA:
iso = isoparser.parse(filename)
self.assertEqual(len(iso.root.children), len(content))
self.recursive_test_record(iso.root, content)
iso.close()
if node.is_directory and len(node.children):
#all = [get_latest_timestamp(entry) for entry in node.children]
#print(all)
oldest = max([get_latest_timestamp(entry) for entry in node.children])
return oldest
else:
#print(node.name, '\t\t', node.datetime)
#return datetime.datetime.strptime(node.datetime, "%Y-%m-%d %H:%M:%S")
return node.datetime
timestamps = []
for filename in isos:
print(filename)
iso = isoparser.parse(filename)
ts = get_latest_timestamp(iso.root)
#print(iso, ts)
#abort()
timestamps += [ts]
sorted_pairs = sorted(zip(isos, timestamps), key=lambda pair: pair[1])
isos, ts = zip(*sorted_pairs)
print('======================')
print('\n'.join([str(p) for p in sorted_pairs]))
#print('OK?'); input()
# pass 2: extract ISOs and commit chronologically
EXTRACTED_ISO_DIR = '../TempleOS_Archive'
SKIP = 1