def upload(request):
desc = request.POST['description']
brnd = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
f = request.FILES['file']
path = settings.UPLOAD_DIR + f.name
handle_uploaded_file(f, path)
md5 = Extractor.io_md5(path)
deleteOld(md5)
brand=get_brand(brnd)
print("Brand: " + str(brand))
image = Image(filename=f.name,description=desc,brand_id=brand,hash=md5, rootfs_extracted=False, kernel_extracted=False)
image.save()
FILE_PATH = unicodedata.normalize('NFKD', settings.UPLOAD_DIR+image.filename).encode('ascii','ignore')
#Add a product related to the image (commtented for easier debugging)
# product = Product(iid=image,product=mode,version=vers)
# product.save()
# print image
# print product
#rootfs=True, parallel=False, ,kernel=False,
print("Image ID: "+str(image.id))
#Extract filesystem from firmware file
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False, '127.0.0.1' ,"Netgear")
print('extract--------------------------//')
#We should handle possible errors here
extract.extract()
os.chdir(settings.BASE_DIR)
curimg=str(image.id)+".tar.gz"
#Get architecture and add it in db
outp = subprocess.check_output("./lib/getArch.sh ./extracted/"+curimg, shell=True)
print(outp)
res = outp.split()
iid, files2oids, links, cur = tar2db(str(image.id),'./extracted/'+curimg)
files = object_to_img(iid,files2oids,links)
hierarchy = parseFilesToHierarchy(files)
#print(hierarchy)
image.hierarchy = "[" + (', '.join([json.dumps(x) for x in hierarchy])) + "]"
image.save()
find_treasures(image)
grepfs(image)
print("Architecture: "+res[0])
print("IID: "+res[1])
return HttpResponse("File uploaded // hash : %s" % md5)
def upload(request):
desc = request.POST['description']
brnd = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
f = request.FILES['file']
path = settings.UPLOAD_DIR + f.name
handle_uploaded_file(f, path)
md5 = Extractor.io_md5(path)
brand = get_brand(brnd)
image = Image(filename=f.name,
description=desc,
brand_id=brand,
hash=md5,
rootfs_extracted=False,
kernel_extracted=False)
image.save()
FILE_PATH = unicodedata.normalize(
'NFKD',
settings.UPLOAD_DIR + image.filename).encode('ascii', 'ignore')
#Add a product related to the image (commtented for easier debugging)
# product = Product(iid=image,product=mode,version=vers)
# product.save()
# print image
# print product
#rootfs=True, parallel=False, ,kernel=False,
print(brand)
print(image.id)
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False,
'127.0.0.1', "Netgear")
extract.extract()
#extract_tar_tmp(image.id)
os.chdir(settings.BASE_DIR)
curimg = str(image.id) + ".tar.gz"
#run("./lib/getArch.sh ./extracted/"+curimg)
outp = subprocess.check_output("./lib/getArch.sh ./extracted/" + curimg,
shell=True)
res = outp.split()
tar2db(str(image.id), './extracted/' + curimg)
print(res)
print("Architecture: " + res[0])
print("IID: " + res[1])
return HttpResponse("File uploaded // hash : %s" % md5)
def process(self, image):
self.set_image_status(image, "processing")
FILE_PATH = unicodedata.normalize('NFKD', unicode(settings.UPLOAD_DIR+image.filename)).encode('ascii','ignore')
self.stdout.write("Image ID: "+str(image.id))
#Extract filesystem from firmware file
self.stdout.write(FILE_PATH)
self.stdout.write(settings.EXTRACTED_DIR)
try:
brand_obj = image.brand
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False, '127.0.0.1' ,brand_obj.name)
self.stdout.write('extract--------------------------//')
#We should handle possible errors here
extract.extract()
os.chdir(settings.BASE_DIR)
curimg=str(image.id)+".tar.gz"
self.extract_tar_tmp(image.id)
except NotImplementedError:
self.set_image_status(image, "extract failed")
self.stdout.write("Error")
self.set_image_status(image, "30")
self.stdout.write(os.getcwd())
iid, files2oids, links, cur = tar2db(str(image.id),'./extracted/'+curimg)
files = self.object_to_img(iid,files2oids,links)
hierarchy = parseFilesToHierarchy(files)
image.hierarchy = "[" + (', '.join([json.dumps(x) for x in hierarchy])) + "]"
image.save()
#Get architecture and add it in db
self.set_image_status(image, "60")
try:
outp = subprocess.check_output("./lib/getArch.sh extracted/"+curimg, shell=True)
self.find_treasures(image)
self.grepfs(image)
res = outp.split()
self.stdout.write("Architecture: "+res[0])
self.stdout.write("IID: "+res[1])
self.stdout.write('----------------------')
except subprocess.CalledProcessError as e:
self.set_image_status(image, "arch detection failed")
self.stdout.write("GetArch failed a bit...")
self.set_image_status(image, "done")
def extract(self):
candidates = {}
self.output = os.path.join(self.workspace, "extracted")
print("Start extract")
extractor = FirmadyneExtractor(indir=self.input,
outdir=self.output,
rootfs=True,
kernel=True,
numproc=True,
server=None,
brand=None)
extractor.extract()
self.process_extraction()
print("Extract completed")
return os.path.join(self.workspace, "extracted")
def extract(self):
candidates = {}
self.output = os.path.join(self.workspace, "extracted")
print("Start extract")
extractor = FirmadyneExtractor(indir=self.input,
outdir=self.output,
rootfs=True,
kernel=True,
numproc=True,
server=None,
brand=None)
extractor.extract()
self.process_extraction()
print("Extract completed")
return os.path.join(self.workspace, "extracted")
def test_with_inner_struct():
testcase = os.path.join(os.path.dirname(__file__), 'testcases',
'extractor', 'test_extractor_inner_struct.h')
defs = [definition for definition in Extractor(testcase)]
with open(testcase, 'rb') as f:
content = [s.strip() for s in f.read().decode('utf-8').split('=' * 20)]
assert defs == content
def test_simple():
testcase = os.path.join(os.path.dirname(__file__), 'testcases',
'extractor', 'test_extractor_simple.h')
defs = list(iter(Extractor(testcase)))
with open(testcase) as f:
content = f.read().strip()
assert len(defs) == 1
assert defs[0] == content
def upload(request):
desc = request.POST['description']
brnd = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
f = request.FILES['file']
path = settings.UPLOAD_DIR + f.name
handle_uploaded_file(f, path)
md5 = Extractor.io_md5(path)
brand=get_brand(brnd)
image = Image(filename=f.name,description=desc,brand_id=brand,hash=md5, rootfs_extracted=False, kernel_extracted=False)
image.save()
FILE_PATH = unicodedata.normalize('NFKD', settings.UPLOAD_DIR+image.filename).encode('ascii','ignore')
#Add a product related to the image (commtented for easier debugging)
# product = Product(iid=image,product=mode,version=vers)
# product.save()
# print image
# print product
#rootfs=True, parallel=False, ,kernel=False,
print(brand)
print(image.id)
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False, '127.0.0.1' ,"Netgear")
extract.extract()
#extract_tar_tmp(image.id)
os.chdir(settings.BASE_DIR)
curimg=str(image.id)+".tar.gz"
#run("./lib/getArch.sh ./extracted/"+curimg)
outp = subprocess.check_output("./lib/getArch.sh ./extracted/"+curimg, shell=True)
res = outp.split()
tar2db(str(image.id),'./extracted/'+curimg)
print(res)
print("Architecture: "+res[0])
print("IID: "+res[1])
return HttpResponse("File uploaded // hash : %s" % md5)
def test_multiple_definitions():
testcase = os.path.join(os.path.dirname(__file__), 'testcases',
'extractor',
'test_extractor_multiple_definitions.h')
defs = list(Extractor(testcase))
print(defs)
with open(testcase, 'rb') as f:
content = [s.strip() for s in f.read().decode('utf-8').split('=' * 20)]
assert defs == content
def upload(request):
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
desc = request.POST['description']
brand = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
f = request.FILES['file']
file_name = f.name
path = settings.UPLOAD_DIR + file_name
write_file(f, path)
md5 = Extractor.io_md5(path)
brand_obj = get_brand(brand)
brand_id = brand_obj.id
print("Brand: " + str(brand_id))
deleteOld(md5)
image = Image(filename = file_name,
description = desc,
brand_id = brand_id,
hash = md5,
rootfs_extracted=False,
kernel_extracted=False)
fsize = sizeof_fmt(os.path.getsize(path))
image.filesize = fsize
try:
image.save()
return JsonResponse({"status": "new", "hash": md5})
except IntegrityError:
# Firmware already processed
#return JsonResponse({"status": "repost", "hash": md5})
print ("repost")
def upload(request):
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
desc = request.POST['description']
brand = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
f = request.FILES['file']
file_name = f.name
path = settings.UPLOAD_DIR + file_name
write_file(f, path)
md5 = Extractor.io_md5(path)
brand_obj = get_brand(brand)
brand_id = brand_obj.id
print("Brand: " + str(brand_id))
deleteOld(md5)
image = Image(filename=file_name,
description=desc,
brand_id=brand_id,
hash=md5,
rootfs_extracted=False,
kernel_extracted=False)
fsize = sizeof_fmt(os.path.getsize(path))
image.filesize = fsize
try:
image.save()
return JsonResponse({"status": "new", "hash": md5})
except IntegrityError:
# Firmware already processed
#return JsonResponse({"status": "repost", "hash": md5})
print("repost")
def upload(request):
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
desc = request.POST['description']
brand = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
f = request.FILES['file']
file_name = f.name
path = settings.UPLOAD_DIR + file_name
write_file(f, path)
md5 = Extractor.io_md5(path)
brand_obj = get_brand(brand)
brand_id = brand_obj.id
print("Brand: " + str(brand_id))
deleteOld(md5)
image = Image(filename = file_name,
description = desc,
brand_id = brand_id,
hash = md5,
rootfs_extracted = False,
kernel_extracted=False)
fsize = sizeof_fmt(os.path.getsize(path))
image.filesize = fsize
try:
image.save()
except IntegrityError:
# Firmware already processed
#return JsonResponse({"status": "repost", "hash": md5})
print ("repost")
FILE_PATH = unicodedata.normalize('NFKD', unicode(settings.UPLOAD_DIR+image.filename)).encode('ascii','ignore')
#Add a product related to the image
# product = Product(iid=image,product=mode,version=vers)
# product.save()
print("Image ID: "+str(image.id))
#Extract filesystem from firmware file
print(FILE_PATH)
print(settings.EXTRACTED_DIR)
try:
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False, '127.0.0.1' ,brand_obj.name)
print('extract--------------------------//')
#We should handle possible errors here
extract.extract()
os.chdir(settings.BASE_DIR)
curimg=str(image.id)+".tar.gz"
print image.id
extract_tar_tmp(image.id)
except NotImplementedError:
return JsonResponse({"error": "extract error"})
print(os.getcwd())
iid, files2oids, links, cur = tar2db(str(image.id),'./extracted/'+curimg)
files = object_to_img(iid,files2oids,links)
hierarchy = parseFilesToHierarchy(files)
myimg=Image.objects.get(hash=md5)
myimg.hierarchy = "[" + (', '.join([json.dumps(x) for x in hierarchy])) + "]"
myimg.save()
#Get architecture and add it in db
outp = subprocess.check_output("./lib/getArch.sh extracted/"+curimg, shell=True)
print(outp)
find_treasures(image)
grepfs(image)
res = outp.split()
print("Architecture: "+res[0])
print("IID: "+res[1])
print('----------------------')
"""
print(os.environ["FIRMWARE_DIR"])
os.chdir(os.environ["FIRMWARE_DIR"])
print(os.getcwd())
"""
return JsonResponse({"status": "new", "hash": md5})
from lib.extractor import Extractor
import sys
if __name__ == '__main__':
extractor = Extractor(sys.argv[1])
for box in extractor.boxes():
print(box)
def check_object(path, protocol):
(status, value) = Extractor(path, protocol).extract()
# log.debug(path)
# log.debug(value)
return Finder(value).find() if status == cfg.STATUS_PROC else (status,
value)
def test_no_such_file():
testcase = os.path.join(os.path.dirname(__file__), 'testcases',
'extractor', 'no_such_file.h')
Extractor(testcase)
def upload(request):
desc = request.POST['description']
brnd = request.POST['brand']
vers = request.POST['version']
mode = request.POST['model']
if not request.method == 'POST':
return HttpResponse("POST only")
if not 'file' in request.FILES:
return HttpResponse("No file")
f = request.FILES['file']
path = settings.UPLOAD_DIR + f.name
handle_uploaded_file(f, path)
md5 = Extractor.io_md5(path)
brand = get_brand(brnd)
print("Brand: " + brand)
image = Image(filename=f.name,
description=desc,
brand_id=brand,
hash=md5,
rootfs_extracted=False,
kernel_extracted=False)
image.save()
FILE_PATH = unicodedata.normalize(
'NFKD',
settings.UPLOAD_DIR + image.filename).encode('ascii', 'ignore')
#Add a product related to the image (commtented for easier debugging)
# product = Product(iid=image,product=mode,version=vers)
# product.save()
# print image
# print product
#rootfs=True, parallel=False, ,kernel=False,
print("Image ID: " + image.id)
#Extract filesystem from firmware file
extract = Extractor(FILE_PATH, settings.EXTRACTED_DIR, True, False, False,
'127.0.0.1', "Netgear")
extract.extract()
#To decompress .tar.gz in /tmp
#extract_tar_tmp(image.id)
os.chdir(settings.BASE_DIR)
curimg = str(image.id) + ".tar.gz"
#Get architecture and add it in db
outp = subprocess.check_output("./lib/getArch.sh ./extracted/" + curimg,
shell=True)
res = outp.split()
iid, files2oids, links, cur = tar2db(str(image.id),
'./extracted/' + curimg)
#Get file hierarchy and save it in db
hierarchy = parseFilesToHierarchy(files2oids, links)
print(hierarchy)
image.hierarchy = ', '.join([str(x) for x in hierarchy])
image.save()
#Add filenames/path in db
object_to_img(iid, files2oids, links)
print("Architecture: " + res[0])
print("IID: " + res[1])
return HttpResponse("File uploaded // hash : %s" % md5)