def _get_previews(self):
"""Extract previews."""
if len(self.metadata.get_preview_properties()) > 0:
# Fetched previews key.
self.results["preview"] = []
for preview in self.metadata.get_preview_properties():
p = AutoVivification()
p["mime_type"] = preview.get_mime_type()
p["size"] = len(self.metadata.get_preview_image(preview).get_data())
p["ext"] = preview.get_extension()
p["dimensions"] = [preview.get_width(), preview.get_height()]
# Resize if needed, and store.
try:
img = utils.str2image(self.metadata.get_preview_image(preview).get_data())
if preview.get_width() > 256 or preview.get_height() > 160:
p["original_file"] = save_file(utils.image2str(img), content_type="image/jpeg")
img.thumbnail([256, 160], Image.ANTIALIAS)
p["file"] = save_file(utils.image2str(img), content_type="image/jpeg")
except Exception as e:
logger.warning("Error reading preview: {0}".format(e))
continue
finally:
# Save.
self.results["preview"].append(p)
def edit(path):
if request.args.get('f') == 'config':
config = db.load_config()
return_resource = db.Resource(config['source_dir'], path)
return_path = return_resource.path
resource = db.Resource(move_path=db.CONFIG)
internal = True
else:
config = db.load_config()
resource = db.Resource(config['source_dir'], path)
return_path = resource.path
return_resource = resource
internal = False
if not resource.is_file(): abort(404)
if request.form:
db.save_file(resource.os_path, request.form['content'])
if internal:
if return_resource.is_file():
return flask.redirect(url_for('view', path=return_resource.path))
else:
return flask.redirect(url_for('list', path=return_resource.path))
else:
return flask.redirect(url_for('view', path=resource.path))
content = db.load_file(resource.os_path)
return render_template('edit.jinja2',
return_path=return_path,
return_resource=return_resource,
content=content,
resource=resource,
internal=internal
)
def _get_previews(self):
"""Extract previews."""
if len(self.metadata.get_preview_properties()) > 0:
# Fetched previews key.
self.results["metadata"]["preview"] = []
for preview in self.metadata.get_preview_properties():
p = AutoVivification()
p["mime_type"] = preview.get_mime_type()
p["size"] = len(self.metadata.get_preview_image(preview).get_data())
p["ext"] = preview.get_extension()
p["dimensions"] = [preview.get_width(), preview.get_height()]
# Resize if needed, and store.
try:
img = str2image(self.metadata.get_preview_image(preview).get_data())
if preview.get_width() > 256 or preview.get_height() > 160:
p["original_file"] = save_file(image2str(img), content_type="image/jpeg")
img.thumbnail([256, 160], Image.ANTIALIAS)
p["file"] = save_file(image2str(img), content_type="image/jpeg")
except Exception as e:
logger.warning("Error reading preview: {0}".format(e))
continue
finally:
# Save.
self.results["metadata"]["preview"].append(p)
def new_image(request):
"""Upload a new image."""
user = api_authenticate(request.POST.get("api_key"))
if request.POST.get("case_id"):
case = get_object_or_404(Case, pk=request.POST.get("case_id"))
# Security check.
if not case.can_write(user):
return HttpResponse("You are not authorized to add image to this", status=400)
if case.state == "C":
return HttpResponse("You cannot add an image to a closed case", status=400)
else:
case = None
task = Analysis.add_task(request.FILES["image"].temporary_file_path(),
file_name=request.FILES["image"].name, case=case, user=user,
content_type=request.FILES["image"].content_type,
image_id=save_file(file_path=request.FILES["image"].temporary_file_path(),
content_type=request.FILES["image"].content_type),
thumb_id=create_thumb(request.FILES["image"].temporary_file_path()))
# Auditing.
log_activity("I",
"Created new analysis via API %s" % task.file_name,
request,
user=user)
response_data = {"id": task.id}
return HttpResponse(json.dumps(response_data), content_type="application/json")
def new_image(request, case_id):
"""Upload a new image."""
case = get_object_or_404(Case, pk=case_id)
# Security check.
if not request.user.is_superuser and not request.user in case.users.all():
return render_to_response("error.html",
{"error": "You are not authorized to add image to this."},
context_instance=RequestContext(request))
if case.state == "C":
return render_to_response("error.html",
{"error": "You cannot add an image to a closed case."},
context_instance=RequestContext(request))
if request.method == "POST":
form = forms.UploadImageForm(request.POST, request.FILES)
if form.is_valid():
content_type = get_content_type_from_file(request.FILES["image"].temporary_file_path())
task = Analysis.add_task(request.FILES["image"].temporary_file_path(), case=case,
user=request.user, content_type=content_type,
image_id=save_file(file_path=request.FILES["image"].temporary_file_path(),
content_type=content_type),
thumb_id=create_thumb(request.FILES["image"].temporary_file_path()),
file_name=request.FILES["image"].name)
# Auditing.
log_activity("I",
"Created new analysis %s" % task.file_name,
request)
# Response designed for Plupload component.
response = HttpResponse('{"jsonrpc": "2.0", "result": null, "id": "id"}', content_type="application/json")
# Never cache AJAX response.
response["Expires"] = "Mon, 1 Jan 2000 01:00:00 GMT"
response["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"
response["Pragma"] = "no-cache"
return response
else:
# Deal with a validation error. We are using Plupload which basically is an AJAX component
# so we have to deal with custom validation errors passing in JSON.
# Plupload needs a status code 200/OK to get additional data passed from the web server.
response = HttpResponse(json.dumps({"jsonrpc" : "2.0",
"error" : {"code": 88,
"message": " ".join([(" ".join([force_text(i) for i in v])) for k, v in form.errors.items()])},
"id" : "id"}),
content_type="application/json")
# Never cache AJAX response.
response["Expires"] = "Mon, 1 Jan 2000 01:00:00 GMT"
response["Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"
response["Pragma"] = "no-cache"
return response
else:
# Request is not a POST.
form = forms.UploadImageForm()
return render_to_response("analyses/images/new_image.html",
{"form": form, "case": case},
context_instance=RequestContext(request))
def run(self, task):
# NOTE: used to allow processing of malformed images.
# With default settings they raise errors like "IOError: image file is truncated (0 bytes not processed)".
# See: http://stackoverflow.com/questions/9211719/python-pil-image-error-after-image-load
ImageFile.LOAD_TRUNCATED_IMAGES = True
# Check this kind of analysis should be run on this file type.
# It can be applied only on PNG and JPEG formats.
if "mime_type" in self.data and not (self.data["mime_type"] == "image/png" or self.data["mime_type"] == "image/jpeg"):
return self.results
# Create temporary file.
handle, resaved = tempfile.mkstemp()
tmp_file = str2temp_file(task.get_file_data)
# Open file and resave it.
try:
im = Image.open(tmp_file.name)
im.save(resaved, "JPEG", quality=95)
resaved_im = Image.open(resaved)
except IOError as e:
logger.warning("[Task {0}]: ELA error opening image: {1}".format(task.id, e))
return self.results
finally:
tmp_file.close()
os.close(handle)
os.remove(resaved)
# Trick to convert images like PNG to a format comparable with JPEG.
if im.mode != "RGB":
im = im.convert("RGB")
# Create ELA image.
try:
ela_im = ImageChops.difference(im, resaved_im)
except Exception as e:
logger.warning("[Task {0}]: Unable to calculate ELA difference: {0}".format(task.id, e))
return self.results
# Calculate difference
extrema = ela_im.getextrema()
max_diff = max([ex[1] for ex in extrema])
if not max_diff:
return self.results
scale = 255.0/max_diff
ela_im = ImageEnhance.Brightness(ela_im).enhance(scale)
self.results["ela"]["max_difference"] = max([ex[1] for ex in extrema])
# Resize image if it's too big.
width, height = ela_im.size
if width > 1800:
ela_im.thumbnail([1800, 1800], Image.ANTIALIAS)
# Save image.
try:
img = image2str(ela_im)
self.results["ela"]["ela_image"] = save_file(img, content_type="image/jpeg")
except Exception as e:
logger.warning("[Task {0}]: ELA error saving image: {0}".format(task.id, e))
finally:
return self.results
def new_folder(request, case_id):
"""Load files from a local directory."""
case = get_object_or_404(Case, pk=case_id)
# Security check.
if not(request.user.is_superuser or request.user in case.users.all()):
return render_to_response("error.html",
{"error": "You are not authorized to add image to this."},
context_instance=RequestContext(request))
if case.state == "C":
return render_to_response("error.html",
{"error": "You cannot add an image to a closed case."},
context_instance=RequestContext(request))
if request.method == "POST":
form = forms.ImageFolderForm(request.POST)
if form.is_valid():
# Check.
if not os.path.exists(request.POST.get("path")):
return render_to_response("error.html",
{"error": "Folder does not exist."},
context_instance=RequestContext(request))
elif not os.path.isdir(request.POST.get("path")):
return render_to_response("error.html",
{"error": "Folder is not a directory."},
context_instance=RequestContext(request))
# Add all files in directory.
mime = magic.Magic(mime=True)
for file in os.listdir(request.POST.get("path")):
content_type = mime.from_file(os.path.join(request.POST.get("path"), file))
# Check if content type is allowed.
if not check_allowed_content(content_type):
# TODO: add some kind of feedback.
pass
task = Analysis()
task.owner = request.user
task.case = case
task.file_name = file
task.image_id = save_file(file_path=os.path.join(request.POST.get("path"), file),
content_type=content_type)
task.thumb_id = create_thumb(os.path.join(request.POST.get("path"), file))
task.save()
# Auditing.
log_activity("I",
"Created new analysis {0}".format(task.file_name),
request)
return HttpResponseRedirect(reverse("analyses.views.show_case", args=(case.id, "list")))
else:
form = forms.ImageFolderForm()
return render_to_response("analyses/images/new_folder.html",
{"form": form, "case": case},
context_instance=RequestContext(request))
def create_thumb(file_path):
"""Create thumbnail
@param file_path: file path
@return: GridFS ID
"""
try:
thumb = Image.open(file_path)
thumb.thumbnail([200, 150], Image.ANTIALIAS)
return save_file(data=image2str(thumb), content_type="image/jpeg")
except:
return None
def create_thumb(file_path):
"""Create thumbnail
@param file_path: file path
@return: GridFS ID
"""
try:
thumb = Image.open(file_path)
thumb.thumbnail([200, 150], Image.ANTIALIAS)
return save_file(data=image2str(thumb), content_type="image/jpeg")
except:
return None
def _create_file():
result = 'ok'
motive = ''
name = request.args.get('name', '')
path = request.args.get('path', '')
if not name:
result = 'error'
motive = 'Must set a name for the new document'
else:
config = db.load_config()
resource = db.Resource(config['source_dir'], path)
resource.move(name)
try:
db.save_file(resource.os_path, '')
except Exception as e:
result = 'error'
motive = e.message
return jsonify(result=result, motive=motive)
def _create_file():
result = 'ok'
motive = ''
name = request.args.get('name', '')
path = request.args.get('path', '')
if not name:
result = 'error'
motive = 'Must set a name for the new document'
else:
config = db.load_config()
resource = db.Resource(config['source_dir'], path)
resource.move(name)
try:
db.save_file(resource.os_path, '')
except Exception as e:
result = 'error'
motive = e.message
return jsonify(result=result, motive=motive)
def run(self):
"""Run data analysis."""
# Create temporary file.
handle, resaved = tempfile.mkstemp()
# Open file and resave it.
try:
im = Image.open(self.file_path)
im.save(resaved, "JPEG", quality=95)
resaved_im = Image.open(resaved)
except IOError as e:
logger.warning("ELA error opening image: {0}".format(e))
return
finally:
os.close(handle)
os.remove(resaved)
# Trick to convert images like PNG to a format comparable with JPEG.
if im.mode != "RGB":
im = im.convert("RGB")
# Create ELA image.
try:
ela_im = ImageChops.difference(im, resaved_im)
except Exception as e:
logger.warning("Unable to calculate ELA difference: {0}".format(e))
return
# Calculate difference
extrema = ela_im.getextrema()
max_diff = max([ex[1] for ex in extrema])
if not max_diff:
return
scale = 255.0 / max_diff
ela_im = ImageEnhance.Brightness(ela_im).enhance(scale)
self.results["max_difference"] = max([ex[1] for ex in extrema])
# Resize image if it's too big.
width, height = ela_im.size
if width > 1800:
ela_im.thumbnail([1800, 1800], Image.ANTIALIAS)
# Save image.
try:
img = utils.image2str(ela_im)
self.results["ela_image"] = save_file(img,
content_type="image/jpeg")
except Exception as e:
logger.warning("ELA error saving image: {0}".format(e))
finally:
return self.results
def run(self):
"""Run data analysis."""
# Create temporary file.
handle, resaved = tempfile.mkstemp()
# Open file and resave it.
try:
im = Image.open(self.file_path)
im.save(resaved, "JPEG", quality=95)
resaved_im = Image.open(resaved)
except IOError as e:
logger.warning("ELA error opening image: {0}".format(e))
return
finally:
os.close(handle)
os.remove(resaved)
# Trick to convert images like PNG to a format comparable with JPEG.
if im.mode != "RGB":
im = im.convert("RGB")
# Create ELA image.
try:
ela_im = ImageChops.difference(im, resaved_im)
except Exception as e:
logger.warning("Unable to calculate ELA difference: {0}".format(e))
return
# Calculate difference
extrema = ela_im.getextrema()
max_diff = max([ex[1] for ex in extrema])
if not max_diff:
return
scale = 255.0 / max_diff
ela_im = ImageEnhance.Brightness(ela_im).enhance(scale)
self.results["max_difference"] = max([ex[1] for ex in extrema])
# Resize image if it's too big.
width, height = ela_im.size
if width > 1800:
ela_im.thumbnail([1800, 1800], Image.ANTIALIAS)
# Save image.
try:
img = utils.image2str(ela_im)
self.results["ela_image"] = save_file(img, content_type="image/jpeg")
except Exception as e:
logger.warning("ELA error saving image: {0}".format(e))
finally:
return self.results
def edit(path):
if request.args.get('f') == 'config':
config = db.load_config()
return_resource = db.Resource(config['source_dir'], path)
return_path = return_resource.path
resource = db.Resource(move_path=db.CONFIG)
internal = True
else:
config = db.load_config()
resource = db.Resource(config['source_dir'], path)
return_path = resource.path
return_resource = resource
internal = False
if not resource.is_file(): abort(404)
if request.form:
db.save_file(resource.os_path, request.form['content'])
if internal:
if return_resource.is_file():
return flask.redirect(
url_for('view', path=return_resource.path))
else:
return flask.redirect(
url_for('list', path=return_resource.path))
else:
return flask.redirect(url_for('view', path=resource.path))
content = db.load_file(resource.os_path)
return render_template('edit.jinja2',
return_path=return_path,
return_resource=return_resource,
content=content,
resource=resource,
internal=internal)
def create_thumb(file_path):
"""Create thumbnail
@param file_path: file path
@return: GridFS ID
"""
try:
thumb = Image.open(file_path)
# Check if conversion to RGBA is needed (i.e. for PNG files).
if thumb.mode is not "RGBA":
thumb.convert("RGBA")
thumb.thumbnail([200, 150], Image.ANTIALIAS)
return save_file(data=image2str(thumb), content_type="image/jpeg")
except:
return None
def create_thumb(file_path):
"""Create thumbnail
@param file_path: file path
@return: GridFS ID
"""
try:
thumb = Image.open(file_path)
# Check if conversion to RGBA is needed (i.e. for PNG files).
if thumb.mode is not "RGBA":
thumb = thumb.convert("RGBA")
thumb.thumbnail([200, 150], Image.ANTIALIAS)
return save_file(data=image2str(thumb), content_type="image/jpeg")
except:
return None
def _add_task(self, file, case, user):
"""Adds a new task to database.
@param file: file path
@param case: case id
@param user: user id
"""
# File type check.
mime = magic.Magic(mime=True)
content_type = mime.from_file(file)
if not check_allowed_content(content_type):
print "WARNING: Skipping %s: file type not allowed." % file
else:
# Add to analysis queue.
task = Analysis()
task.owner = user
task.case = case
task.file_name = os.path.basename(file)
task.image_id = save_file(file_path=file, content_type=content_type)
task.thumb_id = create_thumb(file)
task.save()
def add_task(file_path, file_name=None, case=None, user=None, content_type=None, image_id=None, thumb_id=None):
"""Adds a new task to database.
@param file_path: file path
@param file_name: file name
@param case: case id
@param user: user id
@param content_type: file content type
@param image_id: original image gridfs id
@param thumb_id: thumbnail gridfs id
"""
# TODO: re enable with py3 support.
# assert isinstance(file_path, str)
# File name.
if not file_name:
file_name = os.path.basename(file_path)
# File type check.
if not content_type:
content_type = get_content_type_from_file(file_path)
# If image is not already stored on gridfs.
if not image_id:
image_id = save_file(file_path=file_path, content_type=content_type)
# If image thumbnail is available.
if not thumb_id:
thumb_id = create_thumb(file_path)
# Check on allowed file type.
if not check_allowed_content(content_type):
raise GhiroValidationException("Skipping %s: file type not allowed." % file_name)
else:
# Add to analysis queue.
return Analysis.objects.create(
owner=user, case=case, file_name=file_name, image_id=image_id, thumb_id=thumb_id
)
def new_url(request, case_id):
"""Upload a new image via URL."""
case = get_object_or_404(Case, pk=case_id)
# Security check.
if not request.user.is_superuser and not request.user in case.users.all():
return render_to_response("error.html",
{"error": "You are not authorized to add image to this."},
context_instance=RequestContext(request))
if case.state == "C":
return render_to_response("error.html",
{"error": "You cannot add an image to a closed case."},
context_instance=RequestContext(request))
if request.method == "POST":
form = forms.UrlForm(request.POST)
if form.is_valid():
# Download file.
try:
url = urllib2.urlopen(request.POST.get("url"), timeout=5)
except urllib2.URLError as e:
if hasattr(e, "reason"):
return render_to_response("error.html",
{"error": "We failed to reach a server, reason: %s" % e.reason},
context_instance=RequestContext(request))
elif hasattr(e, "code"):
return render_to_response("error.html",
{"error": "The remote server couldn't fulfill the request, HTTP error code %s" % e.code},
context_instance=RequestContext(request))
# Store temp file.
url_temp = NamedTemporaryFile(delete=True)
url_temp.write(url.read())
url_temp.flush()
# Convert to File object.
url_file = File(url_temp).name
# Check content type.
mime = magic.Magic(mime=True)
content_type = mime.from_file(url_file)
if not check_allowed_content(content_type):
return render_to_response("error.html",
{"error": "File type not supported"},
context_instance=RequestContext(request))
# Create analysis task.
task = Analysis()
task.owner = request.user
task.case = case
task.file_name = os.path.basename(urlparse.urlparse(request.POST.get("url")).path)
task.image_id = save_file(file_path=url_file, content_type=content_type)
task.thumb_id = create_thumb(url_file)
task.save()
# Auditing.
log_activity("I",
"Created new analysis {0} from URL {1}".format(task.file_name, request.POST.get("url")),
request)
return HttpResponseRedirect(reverse("analyses.views.show_case", args=(case.id, "list")))
else:
# Request is not a POST.
form = forms.UrlForm()
return render_to_response("analyses/images/new_url.html",
{"form": form, "case": case},
context_instance=RequestContext(request))
status=400)
if case.state == "C":
return HttpResponse("You cannot add an image to a closed case",
status=400)
else:
case = None
task = Analysis.add_task(
request.FILES["image"].temporary_file_path(),
file_name=request.FILES["image"].name,
case=case,
user=user,
content_type=request.FILES["image"].content_type,
image_id=save_file(
file_path=request.FILES["image"].temporary_file_path(),
content_type=request.FILES["image"].content_type),
thumb_id=create_thumb(request.FILES["image"].temporary_file_path()))
# Auditing.
log_activity("I",
"Created new analysis via API %s" % task.file_name,
request,
user=user)
response_data = {"id": task.id}
return HttpResponse(json.dumps(response_data),
content_type="application/json")
@require_POST
class Command(BaseCommand):
"""Image submission via command line."""
option_list = BaseCommand.option_list + (
make_option("--target",
"-t",
dest="target",
help="Path of the file or directory to submit"),
make_option("--case",
"-c",
dest="case",
help="Case ID, images will be attached to it"),
make_option("--username", "-u", dest="username", help="Username"),
make_option("--recurse",
"-r",
dest="recurse",
default=False,
action="store_true",
help="Recurse inside subdirectories"),
)
help = "Task submission"
def handle(self, *args, **options):
"""Runs command."""
# Validation.
if not options["username"] or not options["case"] or not options[
"target"]:
print "Options -t (target), -c (case) and -u (user are mandatory. Exiting."
sys.exit(1)
# Get options.
user = Profile.objects.get(username=options["username"].strip())
case = Case.objects.get(pk=options["case"].strip())
# Add directory or files.
if os.path.isdir(options["target"]) and options["recurse"]:
for dirname, dirnames, filenames in os.walk(options["target"]):
for filename in filenames:
target = os.path.join(dirname, filename)
print "INFO: processing {0}".format(target)
self._add_task(target, case, user)
elif os.path.isdir(options["target"]):
for file_name in os.listdir(options["target"]):
print "INFO: processing {0}".format(file_name)
self._add_task(os.path.join(options["target"], file_name),
case, user)
elif os.path.isfile(options["target"]):
print "INFO: processing {0}".format(options["target"])
self._add_task(options["target"], case, user)
else:
print "ERROR: target is not a file or directory"
def _add_task(self, file, case, user):
"""Adds a new task to database.
@param file: file path
@param case: case id
@param user: user id
"""
# File type check.
mime = magic.Magic(mime=True)
content_type = mime.from_file(file)
if not check_allowed_content(content_type):
print "WARNING: Skipping %s: file type not allowed." % file
else:
# Add to analysis queue.
task = Analysis()
task.owner = user
task.case = case
task.file_name = os.path.basename(file)
task.image_id = save_file(file_path=file,
content_type=content_type)
task.thumb_id = create_thumb(file)
task.save()
@param thumb_id: thumbnail gridfs id
"""
# TODO: re enable with py3 support.
# assert isinstance(file_path, str)
# File name.
if not file_name:
file_name = os.path.basename(file_path)
# File type check.
if not content_type:
content_type = get_content_type_from_file(file_path)
# If image is not already stored on gridfs.
if not image_id:
image_id = save_file(file_path=file_path,
content_type=content_type)
# If image thumbnail is available.
if not thumb_id:
thumb_id = create_thumb(file_path)
# Check on allowed file type.
if not check_allowed_content(content_type):
raise GhiroValidationException(
"Skipping %s: file type not allowed." % file_name)
else:
# Add to analysis queue.
return Analysis.objects.create(owner=user,
case=case,
file_name=file_name,
image_id=image_id,
if case.state == "C":
return render_to_response(
"error.html",
{"error": "You cannot add an image to a closed case."},
context_instance=RequestContext(request))
if request.method == "POST":
form = forms.UploadImageForm(request.POST, request.FILES)
if form.is_valid():
task = form.save(commit=False)
task.owner = request.user
task.case = case
task.file_name = request.FILES["image"].name
task.image_id = save_file(
file_path=request.FILES["image"].temporary_file_path(),
content_type=request.FILES["image"].content_type)
task.thumb_id = create_thumb(
request.FILES["image"].temporary_file_path())
task.save()
# Auditing.
log_activity("I", "Created new analysis %s" % task.file_name,
request)
# Response designed for Plupload component.
response = HttpResponse(
'{"jsonrpc": "2.0", "result": null, "id": "id"}',
content_type="application/json")
# Never cache AJAX response.
response["Expires"] = "Mon, 1 Jan 2000 01:00:00 GMT"
response[
"Cache-Control"] = "no-store, no-cache, must-revalidate, post-check=0, pre-check=0"
