def add_malware(request):
if request.method == 'POST':
form = AddMalwareForm(request.POST, request.FILES)
if form.is_valid():
file_path = store_sample(request.FILES['file'].read())
obj = File(file_path=file_path)
tags = request.POST.get('tags')
orig_url = request.POST.get('url')
file_name = request.FILES['file'].name
if isinstance(obj, File):
malware_entry = malware(md5=obj.get_md5(),
crc32=obj.get_crc32(),
sha1=obj.get_sha1(),
sha256=obj.get_sha256(),
sha512=obj.get_sha512(),
file_size=obj.get_size(),
file_type=obj.get_type(),
ssdeep=obj.get_ssdeep(),
orig_url=orig_url,
file_name=file_name)
malware_entry.save()
if store_encoded:
encode_sample(file_path)
delete_file(file_path)
if tags:
tags = tags.strip()
if "," in tags:
tags = tags.split(",")
else:
tags = tags.split(" ")
for t in tags:
t = t.strip().lower()
if t == "":
continue
if tag.objects.filter(tag=t).exists():
malware_entry.tags.add(tag.objects.get(tag=t))
continue
malware_entry.tags.add(tag.objects.create(tag=t))
return HttpResponse(
jsonize({"message": file_name + " added to repository"}))
else:
return HttpResponse('METHOD not supported for URL')
def get_files():
""" Return list of all Files from MySQL DB."""
c.execute('select * from files')
files = []
for (_id, name, sha1, modified, data) in c:
files.append(File(name, sha1, modified, data))
return files
def get_file(name):
""" Get a single file (name) from MongoDB as a File object."""
results = collection.find_one({"name": name})
return File(name=results['name'],
sha1=results['sha1'],
modified=results['modified'],
data=results['bin'])
def get_time_sorted_files():
time_sorted_files = []
for name, folder in zip(names, folders):
fileutils.create_directory(folder + 'processed/')
for file_name in fileutils.get_camera_image_names(folder):
if file_name != 'processed' and file_name != 'Thumbs.db' and file_name.find('.lock') is -1:
try:
time_sorted_files.append(
File.File(
name,
folder,
file_name,
None,
None,
None,
None,
None,
None,
None,
fileutils.get_file_mtime(folder, file_name)
)
)
except Exception as e:
print(file_name + ' file is already taken.')
time_sorted_files.sort(key=lambda x: x.getmtime, reverse=False)
return time_sorted_files
def add_malware():
tags = request.forms.get("tags")
data = request.files.file
info = File(file_path=store_sample(data.file.read()))
db.add(obj=info, file_name=data.filename, tags=tags)
return jsonize({"message": "added"})
def app():
# Files for processing
sorted_files = get_time_sorted_files()
# Image objects ready for processing are stored here
image_file_objects = []
# Count of files taken / to take
taken_files_count = 0
max_files_to_take = 4
# Create image objects
for sorted_file in sorted_files:
# File locking check and locking
if not os.path.isfile(sorted_file.file_path + sorted_file.file_name +
'.lock'):
try:
# Limited amount of files
taken_files_count = taken_files_count + 1
if taken_files_count > max_files_to_take:
break
# Lock file and process
open(sorted_file.file_path + sorted_file.file_name + '.lock',
'a').close()
# Append for processing
gm_time = fileutils.get_file_create_time(
sorted_file.file_path, sorted_file.file_name)
file = File.File(
sorted_file.name, sorted_file.file_path,
sorted_file.file_name,
fileutils.get_file_extension(sorted_file.file_path,
sorted_file.file_name),
fileutils.get_file_create_year(gm_time),
fileutils.get_file_create_month(gm_time),
fileutils.get_file_create_day(gm_time),
fileutils.get_file_create_hour(gm_time, time_offset_hours),
fileutils.get_file_create_minute(gm_time),
fileutils.get_file_create_second(gm_time), None)
image_file_objects.append(file)
except FileNotFoundError as e:
pass
# Analyze image objects
for image_object in image_file_objects:
try:
# Also runs all sub processes
object_detection.analyze_image(
image_object,
move_to_processed,
)
except Exception as e:
print(e)
finally:
# Finally remove lock file so those don't pile up
remove_file_lock(image_object.file_path + image_object.file_name +
'.lock')
def add_malware(request):
if request.method == 'POST':
form = AddMalwareForm(request.POST, request.FILES)
if form.is_valid():
file_path=store_sample(request.FILES['file'].read())
obj = File(file_path=file_path)
tags = request.POST.get('tags')
orig_url = request.POST.get('url')
file_name = request.FILES['file'].name
if isinstance(obj, File):
malware_entry = malware(md5=obj.get_md5(),
crc32=obj.get_crc32(),
sha1=obj.get_sha1(),
sha256=obj.get_sha256(),
sha512=obj.get_sha512(),
file_size=obj.get_size(),
file_type=obj.get_type(),
ssdeep=obj.get_ssdeep(),
orig_url=orig_url,
file_name=file_name)
malware_entry.save()
if store_encoded:
encode_sample(file_path)
delete_file(file_path)
if tags:
tags = tags.strip()
if "," in tags:
tags = tags.split(",")
else:
tags = tags.split(" ")
for t in tags:
t = t.strip().lower()
if t == "":
continue
if tag.objects.filter(tag=t).exists():
malware_entry.tags.add(tag.objects.get(tag=t))
continue
malware_entry.tags.add(tag.objects.create(tag=t))
return HttpResponse(jsonize({"message" : file_name + " added to repository"}))
else:
return HttpResponse('METHOD not supported for URL')
def get_files():
""" Return list of all Files from Mongo."""
files = []
for result in collection.find():
files.append(
File(name=result['name'],
sha1=result['sha1'],
modified=result['modified'],
data=result['bin']))
return files
def get_file(name):
""" Get a single file from MySQL DB as a File object.
name (str): name of file to return
"""
c.execute('select * from files where name=%s', name)
results = c.fetchone()
return File(name=results[1],
sha1=results[2],
modified=results[3],
data=results[4])
def store_sample(data):
sha256 = File(file_data=data).get_sha256()
folder = os.path.join(Config().api.repository, sha256[0], sha256[1],
sha256[2], sha256[3])
if not os.path.exists(folder):
os.makedirs(folder, 0750)
file_path = os.path.join(folder, sha256)
if not os.path.exists(file_path):
sample = open(file_path, "wb")
sample.write(data)
sample.close()
return file_path
def store_sample(data):
sha256 = File(file_data=data).get_sha256()
folder = os.path.join(malware_root, sha256[0], sha256[1], sha256[2],
sha256[3])
if not os.path.exists(folder):
os.makedirs(folder, 0755)
file_path = os.path.join(folder, sha256)
if not os.path.exists(file_path):
sample = open(file_path, "wb")
sample.write(data)
sample.close()
return file_path
def uploadFile():
""" REST service to which files can be uploaded. """
try:
file_obj = request.files.get('file')
file_name = file_obj.filename
file_data = file_obj.stream.read()
hash_obj = sha1()
hash_obj.update(file_data)
file_sha1 = hash_obj.hexdigest()
file_modified = datetime.now()
db_file = File(name=file_name,
sha1=file_sha1,
modified=file_modified,
data=file_data
)
print("Received file to upload: %s" % file_name)
# Upload file to each data source
for mod in db_modules:
mod_name = mod.__name__[:mod.__name__.index('_')]
print("Checking '%s' file list..." % mod_name)
mod.connect()
files = mod.get_files()
if db_file.name not in [f.name for f in files]:
mod.insert_file(db_file)
print("Uploaded '%s' to '%s'" % (db_file.name, mod_name))
else:
ext_file = [f for f in files if f.name == db_file.name][0]
if (ext_file.sha1 != db_file.sha1
and ext_file.modified < db_file.modified):
mod.update_file(db_file)
print("Updated '%s' on '%s'" % (db_file.name, mod_name))
else:
print("'%s' skipped on '%s': newest version present already"
% (db_file.name, mod_name)
)
return jsonify({"Status": "Uploaded to all databases successfully."})
except:
print_exc()
return jsonify({"Status": "ERROR"})
def get_image_objects():
image_objects = []
fileutils.create_directory(offsite_images_input + 'processed/')
for file_name in fileutils.get_camera_image_names(offsite_images_input):
if file_name != 'processed' and file_name != 'Thumbs.db' and file_name.find(
'.lock') is -1:
gm_time = fileutils.get_file_create_time(offsite_images_input,
file_name)
image_objects.append(
File.File(
None, offsite_images_input, file_name,
fileutils.get_file_extension(offsite_images_input,
file_name),
fileutils.get_file_create_year(gm_time),
fileutils.get_file_create_month(gm_time),
fileutils.get_file_create_day(gm_time),
fileutils.get_file_create_hour(gm_time, 0),
fileutils.get_file_create_minute(gm_time),
fileutils.get_file_create_second(gm_time),
fileutils.get_file_mtime(offsite_images_input, file_name)))
return image_objects
from objects import File, Folder, Object, User
if __name__ == "__main__":
_file = File(name="park",
owner="user",
permission=755,
parent_path="/",
content="content")
def get_file_by_id(file_id):
""" Get File from Box.com based on Box file id."""
file_obj = client.file(file_id).get()
return File(file_obj.name, file_obj.sha1, file_obj.modified_at,
file_obj.content())