def check_file_typ(self, filename):
full_file_name = os.path.join(self.base_folder, self.upload_folder,
filename)
if os.name == 'nt':
#if its windows we need to specify magic file explicitly
#https://github.com/ahupp/python-magic#dependencies
magic = Magic(magic_file=current_app.config['MAGIC_FILE_WIN32'],
mime=True)
else:
magic = Magic(mime=True)
try:
file_type = magic.from_file(full_file_name)
except IOError:
app.logger.error(
"check_file_type is called with non existing file or I/O error while opening :%s"
% full_file_name)
return None
if file_type == 'image/gif':
return FILE_TYPE_IMAGE
elif file_type == 'image/png':
return FILE_TYPE_IMAGE
elif file_type == 'image/jpeg':
return FILE_TYPE_IMAGE
elif file_type == 'application/pdf':
return FILE_TYPE_PDF
else:
return FILE_TYPE_UNKNOWN
def get_arch(target_path):
default_magic = Magic()
magic_checks = [
(Magic(magic_file=CGC_MAGIC), CGC_REGEX, CGCProjectConfiguration,
'i386'),
(default_magic, ELF32_REGEX, LinuxProjectConfiguration, 'i386'),
(default_magic, ELF64_REGEX, LinuxProjectConfiguration, 'x86_64'),
(default_magic, DLL32_REGEX, WindowsDLLProjectConfiguration, 'i386'),
(default_magic, DLL64_REGEX, WindowsDLLProjectConfiguration, 'x86_64'),
(default_magic, PE32_REGEX, WindowsProjectConfiguration, 'i386'),
(default_magic, PE64_REGEX, WindowsProjectConfiguration, 'x86_64'),
(default_magic, MSDOS_REGEX, WindowsProjectConfiguration, 'i386')
]
# Check the target program against the valid file types
for magic_check, regex, proj_config_class, arch in magic_checks:
magic = magic_check.from_file(target_path)
matches = regex.match(magic)
# If we find a match, create that project. The user instructions
# are returned
if matches:
return arch, proj_config_class
return None, None
def test_filter_with_objects(self):
m1 = Magic()
m1.filter(__lt=5, __gt=2)
m2 = Magic()
m2.filter(__in=[3])
self.m.set_data([0, 1, 2, 3, 4, 5, 7])
self.m.filter(m1)._not(m2)
self.assertEqual(self.m.get_filter_data(), [4])
def get_videos(path, clear_files):
if os.path.isfile(path):
mime = Magic(mime=True).from_file(path)
if "video" in mime:
if mime == "video/x-matroska":
return [path]
else:
try:
print(
f"[{colored('+','green')}] Converting {path2title(path)} to MKV",
end="",
)
from audio_extract import convert2mkv
new_file = convert2mkv(path)
clear_files.append(new_file)
return [new_file]
except Exception as e:
print(e)
return []
return []
if os.path.isdir(path):
ans = []
for file in os.listdir(path):
ans.extend(get_videos(path + "/" + file, clear_files))
return ans
def _write(self, path, iname=None, data=None, replace=False):
if not (iname or data):
raise Exeption('Either iname or data need to be passed')
oname = join(self.root_dir, path)
if not replace and exists(oname):
raise Exception('file (%s) already exists, use replace' % path)
if not exists(dirname(oname)):
makedirs(dirname(oname))
temppath = join(self.root_dir, path + str(random()))
f = {
'@id': quote(path),
'urn': uuid4().urn,
'@type': 'Resource',
'path': path
}
h = sha256()
try:
if iname:
with open(iname, 'rb') as stream:
with open(temppath, 'wb') as out:
data, length = None, 0
while data != b'':
data = stream.read(100 * 1024)
out.write(data)
h.update(data)
size = out.tell()
else:
data = dumps(data) if isinstance(data, dict) or isinstance(
data, list) else data
data = data.encode('utf-8') if isinstance(data, str) else data
with open(temppath, 'wb') as out:
h.update(data)
out.write(data)
size = len(data)
except:
raise
else:
f['size'] = size
f['checksum'] = 'SHA256:' + h.hexdigest()
if path in self and self[path]['checksum'] == f['checksum']:
f = self[path]
else:
move(temppath, oname)
with Magic(flags=MAGIC_MIME) as m:
f['mime_type'] = m.id_filename(oname).split(';')[0]
finally:
if exists(temppath):
remove(temppath)
return f
def update_evidence(event_id: str, evidence_type: str, file) -> bool:
event = get_event(event_id)
success = False
if cast(List[Dict[str, str]], event.get('historic_state',
[]))[-1].get('state') == 'SOLVED':
raise EventAlreadyClosed()
project_name = str(event.get('project_name', ''))
try:
mime = Magic(mime=True).from_buffer(file.file.getvalue())
extension = {
'image/gif': '.gif',
'image/jpeg': '.jpg',
'image/png': '.png',
'application/pdf': '.pdf',
'application/zip': '.zip',
'text/csv': '.csv',
'text/plain': '.txt'
}[mime]
except AttributeError:
extension = ''
evidence_id = f'{project_name}-{event_id}-{evidence_type}{extension}'
full_name = f'{project_name}/{event_id}/{evidence_id}'
if event_dal.save_evidence(file, full_name):
success = event_dal.update(event_id, {evidence_type: evidence_id})
return success
def from_ldap(value, field, dn, ldap_field, instance=None):
"""Convert an LDAP value to the Python type of the target field.
This conversion is prone to error because LDAP deliberately breaks
standards to cope with ASN.1 limitations.
"""
from ldapdb.models.fields import datetime_from_ldap # noqa
# Pre-convert DateTimeField and DateField due to ISO 8601 limitations in RFC 4517
if isinstance(field, (fields.DateField, fields.DateTimeField)):
# Be opportunistic, but keep old value if conversion fails
value = datetime_from_ldap(value) or value
elif isinstance(field, FileField) and instance is not None:
content = File(io.BytesIO(value))
basename = ldap_field_to_filename(dn, ldap_field)
if ldap_field == "jpegphoto":
extension = "jpeg"
else:
extension = Magic(
extension=True).from_buffer(content).split("/")[0]
name = f"{basename}.{extension}"
# Pre-save field file instance
fieldfile = getattr(instance, field.attname)
fieldfile.save(name, content)
return fieldfile
# Finally, use field's conversion method as default
return field.to_python(value)
def post(self):
"""
Recieves file and options, checks file mimetype,
validates options and creates converting task
"""
if "file" not in request.files:
return abort(400, message="file field is required")
else:
with NamedTemporaryFile(delete=False, prefix=app.config["MEDIA_PATH"]) as tmp_file:
request.files["file"].save(tmp_file)
tmp_file.flush()
tmp_file.close()
remove_file.schedule(
datetime.timedelta(seconds=app.config["ORIGINAL_FILE_TTL"])
, tmp_file.name)
with Magic() as magic: # detect mimetype
mimetype = magic.from_file(tmp_file.name)
if mimetype not in app.config["SUPPORTED_MIMETYPES"]:
return abort(400, message="Not supported mimetype: '{0}'".format(mimetype))
options = request.form.get("options", None)
if options: # options validation
options = ujson.loads(options)
formats = options.get("formats", None)
if not isinstance(formats, list) or not formats:
return abort(400, message="Invalid 'formats' value")
else:
for fmt in formats:
supported = (fmt in app.config["SUPPORTED_MIMETYPES"][mimetype]["formats"])
if not supported:
message = "'{0}' mimetype can't be converted to '{1}'"
return abort(400, message=message.format(mimetype, fmt))
thumbnails = options.get("thumbnails", None)
if thumbnails:
if not isinstance(thumbnails, dict):
return abort(400, message="Invalid 'thumbnails' value")
else:
thumbnails_size = thumbnails.get("size", None)
if not isinstance(thumbnails_size, str) or not thumbnails_size:
return abort(400, message="Invalid 'size' value")
else:
try:
(width, height) = map(int, thumbnails_size.split("x"))
except ValueError:
return abort(400, message="Invalid 'size' value")
else:
options["thumbnails"]["size"] = (width, height)
else:
if mimetype == "application/pdf":
options = {
"formats": ["html"]
}
else:
options = app.config["DEFAULT_OPTIONS"]
task = process_document.queue(tmp_file.name, options, {
"mimetype": mimetype,
})
return {
"id": task.id,
"status": task.status,
}
def _get_mime_type(file_name):
"""
Return the mime type of a file.
The mime_type will be used to determine if a configuration file is text.
"""
return Magic(mime=True).from_file(file_name)
def identify_platform(self, filepath):
filemagic = Magic()
filetype = ""
try:
filetype = filemagic.id_filename(filepath)
except Exception as e:
# certain version of libmagic throws error while parsing file, the CPU information is however included in the error in somecases
filetype = str(e)
# filemagic.close()
if "ELF 32-bit" in filetype:
if "ARM" in filetype:
return "ELF", "arm"
if "80386" in filetype:
return "ELF", "x86"
if ("MIPS" in filetype) and ("MSB" in filetype):
return "ELF", "mips"
if "MIPS" in filetype:
return "ELF", "mipsel"
if "PowerPC" in filetype:
return "ELF", "powerpc"
if "ELF 64-bit" in filetype:
if "x86-64" in filetype:
return "ELF", "x86-64"
return filetype, self.default_cpu
def check_file_extension(filename, resource_path, extensions):
"""
Renvoyer le nom d'un fichier avec l'extension correspondant à son type MIME
:param extensions: extensions autorisées pour le renommage
:param filename: nom de fichier
:param resource_path: chemin du fichier de données à analyser
"""
if os.path.exists(resource_path) and filename:
# Réparer l'extension de fichier
extension_add = True
extensions = make_iterable(extensions)
for extension in extensions:
if filename.lower().endswith(extension.lower()):
extension_add = False
if extension_add:
# Trouver le type MIME
mime = Magic(mime=True, keep_going=True)
mimetype = mime.from_file(resource_path)
new_extension = guess_extension(mimetype)
if filename and new_extension:
filename = "{}{}".format(filename, new_extension)
if new_extension.lower() in extensions:
return filename
else:
return None
else:
return filename
return filename
async def highlight(context):
""" Generates syntax highlighted images. """
if context.fwd_from:
return
reply = await context.get_reply_message()
reply_id = None
await context.edit(lang('highlight_processing'))
if reply:
reply_id = reply.id
target_file_path = await context.client.download_media(
await context.get_reply_message())
if target_file_path is None:
message = reply.text
else:
if Magic(mime=True).from_file(target_file_path) != 'text/plain':
message = reply.text
else:
with open(target_file_path, 'r') as file:
message = file.read()
remove(target_file_path)
else:
if context.arguments:
message = context.arguments
else:
await context.edit(lang('highlight_no_file'))
return
lexer = guess_lexer(message)
formatter = img.JpgImageFormatter(style="colorful")
result = syntax_highlight(message, lexer, formatter, outfile=None)
await context.edit(lang('highlight_uploading'))
await context.client.send_file(context.chat_id, result, reply_to=reply_id)
await context.delete()
def get(self, notebook_name, note_name):
notebook_name = notebook_name.replace('+', ' ')
note_name = note_name.replace('+', ' ')
action = self.get_argument('a', 'view')
if action == 'delete':
self._delete(notebook_name, note_name, confirmed=False)
elif action == 'edit':
self._edit(notebook_name, note_name, confirmed=False)
else:
path = join(self.settings.repo, notebook_name, note_name)
dot_path = join(self.settings.repo, notebook_name, '.' + note_name)
highlight = self.get_argument('hl', None)
with Magic() as m:
mime = m.id_filename(path)
if 'text' in mime or 'empty' in mime:
self._view_plaintext(notebook_name=notebook_name,
note_name=note_name,
highlight=highlight)
elif exists(dot_path):
download = self.get_argument('dl', False)
if download:
self._view_file(notebook_name=notebook_name,
note_name=note_name)
else:
self._view_plaintext(notebook_name=notebook_name,
note_name=note_name,
highlight=highlight,
dot=True)
else:
self._view_file(notebook_name=notebook_name,
note_name=note_name)
def test_filter_with_object(self):
m1 = Magic()
m1.filter(__lt=5, __gt=2)._or(__eq=1)
self.m.set_data([0, 1, 2, 3, 5, 7])
self.m.filter(m1)
self.assertEqual(self.m.get_filter_data(), [1, 3])
def buildSync(fileName):
service = authorize(token_file='token.pickle')
fileMimetype = Magic(mime=True).from_file(fileName)
fileMetadata = {'name': fileName, 'mimeType': fileMimetype}
mediaBody = MediaFileUpload(filename=fileName,
mimetype=fileMimetype,
resumable=False)
return service, fileMetadata, mediaBody
def get_mime_type(resource_path):
"""
Renvoyer le type MIME d'un fichier local
:raises: IOError
"""
mime = Magic(mime=True)
mimetype = mime.from_buffer(open(resource_path, 'rb').read())
return mimetype
def __init__(self):
self._passwords = dict()
self._gui = None
self._QApp = None
self._sudo = None
self._homedir = None
self.MAX_MESSAGE_SIZE = 750 * 1024
self.mimeResolver = Magic(mime=True)
self._lock = Lock()
def do_magic(self):
'''Раз в определнное время стреляет магией, как главный герой.'''
if time.time() - self.magic_time > self.for_magic_time:
self.set_magic_time()
if self.is_go_right:
return Magic(*self.rect.topright,
True,
self.damage * 2,
self,
is_boss_magic=True,
group=self.groups())
else:
return Magic(*self.rect.topleft,
False,
self.damage * 2,
self,
is_boss_magic=True,
group=self.groups())
def _set_mimetype(self):
try:
from magic import Magic
except:
# TODO log that this failed
return
self.mimetype = Magic(mime=True).from_file(
self.get_absolute_filepath())
def get_file_mime_type(document):
try:
from magic import Magic
doc_abs_path = os.path.join(settings.MEDIA_ROOT, document.name)
mime_type = Magic(mime=True).from_file(doc_abs_path)
return mime_type
except:
return 'image/png'
def get_magic(xaf_file, magic_file=None):
global MAGIC_OBJECTS_CACHE
key = hashlib.md5(magic_file).hexdigest() \
if magic_file is not None else "system"
if key not in MAGIC_OBJECTS_CACHE:
MAGIC_OBJECTS_CACHE[key] = Magic(magic_file=magic_file)
magic = MAGIC_OBJECTS_CACHE[key]
tag_magic = magic.from_file(xaf_file.filepath)
return tag_magic
def assert_uploaded_file_mime(file_instance, allowed_mimes):
mime = Magic(mime=True)
if isinstance(file_instance, TemporaryUploadedFile):
mime_type = mime.from_file(file_instance.temporary_file_path())
elif isinstance(file_instance, InMemoryUploadedFile):
mime_type = mime.from_buffer(file_instance.file.getvalue())
else:
raise Exception('Provided file is not a valid django upload file. \
Use util.assert_file_mime instead.')
return mime_type in allowed_mimes
def do_magic(self):
'''Магическая атака. Возвращает экземпляр класса магия.'''
if self.cook_count > 1:
if self.skills[2] == '1':
if time.time() - self.magic_time > 0.7:
self.change_cook_count(-2)
self.set_magic_time()
if self.is_go_right:
return Magic(*self.rect.topright,
True,
self.damage * 2,
self,
group=self.groups())
else:
return Magic(*self.rect.topleft,
False,
self.damage * 2,
self,
group=self.groups())
def clean(self, data, initial=None):
rv = super(AudioField, self).clean(data, initial)
# make sure the blob is small enough to fit in the ResourceDatabase
# without raising SizeTooLargeError
max_blob_size = get_resource_database().max_blob_size
if data.size > max_blob_size:
raise forms.ValidationError(self.error_messages['file_too_large'] %
max_blob_size)
filename_requires_cleanup = False
oggfile_requires_cleanup = False
try:
if hasattr(data, 'temporary_file_path'):
filename = data.temporary_file_path()
else:
fd, filename = mkstemp()
filename_requires_cleanup = True
f = os.fdopen(fd, 'wb')
try:
for chunk in data.chunks():
f.write(chunk)
finally:
f.close()
from magic import Magic
mime_type = Magic(mime=True).from_file(filename)
try:
logger.debug("Mime type detected: %s", mime_type)
verify_file_type = verification_map[mime_type]
except KeyError:
raise forms.ValidationError(
self.error_messages['unrecognized_file_type'])
mime_type = verify_file_type(filename, self.error_messages)
rv.ductus_mime_type = mime_type
# convert Wav files to ogg, so we don't waste precious space
# this code will disappear as soon as we have a clean way to compress
# audio on the client
if mime_type == 'audio/wav':
oggfile_requires_cleanup = True
ogg_filename = convert_wav_to_ogg(filename)
rv.ductus_mime_type = 'audio/ogg'
rv.content_type = 'audio/ogg'
rv.file = open(ogg_filename)
return rv
finally:
if filename_requires_cleanup:
os.remove(filename)
if oggfile_requires_cleanup:
os.remove(ogg_filename)
def _determine_arch_and_proj(target_path):
"""
Check that the given target is supported by S2E.
The target's magic is checked to see if it is a supported file type (e.g.
ELF, PE, etc.). The architecture and operating system that the target was
compiled for (e.g., i386 Windows, x64 Linux, etc.) is also checked.
Returns:
A tuple containing the target's architecture, operating system and a
project class. A tuple containing three ``None``s is returned on
failure.
"""
default_magic = Magic()
magic_checks = (
(Magic(magic_file=CGC_MAGIC), CGC_REGEX, CGCProject, 'i386', 'decree'),
(default_magic, ELF32_REGEX, LinuxProject, 'i386', 'linux'),
(default_magic, ELF64_REGEX, LinuxProject, 'x86_64', 'linux'),
(default_magic, DLL32_REGEX, WindowsDLLProject, 'i386', 'windows'),
(default_magic, DLL64_REGEX, WindowsDLLProject, 'x86_64', 'windows'),
(default_magic, WIN32_DRIVER_REGEX, WindowsDriverProject, 'i386',
'windows'),
(default_magic, WIN64_DRIVER_REGEX, WindowsDriverProject, 'x86_64',
'windows'),
(default_magic, PE32_REGEX, WindowsProject, 'i386', 'windows'),
(default_magic, PE64_REGEX, WindowsProject, 'x86_64', 'windows'),
(default_magic, MSDOS_REGEX, WindowsProject, 'i386', 'windows'),
)
# Need to resolve symbolic links, otherwise magic will report the file type
# as being a symbolic link
target_path = os.path.realpath(target_path)
# Check the target program against the valid file types
for magic_check, regex, proj_class, arch, operating_sys in magic_checks:
magic = magic_check.from_file(target_path)
# If we find a match, create that project
if regex.match(magic):
return arch, operating_sys, proj_class
return None, None, None
def main(args):
include_str = args.i
print()
if os.path.isdir(args.f):
if args.f.endswith('\\') or args.f.endswith('/'):
smali_dir = args.f[:-1]
else:
smali_dir = args.f
dex_file = smali(smali_dir, os.path.basename(smali_dir) + '.dex')
dexsim(dex_file, smali_dir, include_str)
smali(smali_dir, os.path.basename(smali_dir) + '.sim.dex')
elif Magic(args.f).get_type() == 'apk':
apk_path = args.f
apk_sim_path = os.path.splitext(args.f)[0] + '.sim.apk'
shutil.copyfile(apk_path, apk_sim_path)
try:
pzip = powerzip.PowerZip(apk_path)
except zipfile.BadZipFile:
print(
"It seems the apk is corrupted. Please re-zip this apk, test again."
)
return
dexnames = []
for name in pzip.namelist():
if name.startswith('classes') and name.endswith('.dex'):
pzip.extract(name)
dexnames.append(name)
dex_file = name
smali_dir = baksmali(dex_file)
dexsim(dex_file, smali_dir, include_str)
smali(smali_dir, dex_file)
shutil.rmtree(smali_dir)
pzip.close()
pzip = powerzip.PowerZip(apk_sim_path)
for name in dexnames:
pzip.add(name, name)
os.remove(name)
pzip.save(apk_sim_path)
pzip.close()
else:
dex_file = os.path.basename(args.f)
temp_dir = tempfile.TemporaryDirectory()
smali_dir = baksmali(dex_file, temp_dir.name)
dexsim(dex_file, smali_dir, include_str)
smali(smali_dir,
os.path.splitext(os.path.basename(dex_file))[0] + '.sim.dex')
def save_bits_to_file(filepath, bits):
# get file extension
bitstring = Bits(bin=bits)
mime = Magic(mime=True)
mime_type = mime.from_buffer(bitstring.tobytes())
with open(f"{filepath}/file{mimetypes.guess_extension(type=mime_type)}",
"wb") as f:
bitstring.tofile(f)
def handle(self, *args, **options):
# Need an absolute path for the target in order to simplify
# symlink creation.
target_path = options['target'][0]
target_path = os.path.realpath(target_path)
# Check that the target actually exists
if not os.path.isfile(target_path):
raise CommandError('Target %s does not exist' % target_path)
default_magic = Magic()
magic_checks = [
(Magic(magic_file=CGC_MAGIC), CGC_REGEX, CGCProjectConfiguration,
'i386'),
(default_magic, ELF32_REGEX, LinuxProjectConfiguration, 'i386'),
(default_magic, ELF64_REGEX, LinuxProjectConfiguration, 'x86_64'),
(default_magic, DLL32_REGEX, WindowsDLLProjectConfiguration,
'i386'),
(default_magic, DLL64_REGEX, WindowsDLLProjectConfiguration,
'x86_64'),
(default_magic, PE32_REGEX, WindowsProjectConfiguration, 'i386'),
(default_magic, PE64_REGEX, WindowsProjectConfiguration, 'x86_64'),
(default_magic, MSDOS_REGEX, WindowsProjectConfiguration, 'i386')
]
# Check the target program against the valid file types
for magic_check, regex, proj_config_class, arch in magic_checks:
magic = magic_check.from_file(target_path)
matches = regex.match(magic)
# If we find a match, create that project. The user instructions
# are returned
if matches:
options['target'] = target_path
options['target_arch'] = arch
return call_command(Project(proj_config_class), **options)
# Otherwise no valid file type was found
raise CommandError('%s is not a valid target for S2E analysis' %
target_path)
def retrieve_image(request, img_file):
if util.assert_file_mime(img_file, ["image/png", "image/jpeg",
"image/gif"]):
with open(img_file, "rb") as file_obj:
mime = Magic(mime=True)
mime_type = mime.from_file(img_file)
return HttpResponse(file_obj.read(), content_type=mime_type)
else:
rollbar.report_message('Error: Invalid evidence image format',
'error', request)
return HttpResponse("Error: Invalid evidence image format",
content_type="text/html")
def __init__(self, path):
self.path = path
if '/' in path:
self.filename = path.rsplit('/', 1)[1]
else:
self.filename = path
self.extension = path.rsplit('.', 1)[1]
self.mime_type = Magic(mime=True).from_file(path)
if not self.is_good_name():
self.new_name = "{}.{}".format(randint(0, 10**9), self.extension)
else:
self.new_name = self.filename