class TestMagic(unittest.TestCase):
mime = False
def setUp(self):
self.m = Magic(mime=self.mime)
def testFileTypes(self):
for filename, desc, mime in testfile:
filename = path.join(path.dirname(__file__),
"testdata",
filename)
if self.mime:
target = mime
else:
target = desc
self.assertEqual(target, self.m.from_buffer(open(filename).read(1024)))
self.assertEqual(target, self.m.from_file(filename), filename)
def testErrors(self):
self.assertRaises(IOError, self.m.from_file, "nonexistent")
self.assertRaises(MagicException, Magic, magic_file="noneexistent")
os.environ['MAGIC'] = '/nonexistetn'
self.assertRaises(MagicException, Magic)
del os.environ['MAGIC']
class Unpack:
step = "unpack"
def __init__(self):
self.magic = Magic(mime=True, uncompress=True)
self.magic_formatted = Magic(mime=False, uncompress=True)
def __call__(self, source, dest):
pipeline_logger(f"{Unpack.step}::call", source=source, dest=dest)
source = Path(source)
dest = Path(dest)
mime = self.magic.from_file(str(source))
pipeline_logger(f"{Unpack.step}::detect_mime",
source=source,
mime=mime)
if mime == 'application/x-tar':
dest.mkdir(parents=True, exist_ok=True)
with tarfile.open(source, "r:*") as tar:
tar.extractall(dest)
elif mime == 'text/x-tex':
dest.mkdir(parents=True, exist_ok=True)
with gzip.open(source, "rb") as src, open(dest / "main.tex",
"wb") as dst:
copyfileobj(src, dst)
elif mime == 'application/pdf':
raise UnpackError(
f"No LaTeX source code available for this paper, PDF only")
elif mime == 'text/plain' and 'withdrawn' in self.magic_formatted.from_file(
str(source)):
raise UnpackError(f"The paper has been withdrawn and there is"
f" no LaTeX source code available")
else:
raise UnpackError(f"Cannot unpack file of type {mime}")
def get_info_about(self):
"""
return (encoding, info) tuple
info is a plain string and has to be parsed
"""
db = csv2rdf.database.DatabasePlainFiles(csv2rdf.config.config.resources_path)
filename = db.get_path_to_file(self.filename)
mgc_encoding = Magic(mime=False, mime_encoding=True)
mgc_string = Magic(mime=False, mime_encoding=False)
encoding = mgc_encoding.from_file(filename)
info = mgc_string.from_file(filename)
return (encoding, info)
def get_player(self, id=None):
if id:
id_attr = u" id=\"%s\"" % id
else:
id_attr = u" id=\"%s\"" % self.id
if self.file:
output = u"<a%s href='%s'>%s</a>" \
% (id_attr, self.file.url, self.file.name)
get_mimetype = Magic(mime=True)
mimetype = get_mimetype.from_file(self.file.path)
if mimetype:
main_type = mimetype.split("/")[0]
if main_type == "audio":
output = u"<audio%s src='%s' preload='none' type='%s'></audio>" \
% (id_attr, self.file.url, mimetype)
elif main_type == "video":
output = u"<video%s src='%s' preload='none' type='%s'></video>" \
% (id_attr, self.file.url, mimetype)
elif main_type == "image":
output = u"<img%s src='%s'/>" \
% (id_attr, self.file.url, self.file.name)
elif self.url:
output = u"<a%s href='%s'>%s</a>" \
% (id_attr, self.url, self.title)
else:
output = u"%s" % self.title
return output
def check_file_extension(filename, resource_path, extensions):
"""
Renvoyer le nom d'un fichier avec l'extension correspondant à son type MIME
:param extensions: extensions autorisées pour le renommage
:param filename: nom de fichier
:param resource_path: chemin du fichier de données à analyser
"""
if os.path.exists(resource_path) and filename:
# Réparer l'extension de fichier
extension_add = True
extensions = make_iterable(extensions)
for extension in extensions:
if filename.lower().endswith(extension.lower()):
extension_add = False
if extension_add:
# Trouver le type MIME
mime = Magic(mime=True, keep_going=True)
mimetype = mime.from_file(resource_path)
new_extension = guess_extension(mimetype)
if filename and new_extension:
filename = "{}{}".format(filename, new_extension)
if new_extension.lower() in extensions:
return filename
else:
return None
else:
return filename
return filename
def check_file_typ(self, filename):
full_file_name = os.path.join(self.base_folder, self.upload_folder,
filename)
if os.name == 'nt':
#if its windows we need to specify magic file explicitly
#https://github.com/ahupp/python-magic#dependencies
magic = Magic(magic_file=current_app.config['MAGIC_FILE_WIN32'],
mime=True)
else:
magic = Magic(mime=True)
try:
file_type = magic.from_file(full_file_name)
except IOError:
app.logger.error(
"check_file_type is called with non existing file or I/O error while opening :%s"
% full_file_name)
return None
if file_type == 'image/gif':
return FILE_TYPE_IMAGE
elif file_type == 'image/png':
return FILE_TYPE_IMAGE
elif file_type == 'image/jpeg':
return FILE_TYPE_IMAGE
elif file_type == 'application/pdf':
return FILE_TYPE_PDF
else:
return FILE_TYPE_UNKNOWN
class TestMagicMimeEncoding(unittest.TestCase):
def setUp(self):
self.m = Magic(mime_encoding=True)
def testFileEncoding(self):
for filename, encoding in testFileEncoding:
filename = path.join(path.dirname(__file__), "testdata", filename)
self.assertEqual(encoding, self.m.from_buffer(open(filename).read(1024)))
self.assertEqual(encoding, self.m.from_file(filename), filename)
def assert_uploaded_file_mime(file_instance, allowed_mimes):
mime = Magic(mime=True)
if isinstance(file_instance, TemporaryUploadedFile):
mime_type = mime.from_file(file_instance.temporary_file_path())
elif isinstance(file_instance, InMemoryUploadedFile):
mime_type = mime.from_buffer(file_instance.file.getvalue())
else:
raise Exception('Provided file is not a valid django upload file. \
Use util.assert_file_mime instead.')
return mime_type in allowed_mimes
def returnType(filePath):
if not filePath:
filePath = self.sourcefile
m = Magic()
filetype = m.from_file(filePath)
for mediatype in SUPPORTED_TYPES.keys():
for filestring in SUPPORTED_TYPES[mediatype]:
if filestring in filetype:
return mediatype
return None
def __get_magic_info(self):
"""
This function will get magic information of given file path.
:return: A tuple will be returned to it's caller. It's first argument will be boolean to tell either function
was successfully executed or not while second will be dictionary with will be holding error / mime string.
"""
try:
magic_object = Magic(mime=True)
file_mime = magic_object.from_file(self.__file_path__, )
return True, {'mime': file_mime}
except FileNotFoundError:
return False, {'error': ' No such file or directory.'}
def get(self, resource):
if ("list.html" == resource):
headers = {'Content-Type': 'text/html'}
files = listdir(Path(FileStorageCtxt.storage_path))
return make_response(render_template('files/list.html', title="List", files=files), 200, headers)
else:
with open(Path(FileStorageCtxt.storage_path + "/" + resource), 'rb') as bites:
mime = Magic(mime=True)
mime_type = mime.from_file(FileStorageCtxt.storage_path + "/" + resource)
return send_file(BytesIO(bites.read()), attachment_filename=resource, mimetype=mime_type)
def retrieve_image(request, img_file):
if util.assert_file_mime(img_file, ["image/png", "image/jpeg",
"image/gif"]):
with open(img_file, "rb") as file_obj:
mime = Magic(mime=True)
mime_type = mime.from_file(img_file)
return HttpResponse(file_obj.read(), content_type=mime_type)
else:
rollbar.report_message('Error: Invalid evidence image format',
'error', request)
return HttpResponse("Error: Invalid evidence image format",
content_type="text/html")
def track_file(track_id: int, download=False):
import pmv
track = db.get_track_by_id(track_id)
file_path = pmv.settings['music_library'] + unquote(track.download_url)
mime = Magic(mime=True)
mimetype = mime.from_file(file_path)
return send_file(file_path,
mimetype=mimetype,
as_attachment=download,
attachment_filename='%s.%s' % (track.name, track.format))
def classify_file(path, verbose=False):
assert type(path) in [str, unicode]
supported = ['MPEG ADTS', 'FLAC', 'MPEG Layer 3', 'Audio', '^data$']
ignored = ['ASCII', 'JPEG', 'PNG', 'text', '^data$', 'AppleDouble']
magic = Magic()
try:
if type(path) == unicode:
m = magic.from_file(path.encode('utf-8'))
else:
m = magic.from_file(path)
except Exception as e:
print 'INTERNAL ERROR: %s: %s' % (path, str(e))
return (None, None)
if verbose:
print('Magic(%s):\n%s' % (path, m))
for s in supported:
match = re.search(s, m)
if match:
try:
audio = mutagen.File(path, easy=True)
if type(audio) == mutagen.mp3.EasyMP3:
return ('mp3', audio)
elif type(audio) == mutagen.flac.FLAC:
return ('flac', audio)
else:
return ('file', None)
return (format, audio)
except AttributeError, e:
print('Unknown file type: %s' % path)
break
except mutagen.mp3.HeaderNotFoundError, e:
print('Header not found: %s' % path)
break
except Exception, e:
print('INTERNAL ERROR: get_children()')
traceback.print_exc()
return (None, None)
def file_getmime(filepath):
'''获取文件mime类型
:param filepath: 提供文件路径
:return: mime类型
'''
mimetypes.init('/usr/local/openresty/nginx/conf/mime.types')
mime = mimetypes.MimeTypes()
mimetype = mime.guess_type(filepath)
if mimetype[0] == "None":
from magic import Magic
f = Magic(mime=True)
return f.from_file(filepath)
return mimetype[0]
class FileType:
def __init__(self):
self.m = Magic()
def get_type(self, fname):
ftype = self.m.from_file(fname)
for k in TYPE_MAPPING.keys():
if k in ftype:
return TYPE_MAPPING[k]
#solutions here from http://stackoverflow.com/questions/9084228/python-to-check-if-a-gzipped-file-is-xml-or-csv
#and http://stackoverflow.com/questions/2984888/check-if-file-has-a-csv-format-with-python
if 'text' in ftype:
with open(fname, 'rb') as fh:
try:
xml.sax.parse(fh, xml.sax.ContentHandler())
return 'xml'
except: # SAX' exceptions are not public
pass
fh.seek(0)
try:
dialect = csv.Sniffer().sniff(fh.read(1024))
return 'csv'
except csv.Error:
pass
return 'txt'
def is_compression(self, fname):
ftype = self.get_type(fname)
return self.is_compression_by_type(ftype)
def is_compression_by_type(self, ftype):
if ftype in COMPRESSION:
return True
return False
def is_archived(self, fname):
ftype = self.get_type(fname)
return self.is_archived_by_type(ftype)
def is_archived_by_type(self, ftype):
if ftype in ARCHIVED:
return True
return False
def transformaArquivosJSON(arquivos: list):
from json import dumps as JSON_Encode
from magic import Magic
mime = Magic(mime=True)
json = []
for arquivo in arquivos:
json.append({
"nome": arquivo.split("/")[-1].split(".")[0],
"mime": mime.from_file(arquivo),
"path": arquivo
})
return JSON_Encode(json)
class FileType:
def __init__(self):
self.m = Magic()
def get_type(self, fname):
ftype = self.m.from_file(fname)
for k in TYPE_MAPPING.keys():
if k in ftype:
return TYPE_MAPPING[k]
#solutions here from http://stackoverflow.com/questions/9084228/python-to-check-if-a-gzipped-file-is-xml-or-csv
#and http://stackoverflow.com/questions/2984888/check-if-file-has-a-csv-format-with-python
if 'text' in ftype:
with open(fname, 'rb') as fh:
try:
xml.sax.parse(fh, xml.sax.ContentHandler())
return 'xml'
except: # SAX' exceptions are not public
pass
fh.seek(0)
try:
dialect = csv.Sniffer().sniff(fh.read(1024))
return 'csv'
except csv.Error:
pass
return 'txt'
def is_compression(self, fname):
ftype = self.get_type(fname)
return self.is_compression_by_type(ftype)
def is_compression_by_type(self, ftype):
if ftype in COMPRESSION:
return True
return False
def is_archived(self, fname):
ftype = self.get_type(fname)
return self.is_archived_by_type(ftype)
def is_archived_by_type(self, ftype):
if ftype in ARCHIVED:
return True
return False
def _get_type(self, file_path=None):
"""Returns the file_type of the classes source media file or the type
of the provided file."""
if not file_path:
file_path = self.sourcefile
magic = Magic()
file_type = magic.from_file(file_path)
for media_type in SUPPORTED_TYPES.keys():
for file_string in SUPPORTED_TYPES[media_type]:
if file_string in file_type:
return media_type
raise AudioSlaveException("File type '%s' not supported!" % file_type)
def isAudio(file):
isAudio = False
try:
mime = Magic(mime=True)
mimetype = mime.from_file(file)
typeOfFile = mimetype.split("/")
if typeOfFile[0] == "audio":
isAudio = True
else:
print inputFile, "is not a valid file"
except IOError:
print inputFile, "is not a valid file or it doesn't exist."
return isAudio
def view_bill(request):
session = Session()
try:
request_body = dict(request.GET)
if not 'token' in request_body:
return Helper.construct_response(401, 'Unauthorized')
token = request_body['token']
param = request.matchdict['hash']
delete_expired_tokens(session)
user = session.query(Users, UserTokens).join(
UserTokens, UserTokens.user_id == Users.id).filter(
UserTokens.token == token).first()
if not user:
return Helper.construct_response(401, 'Unauthorized')
hashObj = session.query(UserTransaction).filter(
UserTransaction.hash == param).first()
if user.Users.role == 0 and not hashObj.user_id == user.Users.id:
return Helper.construct_response(401, 'Unauthorized')
if not hashObj:
return Helper.construct_response(404, 'Not found')
mime = Magic(mime=True)
mime_type = mime.from_file(
os.path.join(os.getcwd(), "kore_task/files/" + param))
session.close()
return FileResponse(os.path.join(os.getcwd(),
"kore_task/files/" + param),
content_type=mime_type)
except DBAPIError as dbErr:
session.close()
'''
SQL error ..................................................................
'''
print("MYSQL Error --------------------------------" + str(dbErr.orig))
# print("MYSQL Error-------------------------" + code,message)
return Helper.construct_response(500, 'Internal Error', '')
except Exception as e:
session.close()
print(e)
return Helper.construct_response()
def generate_binary_response(httpStatusCode, filename):
mime = Magic(mime=True)
mimetype = mime.from_file(filename)
info(f'generating response of {httpStatusCode} for {filename} of type {mimetype}')
with open(filename, 'rb') as file:
enc = b64encode(file.read())
encoded = enc.decode('utf-8')
return {
'isBase64Encoded': True,
'statusCode': httpStatusCode,
'headers': {'content-type': mimetype},
'body': encoded,
}
def post(self):
"""
Resize video frames
"""
# Validate request body with schema model
try:
image_file = request.files['image']
imgae_path = fr'{settings.IMAGE_PATH}\{image_file.filename}'
image_file.save(imgae_path)
mime = Magic(mime=True)
filename = mime.from_file(imgae_path)
if 'image' not in filename:
os.remove(imgae_path)
return ErrorModel(**{'message': 'Not A Valid Image File'}), 400
result = insert_task_to_mq(imgae_path)
except ValueError as e:
return ErrorModel(**{'message': e.args[0]}), 400
return result, 200
def get_image(self, image_name: str, max_sz: int = 0):
path_image = f"{self.path}/images/{image_name}"
# Downsample before sending if max_sz
if max_sz > 0:
im = Image.open(path_image)
im.thumbnail((max_sz, max_sz))
im_bytes = BytesIO()
im.save(im_bytes, format="JPEG")
im_bytes.seek(0)
return send_file(im_bytes, mimetype="image/jpeg")
# Just send the file
else:
magic = Magic(mime=True)
mimetype = magic.from_file(path_image)
return send_file(path_image, mimetype=mimetype)
def detecttype(filepath):
"""Detect the mime type of the text file."""
try:
from magic import Magic
mime = Magic(mime=True)
type = mime.from_file(filepath)
root, ext = os.path.splitext(filepath)
if ext in '.mo':
return "mo"
elif "text/" in type:
return "text"
else:
return type
except (ImportError, TypeError):
root, ext = os.path.splitext(filepath)
if ext in '.mo':
return "mo"
elif ext in listofexts:
return "text"
else:
return "unknown"
def _move_moov_atoms(self, in_file):
'''Server needs the H264 file to have its moov atoms (or boxes) in front
of the data. The qt-faststart.c program - shipped with FFMpeg - rewrites
H264 files so their atoms are placed in the required order. This is only
necessary to do if it's a quicktime file.
$ qt-faststart <infile.mov> <outfile.mov>
'''
moovs = (
'video/x-ms-asf',
'video/quicktime',
'application/octet-stream',
)
magic = Magic(mime=True)
if magic.from_file(self.video.file.path) in moovs:
(name, ext) = os.path.splitext(in_file)
tmp = ''.join([name, '_tmp', ext])
shutil.copyfile(in_file, tmp)
ret = subprocess.call(["qt-faststart", tmp, in_file])
if ret != 0:
raise OSError()
def file(config, override_meta, upload_file):
print("file")
common_meta = dict(config['default'])
bashed_meta = {
k: subprocess.getoutput('echo \'echo "%s"\' | bash' % v)
for k, v in common_meta.items()
}
bashed_meta.update(override_meta)
magic_mimetyper = Magic()
mimetype = magic_mimetyper.from_file(upload_file)
meta = bashed_meta
meta_type = 'file'
for autodetect_group in config['autodetect']:
autodetect_patterns = json.loads(
config['autodetect'][autodetect_group])
for autodetect_pattern in autodetect_patterns:
print(autodetect_pattern, mimetype)
if mimetype == autodetect_pattern:
meta_type = autodetect_group
meta.update({'meta_type': 'quicksave/' + meta_type})
if not GLOBAL.dry:
response = API.create(meta)
meta_hash = response['item']['meta']['meta_hash']
else:
print('API.create(%s)' % meta)
meta_hash = ''
with open(upload_file, 'rb') as bfile:
filebase = base64.b64encode(bfile.read()).decode('ascii')
filename = os.path.basename(upload_file)
if not GLOBAL.dry:
API.upload(response['item']['meta']['meta_hash'], mimetype,
filename, filebase)
else:
print('API.upload(__meta_hash__, %s, %s, __base64__ [%sB])' %
(mimetype, filename, len(filebase)))
return meta_hash
def mutate(self, info, **kwargs):
"""Process file input."""
finding_id = kwargs.get('finding_id')
project = integrates_dao.get_finding_project(finding_id)
file_input = info.context.FILES.get('document', None)
mime = Magic(mime=True)
if isinstance(file_input, TemporaryUploadedFile):
mime_type = mime.from_file(file_input.temporary_file_path())
elif isinstance(file_input, InMemoryUploadedFile):
mime_type = mime.from_buffer(file_input.file.getvalue())
else:
mime_type = ''
mib = 1048576
if (file_input
and mime_type in ['text/x-yaml', 'text/plain', 'text/html']):
if file_input.size < 1 * mib:
try:
success = process_file(file_input, finding_id, info)
except (InvalidRange, InvalidSchema, InvalidPort):
raise
else:
success = False
raise InvalidFileSize()
else:
success = False
raise InvalidFileType()
ret = UploadFile(success=success)
if success:
update_last_vuln_date(finding_id)
util.cloudwatch_log(
info.context, 'Security: Uploaded file in {project} \
project succesfully'.format(project=project))
else:
util.cloudwatch_log(
info.context, 'Security: Attempted to delete file \
from {project} project'.format(project=project))
util.invalidate_cache(finding_id)
util.invalidate_cache(project)
return ret
def images_missing_tags():
"""Requests tags for images without tags
Returns:
template: template to get more tags
"""
folder = session.get('folder', None)
file_names = list()
images_missing_tags = list()
mime = Magic(mime=True)
if not folder:
return redirect(url_for('index'))
try:
file_names = request.args.getlist('images_missing_tags')
except KeyError:
pass
# Read image data
for file_name in file_names:
file_path = os.path.join(folder, SandiWorkflow.IMAGES_FOLDER,
file_name)
with open(file_path, 'r') as image:
images_missing_tags.append({
'file_name':
file_name,
'data':
base64.b64encode(image.read()).decode('ascii'),
'type':
mime.from_file(file_path)
})
app.logger.debug('images_missing_tags')
return render_template('images_missing_tags.html',
images_missing_tags=images_missing_tags)
def check_file_typ(self,filename):
full_file_name = os.path.join(self.base_folder,self.upload_folder, filename)
if os.name == 'nt':
#if its windows we need to specify magic file explicitly
#https://github.com/ahupp/python-magic#dependencies
magic = Magic(magic_file=current_app.config['MAGIC_FILE_WIN32'],mime=True)
else:
magic = Magic(mime=True)
try:
file_type = magic.from_file(full_file_name)
except IOError:
app.logger.error("check_file_type is called with non existing file or I/O error while opening :%s"%full_file_name)
return None
if file_type == 'image/gif':
return FILE_TYPE_IMAGE
elif file_type == 'image/png':
return FILE_TYPE_IMAGE
elif file_type == 'image/jpeg':
return FILE_TYPE_IMAGE
elif file_type == 'application/pdf':
return FILE_TYPE_PDF
else:
return FILE_TYPE_UNKNOWN
def dump_to_mongofs(uuid, box):
ret = False
mime = Magic(mime=True)
files = listdir(path.join(box["temp"], uuid))
try:
for file in files:
time_now = datetime.now()
with open(path.join(box["temp"], uuid, file), "rb") as data_file:
item = save_item_fs(
mongo_settings_docker["files"], data_file.read(), file,
uuid, mime.from_file(path.join(box["temp"], uuid,
file)), time_now)
update_task_files(mongo_settings_docker["worker_db"],
mongo_settings_docker["worker_col_logs"],
uuid, {
"name": file,
"objectid": str(item)
})
ret = True
except Exception as e:
ret = False
log_string(uuid, "dump_to_mongofs Failed {} {}".format(e), "Red")
return ret
def mutate(self, info, **kwargs):
"""Process file input."""
finding_id = kwargs.get('finding_id')
origin = kwargs.get('origin', '')
project = finding_domain.get_project(finding_id)
file_input = info.context.FILES['1']
mime = Magic(mime=True)
if isinstance(file_input, TemporaryUploadedFile):
mime_type = mime.from_file(file_input.temporary_file_path())
elif isinstance(file_input, InMemoryUploadedFile):
mime_type = mime.from_buffer(file_input.file.getvalue())
else:
mime_type = ''
mib = 1048576
if (file_input
and mime_type in ['text/x-yaml', 'text/plain', 'text/html']):
if file_input.size < 1 * mib:
success = process_file(file_input, finding_id, info, origin)
else:
raise InvalidFileSize()
else:
raise InvalidFileType()
ret = UploadFile(success=success)
if success:
update_last_vuln_date(finding_id)
util.invalidate_cache(finding_id)
util.invalidate_cache(project)
util.cloudwatch_log(
info.context, 'Security: Uploaded file in {project} \
project succesfully'.format(project=project))
else:
util.cloudwatch_log(
info.context, 'Security: Attempted to delete file \
from {project} project'.format(project=project))
raise ErrorUploadingFileS3()
return ret
def send_file_to_s3(file_name, evidence, event_id):
"""Save evidence files in s2."""
evidence_id = evidence['id']
fileroute = '/tmp/:id.tmp'.replace(':id', evidence_id)
evidence_type = evidence['file_type']
is_file_saved = False
with open(fileroute, 'r') as file_obj:
try:
mime = Magic(mime=True)
mime_type = mime.from_file(fileroute)
if evidence_type.get(mime_type):
file_name_s3 = file_name + evidence_type.get(mime_type)
CLIENT_S3.upload_fileobj(file_obj, BUCKET_S3, file_name_s3)
is_file_saved = True
else:
util.cloudwatch_log_plain(
'File of event {event_id} does not have the right type'
.format(event_id=event_id)
)
except ClientError:
rollbar.report_exc_info()
is_file_saved = False
os.unlink(fileroute)
return is_file_saved
def area(config, override_meta):
print("area")
cmd, cmd_output, meta = get_config_for_option(config, override_meta,
'area')
print(cmd)
print(subprocess.check_output(cmd.split(' ')))
meta.update({'meta_type': 'quicksave/screenshot'})
if not GLOBAL.dry:
response = API.create(meta)
meta_hash = response['item']['meta']['meta_hash']
else:
print('API.create(%s)' % meta)
meta_hash = ''
magic_mimetyper = Magic()
mimetype = magic_mimetyper.from_file(cmd_output)
with open(cmd_output, 'rb') as file:
filebase = base64.b64encode(file.read()).decode('ascii')
filename = 'screenshot.png'
if not GLOBAL.dry:
API.upload(meta_hash, mimetype, filename, filebase)
else:
print('API.upload(__meta_hash__, %s, %s, __base64__)' %
(mimetype, filename))
return meta_hash
# Make sure a file is present
if len(argv) < 2:
print('Please provide an image file')
exit(1)
input_file = argv[1]
extension = '.' + input_file.split('.')[-1]
if not isfile(input_file):
print('File not found: %s' % input_file)
exit(1)
# Make sure the file is an image
mime = Magic(mime=True)
mediatype = mime.from_file(input_file)
media_type = mediatype.split('/')[0]
media_subtype = mediatype.split('/')[-1]
if media_type != 'image':
print('Not an image: %s' % input_file)
exit(1)
if media_subtype != 'jpeg' and media_subtype != 'png':
print('[!] Warning! This tool is currently only tested on jpeg and png images. The type of the current image (%s) is not tested' % media_subtype)
# Do the stripping
image = Image.open(input_file)
image_stripped = Image.new(image.mode, image.size)
image_stripped.putdata(list(image.getdata()))
def get_file_mime_type(file_name):
mime = Magic(mime=True)
return mime.from_file(file_name)
parts = line.split("\t")
type, ext = parts[0], parts[1:]
type_maps[type] = ext
except:
print("Cannot read " + magic_file)
exit(0)
print("{:35s} \t {:10s} \t {:20s}".format('Camouflaged file', 'Claimed ext', 'Expected ext'))
# --------------------------------------------------------------------------------------------- #
# For each file:
# + Get claimed ext
# + Using magic and mime.types to find out expected exts
# --------------------------------------------------------------------------------------------- #
mg = Magic(mime=True, mime_encoding=False, keep_going=True, uncompress=False)
for folder, subs, files in os.walk(directory):
for filename in files:
try:
full_path = folder + "//" + filename
detected_type = str(mg.from_file(full_path).decode("utf-8"))
ext = os.path.splitext(filename)[1][1:]
expected_exts = type_maps[detected_type]
if (ext != "" and ext not in expected_exts):
# Print out highly likely camouflaged files
print_result(full_path, ext, "".join([b + " " for b in expected_exts]))
except:
pass
def get_mime(filename):
f = Magic(mime=True)
return f.from_file(filename)
def from_file(self, filename, mime=True):
return _Magic.from_file(self, filename)
def assert_file_mime(filename: str, allowed_mimes: List[str]) -> bool:
mime = Magic(mime=True)
mime_type = mime.from_file(filename)
return mime_type in allowed_mimes
def take_action(self, parsed_args):
self.log.debug(parsed_args)
self.log.debug("Logged in -- " + self.app.user.ssotoken)
headers = {
"Authorization": self.app.user.ssotoken,
"Content-Type": "application/json"
}
bucket = None
try:
bucket = parsed_args.bucket
except:
bucket = self.app.configoptions["bucket"]
try:
# Build URL to get pre-signed POST
post_url = furl(self.app.configoptions["fileserviceurl"])
post_url.path.segments.extend(
['filemaster', 'api', 'file', parsed_args.fileID, 'post', ''])
# Make request
post_r = requests.get(post_url.url,
headers=headers,
params={"bucket": bucket})
if post_r.ok:
# Parse elements of pre-signed POST from Fileservice
location_id = post_r.json()["locationid"]
post = post_r.json()["post"]
upload_url = post["url"]
fields = post["fields"]
# Get MIME type
magic = Magic(mime=True)
mime_type = magic.from_file(parsed_args.localFile)
# Add file
fields['file'] = (os.path.basename(parsed_args.localFile),
open(parsed_args.localFile, 'rb'), mime_type)
encoder = MultipartEncoder(fields=fields)
monitor = MultipartEncoderMonitor(encoder, upload_callback)
# Add filename
setattr(monitor, 'filename',
os.path.basename(parsed_args.localFile))
setattr(monitor, 'completed', False)
# Make the request
upload_r = requests.post(
upload_url,
data=monitor,
headers={'Content-Type': monitor.content_type})
if upload_r.ok:
# Build URL to get complete file
complete_url = furl(
self.app.configoptions["fileserviceurl"])
complete_url.path.segments.extend([
'filemaster', 'api', 'file', parsed_args.fileID,
'uploadcomplete', ''
])
# Make the request
complete_r = requests.get(complete_url.url,
headers=headers,
params={"location": location_id})
if not complete_r.ok:
sys.stderr.write("Completion error: %s" %
complete_r.content)
# Print upload details
self.app.stdout.write("\n%s,%s,%s\n" %
(parsed_args.fileID, upload_url,
complete_r.json()["filename"]))
else:
sys.stderr.write("Upload POST error: %s" %
upload_r.content)
else:
sys.stderr.write("Upload error: %s" % post_r.content)
except Exception as e:
sys.stderr.write("Pre-signed POST error: %s" % e)
class StaticFileMapping(Observable):
def __init__(self, mapping, reactor=None, **kwargs):
Observable.__init__(self, **kwargs)
self._mapping = {}
self._watchers = {}
self._reactor = reactor
self._magic = Magic(mime=True)
for shortname, filename in mapping.items():
if not isfile(filename):
print(f"file {filename} for {shortname} does not exist.")
del self._mapping[shortname]
else:
if not self._reactor is None:
basedir = dirname(filename)
if not basedir in self._watchers:
self._watchers[basedir] = DirectoryWatcher(
basedir,
self._update,
ModifyFile=True,
MoveInFile=True)
self._updateFile(shortname, filename)
if not self._reactor is None:
for each in self._watchers.values():
self._reactor.addReader(each, each)
def _update(self, event, *args, **kwargs):
for shortname, values in self._mapping.items():
filename = values['filename']
if filename == event.pathname:
self._updateFile(shortname, filename)
break
def _updateFile(self, shortname, filename):
with open(filename, 'rb') as fp:
etag = md5(fp.read()).hexdigest()
self._mapping[shortname] = dict(
filename=filename,
etag=etag,
lastModified=ZuluTime(stat(filename).st_mtime).rfc1123(),
)
def paths(self):
return list(self._mapping.keys())
def handleRequest(self, path, Headers, **kwargs):
if path in self._mapping:
item = self._mapping[path]
defaultHeaders = [
"Date: {}\r\n".format(ZuluTime().rfc1123()),
'etag: {}\r\n'.format(item['etag']),
'Last-Modified: {}\r\n'.format(item['lastModified'])
]
for key in Headers:
if key.lower() == 'etag' and Headers[key] == item['etag']:
yield "HTTP/1.0 304 Not Modified\r\n"
for each in defaultHeaders:
yield each
yield "\r\n"
return
filename = self._mapping[path]['filename']
_, ext = splitext(filename)
yield 'HTTP/1.0 200 OK\r\n'
yield 'Content-type: {}\r\n'.format(
self._magic.from_file(filename))
for each in defaultHeaders:
yield each
yield '\r\n'
with open(filename, "rb") as fp:
yield fp.read()