def getMimeType(self, handle) :
"""
Get the mime type of a stream by inspecting a fixed number of bytes.
The stream is rewinded after use.
@arg handle: Stream to be inspected. Must be a seekable binary file
object.
@type handle: file object
@return: The mime type of a file and a textual description.
@rtype: unicode, unicode
"""
handle.seek(0)
buf = handle.read(BUFFER_SIZE)
MagicInstance = magic.open(magic.MAGIC_MIME)
MagicInstance.load()
mimeType = MagicInstance.buffer(buf).decode('utf-8').split(';')[0]
MagicInstance.close()
MagicInstance = magic.open(magic.MAGIC_NONE)
MagicInstance.load()
description = MagicInstance.buffer(buf).decode('utf-8')
handle.seek(0)
return mimeType, description
def scandirs(self, path):
import os, glob
exploded = None
filepath = "/".join(self.path)
for filename in glob.glob( os.path.join(path, '*') ):
try:
m = magic.open(magic.MIME_TYPE)
except:
m = magic.open(magic.MAGIC_MIME)
m.load()
if os.path.isdir(filename):
self.scandirs(filename)
if os.path.isfile(filename):
f = file(filename, "r")
content = f.read()
ctype = m.buffer(content)
size = os.path.getsize(filename)
pattern = re.compile(r'.*.(zip|rar|7z).*')
m = pattern.match(ctype)
if m != None:
exploded = Tools().unarchive(content, ctype, os.path.basename(filename))[0]
self.files.extend(exploded)
self.path.append(os.path.basename(filename))
fname = os.path.basename(filename)
if fname != None:
self.files.append({'filename': fname, 'filepath': filepath ,'content_type': ctype, 'size': size})
f.close()
def _get_magic_cookies():
"""
@return: a tuple of magic object.
@rtype: (MAGIC_NONE, MAGIC_COMPRESS, MAGIC_MIME, MAGIC_COMPRESS + MAGIC_MIME)
@note: ... not real magic. Just see: man file(1)
"""
thread_id = get_ident()
if thread_id not in _magic_cookies:
_magic_cookies[thread_id] = {
magic.MAGIC_NONE: magic.open(
magic.MAGIC_NONE),
magic.MAGIC_COMPRESS: magic.open(
magic.MAGIC_COMPRESS),
magic.MAGIC_MIME: magic.open(
magic.MAGIC_MIME),
magic.MAGIC_COMPRESS +
magic.MAGIC_MIME: magic.open(
magic.MAGIC_COMPRESS +
magic.MAGIC_MIME),
magic.MAGIC_MIME_TYPE: magic.open(
magic.MAGIC_MIME_TYPE),
}
for key in _magic_cookies[thread_id].keys():
_magic_cookies[thread_id][key].load()
return _magic_cookies[thread_id]
def traverse_multipart(msg, counter=0, include_attachment_data=False):
attachments = {}
if magic:
ms = magic.open(magic.NONE)
ms.load()
if msg.is_multipart():
for part in msg.get_payload():
attachments.update(traverse_multipart(part, counter, include_attachment_data))
else:
lower_keys = dict((k.lower(), v) for k, v in msg.items())
if 'content-disposition' in lower_keys or not msg.get_content_maintype() == 'text':
# if it's an attachment-type, pull out the filename
# and calculate the size in bytes
data = msg.get_payload(decode=True)
file_size = len(data)
filename = msg.get_filename('')
if filename == '':
filename = 'part-{0:03d}'.format(counter)
else:
filename = decode_field(filename)
extension = get_file_extension(filename)
hash_ = get_file_hash(data)
file_id = str(uuid.uuid1())
attachments[file_id] = {}
attachments[file_id]['filename'] = ascii_decode(filename)
attachments[file_id]['size'] = file_size
if extension:
attachments[file_id]['extension'] = ascii_decode(extension)
attachments[file_id]['hash'] = hash_
if magic:
attachments[file_id]['mime_type'] = ms.buffer(data).decode('utf-8')
# attachments[file_id]['mime_type_short'] = attachments[file_id]['mime_type'].split(",")[0]
ms = magic.open(magic.MAGIC_MIME_TYPE)
ms.load()
attachments[file_id]['mime_type_short'] = ms.buffer(data).decode('utf-8')
if include_attachment_data:
attachments[file_id]['raw'] = base64.b64encode(data)
ch = {}
for k, v in msg.items():
k = ascii_decode(k.lower())
if k in ch:
# print "%s<<<>>>%s" % (k, v)
ch[k].append(ascii_decode(v))
else:
ch[k] = [ascii_decode(v)]
attachments[file_id]['content_header'] = ch
counter += 1
return attachments
def __init__(self, options, template):
self.options = options
self.template = template
self.magic_types = magic.open(magic.MAGIC_NONE)
self.magic_types.load()
self.magic_mime = magic.open(magic.MIME_TYPE)
self.magic_mime.load()
self.abort = False
def _getMimeType(fileName):
global _mimeType
if not _mimeType:
if hasattr(magic, "MIME_TYPE"):
_mimeType = magic.open(magic.MIME_TYPE)
else:
_mimeType = magic.open(magic.MAGIC_NONE)
_mimeType.setflags(magic.MAGIC_MIME)
_mimeType.load()
return _mimeType.file(fileName)
def __init__(self):
if not LibMagic.magic:
LibMagic.magic = magic.open(magic.MAGIC_CONTINUE)
if magic.load(LibMagic.magic, config.MAGICFILE) < 0:
raise IOError("Could not open magic file %s" % config.MAGICFILE)
if not LibMagic.mimemagic:
LibMagic.mimemagic=magic.open(magic.MAGIC_MIME | magic.MAGIC_CONTINUE)
if magic.load(LibMagic.mimemagic,config.MAGICFILE) < 0:
raise IOError("Could not open magic file %s" % config.MAGICFILE)
def __init__(self):
self.vfs = vfs.vfs()
self.env = env.env()
self.mime = magic.open(magic.MAGIC_MIME)
self.type = magic.open(magic.MAGIC_NONE)
if os.name == "nt":
self.mime.load("./api/magic/magic")
self.type.load("./api/magic/magic")
else:
self.mime.load()
self.type.load()
def _find_mime(self):
""" returns the mime encoding and type of a file """
mime = magic.open(magic.MIME_TYPE)
mime.load()
type_ = mime.file(self.sources_path)
mime.close()
mime = magic.open(magic.MIME_ENCODING)
mime.load()
encoding = mime.file(self.sources_path)
mime.close()
return dict(encoding=encoding, type=type_)
def __init__(self, data, filepath = False):
if filepath:
self.message = email.message_from_string(open(filepath).read())
else:
self.message = email.message_from_string(data)
# loading magic
try:
self.magic = magic.open(magic.MIME_TYPE)
except:
self.magic = magic.open(magic.MAGIC_MIME)
self.magic.load()
def get_type(self):
"""Get MIME file type.
@return: file type.
"""
file_type = None
if HAVE_MAGIC:
try:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
file_type = ms.file(self.file_path)
except:
try:
file_type = magic.from_file(self.file_path)
except:
pass
finally:
try:
ms.close()
except:
pass
if file_type is None:
try:
p = subprocess.Popen(["file", "-b", self.file_path],
stdout=subprocess.PIPE)
file_type = p.stdout.read().strip()
except:
pass
return file_type
def filetype(data): try: ms=magic.open(magic.MAGIC_NONE) ms.load() return ms.buffer(data) except: return magic.from_buffer(data)
def guess_encoding(blob):
"""
uses file magic to determine the encoding of the given data blob.
:param blob: file content as read by file.read()
:type blob: data
:returns: encoding
:rtype: str
"""
# this is a bit of a hack to support different versions of python magic.
# Hopefully at some point this will no longer be necessary
#
# the version with open() is the bindings shipped with the file source from
# http://darwinsys.com/file/ - this is what is used by the python-magic
# package on Debian/Ubuntu. However it is not available on pypi/via pip.
#
# the version with from_buffer() is available at
# https://github.com/ahupp/python-magic and directly installable via pip.
#
# for more detail see https://github.com/pazz/alot/pull/588
if hasattr(magic, 'open'):
m = magic.open(magic.MAGIC_MIME_ENCODING)
m.load()
return m.buffer(blob)
elif hasattr(magic, 'from_buffer'):
m = magic.Magic(mime_encoding=True)
return m.from_buffer(blob)
else:
raise Exception('Unknown magic API')
def get_content_type(self):
"""Get MIME content file type (example: image/jpeg).
@return: file content type.
"""
file_type = None
if HAVE_MAGIC:
try:
ms = magic.open(magic.MAGIC_MIME)
ms.load()
file_type = ms.file(self.file_path)
except:
try:
file_type = magic.from_file(self.file_path, mime=True)
except:
pass
finally:
try:
ms.close()
except:
pass
if file_type is None:
try:
args = ["file", "-b", "--mime-type", self.file_path]
file_type = subprocess.check_output(args).strip()
except:
pass
return file_type
def get_fstype(blockdev, fst_list, seek=0):
f = open(blockdev, "rb")
f.seek(seek, os.SEEK_SET)
buf = f.read(1024**2) #2 MB
f.close()
#See if we have the native magic module for python
try:
import magic
m = magic.open(magic.MAGIC_RAW)
m.load()
magictext = m.buffer(buf)
#if not, use the file binary
except ImportError as err:
print "Can't find 'magic' module. Using 'file' binary"
import subprocess
p = subprocess.Popen(['file', '-'],
stdout = subprocess.PIPE, stdin=subprocess.PIPE)
stdout, stderr = p.communicate(input=buf)
magictext = stdout
#print "DEBUG:", magictext
#Search the fs type list for a matching type, and return it
for fst in fst_list:
if fst.magicmatch(magictext):
return fst
return None
def _mime_detector(self):
mime_detector = getattr(self._local, 'mime_detector', None)
if mime_detector is None:
mime_detector = magic.open(magic.MAGIC_MIME_TYPE)
mime_detector.load()
self._local.mime_detector = mime_detector
return self._local.mime_detector
def get_binary_extension(
self, cr, uid, model, ids, binary_field, filename_field=None,
context=None):
result = {}
for this in self.pool[model].browse(
cr, uid,
ids if isinstance(ids, collections.Iterable) else [ids],
context=context):
if not this.id:
result[this.id] = False
continue
extension = ''
if filename_field and this[filename_field]:
filename, extension = os.path.splitext(this[filename_field])
if not this[binary_field]:
result[this.id] = False
continue
if not extension:
try:
import magic
ms = magic.open(
hasattr(magic, 'MAGIC_MIME_TYPE') and
magic.MAGIC_MIME_TYPE or magic.MAGIC_MIME)
ms.load()
mimetype = ms.buffer(
base64.b64decode(this[binary_field]))
except ImportError:
(mimetype, encoding) = mimetypes.guess_type(
'data:;base64,' + this[binary_field], strict=False)
extension = mimetypes.guess_extension(
mimetype.split(';')[0], strict=False)
result[this.id] = (extension or '').lstrip('.').lower()
return result if isinstance(ids, collections.Iterable) else result[ids]
def upload_archive(self, data_collected, duration, cluster=None):
"""
Do an HTTPS Upload of the archive
"""
file_name = os.path.basename(data_collected)
try:
import magic
m = magic.open(magic.MAGIC_MIME)
m.load()
mime_type = m.file(data_collected)
except ImportError:
magic = None
logger.debug('python-magic not installed, using backup function...')
from utilities import magic_plan_b
mime_type = magic_plan_b(data_collected)
files = {
'file': (file_name, open(data_collected, 'rb'), mime_type)}
if cluster:
upload_url = self.upload_url + '/' + cluster
else:
upload_url = self.upload_url + '/' + generate_machine_id()
logger.debug("Uploading %s to %s", data_collected, upload_url)
headers = {'x-rh-collection-time': duration}
upload = self.session.post(upload_url, files=files, headers=headers)
logger.debug("Upload status: %s %s %s",
upload.status_code, upload.reason, upload.text)
logger.debug("Upload duration: %s", upload.elapsed)
return upload
def _get_source_filetype(self):
"""
Set the source filetype. First it tries to use magic and
if import error it will just use the extension
"""
if not hasattr(self, '_source_filetype'):
if not isinstance(self.source, basestring):
# Assuming a file-like object - we won't know it's type.
return None
try:
import magic
except ImportError:
self._source_filetype = splitext(self.source)[1].lower().\
replace('.', '').replace('jpeg', 'jpg')
else:
if hasattr(magic, 'from_file'):
# Adam Hupp's ctypes-based magic library
ftype = magic.from_file(self.source)
else:
# Brett Funderburg's older python magic bindings
m = magic.open(magic.MAGIC_NONE)
m.load()
ftype = m.file(self.source)
if ftype.find('Microsoft Office Document') != -1:
self._source_filetype = 'doc'
elif ftype.find('PDF document') != -1:
self._source_filetype = 'pdf'
elif ftype.find('JPEG') != -1:
self._source_filetype = 'jpg'
else:
self._source_filetype = ftype
return self._source_filetype
def get_files_types(self):
"""
Return the files inside the APK with their types (by using python-magic)
"""
try:
import magic
except ImportError:
return {}
l = {}
builtin_magic = 0
try:
getattr(magic, "Magic")
except AttributeError:
builtin_magic = 1
if builtin_magic:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
for i in self.get_files():
l[ i ] = ms.buffer(self.zip.read(i))
else:
m = magic.Magic()
for i in self.get_files():
l[ i ] = m.from_buffer(self.zip.read(i))
return l
def guess_mimetype(blob):
"""
uses file magic to determine the mime-type of the given data blob.
:param blob: file content as read by file.read()
:type blob: data
:returns: mime-type, falls back to 'application/octet-stream'
:rtype: str
"""
mimetype = 'application/octet-stream'
# this is a bit of a hack to support different versions of python magic.
# Hopefully at some point this will no longer be necessary
#
# the version with open() is the bindings shipped with the file source from
# http://darwinsys.com/file/ - this is what is used by the python-magic
# package on Debian/Ubuntu. However, it is not available on pypi/via pip.
#
# the version with from_buffer() is available at
# https://github.com/ahupp/python-magic and directly installable via pip.
#
# for more detail see https://github.com/pazz/alot/pull/588
if hasattr(magic, 'open'):
m = magic.open(magic.MAGIC_MIME_TYPE)
m.load()
magictype = m.buffer(blob)
elif hasattr(magic, 'from_buffer'):
magictype = magic.from_buffer(blob, mime=True)
else:
raise Exception('Unknown magic API')
# libmagic does not always return proper mimetype strings, cf. issue #459
if re.match(r'\w+\/\w+', magictype):
mimetype = magictype
return mimetype
def get_filetype(file_path):
"""
Get file format identifier based on the type of the given file.
@param file_path: file path
@return: file type identifier or magic signature if format is not supported
"""
log = logging.getLogger("Core.GetFileType")
if not os.path.exists(file_path):
return None
data = open(file_path, "rb").read()
# Thanks to Jesse from malc0de.com for this suggestion.
# First try official magic bindings, if something fails try to failover
# on the unofficial bindings.
try:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
file_type = ms.buffer(data)
except:
try:
file_type = magic.from_buffer(data)
except Exception, why:
log.error("Something went wrong while retrieving magic: %s" % why)
return None
def _find_dependencies(root, part_files):
ms = magic.open(magic.NONE)
if ms.load() != 0:
raise RuntimeError('Cannot load magic header detection')
elf_files = set()
fs_encoding = sys.getfilesystemencoding()
for part_file in part_files:
# Filter out object (*.o) files-- we only care about binaries.
if part_file.endswith('.o'):
continue
# No need to crawl links-- the original should be here, too.
path = os.path.join(root, part_file)
if os.path.islink(path):
logger.debug('Skipped link {!r} while finding dependencies'.format(
path))
continue
path = path.encode(fs_encoding, errors='surrogateescape')
# Finally, make sure this is actually an ELF before queueing it up
# for an ldd call.
file_m = ms.file(path)
if file_m.startswith('ELF') and 'dynamically linked' in file_m:
elf_files.add(path)
dependencies = []
for elf_file in elf_files:
dependencies += libraries.get_dependencies(elf_file)
return set(dependencies)
def get_mime_type(filename):
"""
Get file mime type as tuple of main and sub type.
"""
if hasattr(magic, 'from_file'):
# new magic library
mime_type = magic.from_file(filename, mime=True)
elif hasattr(magic, 'open'):
# old magic library
m_cookie = magic.open(magic.MAGIC_MIME)
m_cookie.load()
mime_type = m_cookie.file(filename)
m_cookie.close()
if mime_type:
mime_type_parts = mime_type.split(";")[0].split()[0]
mime_type_parts = mime_type_parts.split('/')
main_type = mime_type_parts[0]
sub_type = ''
if len(mime_type_parts) > 1 and len(mime_type_parts[1]):
sub_type = mime_type_parts[1]
return (main_type, sub_type)
return (u'application', u'octet-stream') # this is the default
def get_type(self):
"""Get MIME file type.
@return: file type.
"""
try:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
file_type = ms.file(self.file_path)
except:
try:
file_type = magic.from_file(self.file_path)
except:
try:
import subprocess
file_process = subprocess.Popen(["file",
"-b",
self.file_path],
stdout = subprocess.PIPE)
file_type = file_process.stdout.read().strip()
except:
return ""
finally:
try:
ms.close()
except:
pass
return file_type
def get_magic(file_name):
""" Determine the file type of a given file based on its magic result."""
log = logging.getLogger('Mastiff.FileType.Magic')
try:
# try to use magic from the file source code
magic_ = magic.open(magic.MAGIC_NONE)
magic_.load()
try:
file_type = magic_.file(file_name)
except:
log.error('Could not determine magic file type.')
return None
magic_.close()
except AttributeError:
# Now we are trying ahupps magic library
try:
file_type = magic.from_file(file_name)
except AttributeError:
log.error('No valid magic libraries installed.')
return None
except MagicException:
log.error('Cound not determing magic file type.')
return None
log.debug('Magic file type is "%s"', file_type)
return file_type
def _find_dependencies(workdir):
ms = magic.open(magic.NONE)
if ms.load() != 0:
raise RuntimeError('Cannot load magic header detection')
elf_files = set()
fs_encoding = sys.getfilesystemencoding()
for root, dirs, files in os.walk(workdir.encode(fs_encoding)):
# Filter out object (*.o) files-- we only care about binaries.
entries = (entry for entry in itertools.chain(files, dirs)
if not entry.endswith(b'.o'))
for entry in entries:
path = os.path.join(root, entry)
if os.path.islink(path):
logger.debug('Skipped link {!r} when parsing {!r}'.format(
path, workdir))
continue
file_m = ms.file(path)
if file_m.startswith('ELF') and 'dynamically linked' in file_m:
elf_files.add(path)
dependencies = []
for elf_file in elf_files:
dependencies += libraries.get_dependencies(elf_file)
return set(dependencies)
def download(self, track):
uri = track.stream #.replace('mp31', 'mp32')
if uri in self.retries:
if self.retries[uri]>9:
return
time.sleep(self.retries[uri]**2)
tmp = os.path.join(functions.install_dir(), "cache")
if not os.path.isdir(tmp):
os.mkdir(tmp)
if not os.path.isdir(os.path.join(tmp, str(track.artist_id))):
os.mkdir(os.path.join(tmp, str(track.artist_id)))
tmp = os.path.join(tmp, str(track.artist_id))
if not os.path.isdir(os.path.join(tmp, str(track.album_id))):
os.mkdir(os.path.join(tmp, str(track.album_id)))
tmp = os.path.join(tmp, str(track.album_id))
target = os.path.join(tmp, str(track.id)) + '.mp3'
#print "[!] downloading %s to %s" % (uri, target)
cmd='(wget -c -O %s.part -q "%s" && mv %s.part %s)' % (target, uri, target, target)
#print "[!] command:", cmd
os.system(cmd)
mp3 = magic.open(magic.MAGIC_NONE)
mp3.load()
type = mp3.file(target)
print uri
print 'file type',type
if type[:4] == 'MPEG' or type[:5] == 'Audio':
track.local = target
else:
os.unlink(target)
self.retries[uri]=self.retries.get(uri,0)+1
print "[!] failed to download %s, putting back in queue (retry:%d)" % (target,self.retries[uri])
self.queue_push(track)
def safe_init(self, options):
self.certsdir = options.configuration.get('openssl', 'certsdir')
if not os.path.exists(self.certsdir):
os.makedirs(self.certsdir)
self.m = magic.open(magic.MAGIC_NONE)
self.m.load('/usr/share/file/magic.mgc')
def redebug(patch_path, source_path):
# parse arguments
start_time = time.time()
# initialize a magic cookie pointer
common.magic_cookie = magic.open(magic.MAGIC_MIME)
common.magic_cookie.load()
ret =[]
# traverse patch files
patch = patchloader.PatchLoader()
npatch = patch.traverse(patch_path)
ret.append(npatch)
# traverse source files
source = sourceloader.SourceLoader()
nmatch = source.traverse(source_path, patch)
ret.append(nmatch)
# generate a report
report = reporter.Reporter(patch, source)
exact_nmatch, html = report.output()
ret.append(exact_nmatch)
ret.append(html)
common.magic_cookie.close()
elapsed_time = time.time() - start_time
#print '[+] %d matches given %d patches ... %.1fs' % (exact_nmatch, npatch, elapsed_time)
ret.append(elapsed_time)
return ret
def __init__(self, root='/'):
"""Initializes the internal state.
Args:
root: Path to the root directory where all the files live. This will be
assumed as the root directory for absolute symlinks.
"""
self._root = root
self._mime = magic.open(magic.MIME_TYPE)
self._mime.load()
def is_sparse_image(image):
"""
Returns True if the image is an 'Android sparse image' else False.
"""
image_magic = magic.open(magic.MAGIC_NONE)
image_magic.load()
if image_magic.file(image).split(',')[0] == 'Android sparse image':
return True
else:
return False
def sample_type(buf):
if not has_magic:
return None
global _magic_cookie
if _magic_cookie is None:
_magic_cookie = magic.open(magic.MAGIC_MIME)
_magic_cookie.load()
typ = _magic_cookie.buffer(buf)
if typ is None: return ''
return typ.split(';')[0]
def fileinfo(fname):
"""Return fileinfo dict for given file."""
m = magic.open(magic.MAGIC_MIME_TYPE)
m.load()
mimetype = m.file(fname)
m.close()
m = magic.open(magic.MAGIC_MIME_ENCODING)
m.load()
charset = m.file(fname)
m.close()
m = magic.open(magic.MAGIC_NONE)
m.load()
version = m.file(fname).split("version ")[-1]
m.close()
fileinfo = {
'filename': fname,
'format': {
'mimetype': mimetype,
'version': version,
'charset': charset
}
}
if mimetype in [
'text/plain', 'text/csv', 'application/xhtml+xml', 'text/xml',
'text/html', 'application/gml+xml',
'application/vnd.google-earth.kml+xml'
]:
fileinfo['format'][
'charset'] = 'UTF-8' if 'UTF-8' in charset else 'ISO-8859-15'
else:
del fileinfo['format']['charset']
if mimetype in ['text/plain', 'text/csv']:
fileinfo['format']['version'] = ''
if mimetype == 'image/tiff':
fileinfo['format']['version'] = '6.0'
return fileinfo
def _get_type(self):
if not IS_MAGIC:
return None
try:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
return ms.buffer(self.file_data)
except:
return None
def _get_file_type(full_targ_path):
# This function takes the full path of a target sample and determines/returns the file type via python-magic.
try:
magicObj = magic.open(magic.MAGIC_NONE)
magicObj.load()
magic_out = str(magicObj.file(full_targ_path))
except AttributeError:
magic_out = str(magic.from_file(full_targ_path))
return(magic_out)
def _GetCookie(self):
magic_cookie = magic.open(self.cookie_count)
self.cookie_count += 1
magic_cookie.load()
if "MAGIC_MIME" in dir(magic):
flag = magic.MAGIC_MIME
elif "MIME" in dir(magic):
flag = magic.MIME
magic_cookie.setflags(flag)
return magic_cookie
def magic(indata, mime=False):
"""
Performs file magic while maintaining compatibility with different libraries.
"""
try:
if mime:
mymagic = magic.open(magic.MAGIC_MIME_TYPE)
else:
mymagic = magic.open(magic.MAGIC_NONE)
mymagic.load()
except AttributeError:
mymagic = magic.Magic(mime)
mymagic.file = mymagic.from_file
try:
return mymagic.file(indata)
except magic.MagicException:
return None
def checkEncoding(self):
url = urllib.parse.urlparse(self.pageUrl)
blob = open(url.path, 'rb').read()
m = magic.open(magic.MAGIC_MIME_ENCODING)
m.load()
encoding = m.buffer(blob)
logging.debug("file encoding: %s", encoding)
if encoding != "utf-8" and encoding != "us-ascii":
return f' [{encoding}]'
return ''
def __init__(self, archivo):
__mime = magic.open(magic.MAGIC_MIME)
__mime.load()
f = NamedTemporaryFile(delete=False)
self.name = f.name
with f.file as destination:
for chunk in archivo.chunks():
destination.write(chunk)
self.mime_type = __mime.file(self.name)
self.file = f.file
def get_good_mime(music_file):
mime = Mime.get_type_by_contents(music_file)
if mime == None: # try magic instead
mime = magic.open(magic.MAGIC_MIME)
mime.load()
mime = mime.file(music_file)
mime = mime.split(";")[0]
else:
mime = str(mime)
return mime
def get_filetype(data):
"""There are two versions of python-magic floating around, and annoyingly, the interface
changed between versions, so we try one method and if it fails, then we try the other"""
if sys.modules.has_key('magic'):
try:
ms = magic.open(magic.MAGIC_NONE)
ms.load()
return ms.buffer(data)
except:
return magic.from_buffer(data)
def scrape_file(self):
"""Scrape text file."""
if not self._check_wellformed and self._only_wellformed:
self.messages('Skipping scraper: Well-formed check not used.')
self._collect_elements()
return
if not os.path.exists(self.filename):
self.errors('File not found.')
self._collect_elements()
return
try:
magic_ = magic.open(magic.MAGIC_MIME_TYPE)
magic_.load()
self._magic_mimetype = magic_.file(encode(self.filename))
magic_.close()
magic_ = magic.open(magic.MAGIC_NONE)
magic_.load()
self._magic_version = magic_.file(
encode(self.filename)).split(self._starttag)[-1]
magic_.close()
if self._endtag:
self._magic_version = self._magic_version.split(
self._endtag)[0]
magic_ = magic.open(magic.MAGIC_MIME_ENCODING)
magic_.load()
self._magic_charset = magic_.file(encode(self.filename))
magic_.close()
if self._mimetype() == self.mimetype or \
(self._mimetype() in MIMETYPE_DICT and
MIMETYPE_DICT[self._mimetype()] == self.mimetype):
self.messages('The file was analyzed successfully.')
else:
self.errors('Given mimetype %s and detected mimetype %s do '
'not match.' % (self.mimetype, self._mimetype()))
except Exception as e: # pylint: disable=invalid-name, broad-except
self.errors('Error in analyzing file.')
self.errors(str(e))
finally:
self._check_supported()
self._collect_elements()
def get_img_base64(img_binary):
'''返回用于img标签的src的base64字串'''
import base64
import magic
ms = magic.open(magic.MAGIC_MIME)
ms.load()
try:
mime_type = ms.buffer(img_binary).split(' ', 1)[0]
except Exception:
mime_type = 'image/png;'
return 'data:%sbase64,%s' % (mime_type, base64.b64encode(img_binary))
def check_executable(self, file):
# Initialize the PE test looking for "magic patterns"
ms = magic.open(magic.MAGIC_NONE)
ms.load()
ftype = ms.file(file)
ms.close()
self.logger.log_console("Filetype: %s" % ftype, "debug")
if ftype == "PE32 executable for MS Windows (GUI) Intel 80386 32-bit":
return True
else:
False
def magic_cookie(self):
if not self._magic_cookie:
self._magic_cookie = magic.open(self.cookie_count)
self.cookie_count += 1
self._magic_cookie.load()
if "MAGIC_MIME" in dir(magic):
flag = magic.MAGIC_MIME
elif "MIME" in dir(magic):
flag = magic.MIME
self._magic_cookie.setflags(flag)
return self._magic_cookie
def guess_type(self, file):
try:
import magic
guesser = magic.open(magic.MIME_TYPE)
guesser.load()
mime_type = guesser.buffer(file.read(1024))
file.seek(0)
return mime_type or 'application/octet-stream'
except:
return 'application/octet-stream'
def __get_media_type(path):
"""
Get the mime type of a file.
:param path: Path to the file
:return: mime type as string
"""
m = magic.open(magic.MAGIC_MIME_TYPE)
m.load()
mime = m.file(path)
m.close()
return mime
def get_encoding(filepath):
encoding = None
blob = open(filepath, 'rb').read()
try:
m = magic.open(magic.MAGIC_MIME_ENCODING)
m.load()
encoding = m.buffer(blob) # "utf-8" "us-ascii" etc
except Exception as e:
m = magic.Magic(mime_encoding=True)
encoding = m.from_buffer(blob)
return encoding
def __init__(self, library_dir, index_class, database_class):
super(DigitalLibrary, self).__init__()
if not os.path.isdir(library_dir):
os.makedirs(library_dir)
self._index = index_class(os.path.join(library_dir, 'index'))
self._database = database_class(os.path.join(library_dir, 'database.db'))
self._dir_levels = 3
self._documents_dir = os.path.join(library_dir, 'documents')
self._thumbnails_dir = os.path.join(library_dir, 'thumbnails')
self._magic = magic.open(magic.MAGIC_MIME_TYPE | magic.MAGIC_NO_CHECK_TOKENS)
self._magic.load()
def _get_magic(filename):
msc = magic.open(magic.NONE)
msc.load()
try:
result = msc.file(filename)
except UnicodeDecodeError:
result = None
finally:
msc.close()
return result
def __init__(self, pkpass):
# house keeping
self.mimeHelper = magic.open(magic.MAGIC_MIME)
self.mimeHelper.load()
self.file = pkpass
self.fileStruct = {
"info": "pass.json",
"sig": "signature",
"manifest": "manifest.json"
}
def __init__(self, parent=None):
super(self.__class__, self).__init__(parent)
self.setupUi(self)
self.centerOnScreen()
self.BTN_SelFolder.clicked.connect(self.browse_folder)
self.BTN_Process.clicked.connect(self.process_folder)
self.actionAuthors.triggered.connect(self.show_authors)
self.form2 = ProgressClass(self)
self.form3 = Display(self)
self.magic = magic.open(magic.MAGIC_MIME)
self.magic.load()
def guess_encoding(blob):
"""
uses file magic to determine the encoding of the given data blob.
:param blob: file content as read by file.read()
:type blob: data
:returns: encoding
:rtype: str
"""
m = magic.open(magic.MAGIC_MIME_ENCODING)
m.load()
return m.buffer(blob)
def _upload_media(self, media_path, post_id=None):
data = {}
if post_id:
data["post_id"] = post_id
data["name"] = os.path.basename(media_path)
magician = magic.open(magic.MIME_TYPE | magic.SYMLINK)
magician.load()
data["type"] = magician.file(media_path)
with open(media_path, "rb") as file:
data["bits"] = wordpress_xmlrpc.compat.xmlrpc_client.Binary(
file.read())
return self._call(wordpress_xmlrpc.methods.media.UploadFile(data))
def check_pcap(pcap):
try:
bad_magic = 'pcap-ng capture file'
m = magic.open(magic.MAGIC_NONE)
m.load()
f = m.file(pcap)
if bad_magic in f:
return 'BAD'
else:
return f
except Exception as e:
return str(e)
def is_text_file(path):
m = magic.open(magic.MAGIC_MIME)
m.load()
r = m.file(path)
log.debug(
_("Magic type of file %(path)s is %(type)s") % {
'path': path,
'type': r
})
if r.startswith('text') or r.startswith('application/json'):
return True
return False
def __init__(self):
Gtk.VBox.__init__(self)
self.core=Core.Core.get_core()
self.void = True
self.background_img = None
self.set_background()
self.drag_dest_set(Gtk.DestDefaults.ALL, [], Gdk.DragAction.COPY)
self.connect("drag-data-received",self.drag_files)
self.drag_dest_add_text_targets()
self.mime = magic.open(magic.MAGIC_MIME)
self.mime.load()
self.set_name("convertbox")
def batch(flist, newdir, dry=False, ocr_dir=None):
"""Find a name and a date for each file in [flist]
and copy then into [newdir] (if [dry] is [False]). If ocr_dir is not
specified, a new tmp dir will be created.
"""
global OCR_DIR, CUSTOM_OCR_DIR
if ocr_dir is None:
OCR_DIR = tempfile.mkdtemp(prefix='rbc-ocr_')
CUSTOM_OCR_DIR = False
print("Notice: Will save text data in " + OCR_DIR)
else:
if ocr_dir != OCR_DIR:
OCR_DIR = ocr_dir
CUSTOM_OCR_DIR = True
print("Notice: Will use/save text data from/to " + OCR_DIR)
created = []
not_treated = []
remaining = list(flist) # we make a copy
et = exiftool.ExifTool()
mg = magic.open(magic.NONE)
if not mg.load() == 0:
print("RBC ERROR: Cannot use Magic.")
et.start()
i = 0
n = len(flist)
for filename in flist:
i += 1
print("")
print(
"----------------------------------------------------------------")
print("---(%u/%u)--- Processing " % (i, n) + filename)
print(
"----------------------------------------------------------------")
assert (remaining.pop(0) == filename)
if os.path.isfile(filename):
#try:
newfile, title = rename(et, mg, filename, newdir, dry)
#except:
# print ("---- Unexpected error:", sys.exc_info())
# print ("Quitting")
# return (created, remaining)
created.append([filename, newfile, title])
else:
not_treated.append(filename)
if os.path.isdir(filename):
print("WARNING: Skipping directory " + filename)
else:
print("WARNING: Skipping " + filename)
et.terminate()
return (created, not_treated + remaining)
def check_file(filename):
try:
try:
m = magic.open(magic.MAGIC_NONE)
m.load()
f = m.file(filename)
return f
except Exception:
m = magic.from_file(filename)
return m
except Exception as e:
return str(e)
